@balena/pinejs 16.2.0-build-joshbwlng-tasks-a83c83b4c78803915d0cb6297cc8cc1862622d08-1 → 17.0.0-build-wip-large-file-uploads-0c8ef752deac19fa0d6a7dfa9f7173813cab7867-1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pinejs-cache.json +1 -1
- package/.versionbot/CHANGELOG.yml +7 -7
- package/CHANGELOG.md +3 -3
- package/VERSION +1 -1
- package/out/config-loader/env.d.ts +0 -4
- package/out/config-loader/env.js +1 -5
- package/out/config-loader/env.js.map +1 -1
- package/out/database-layer/db.d.ts +0 -3
- package/out/database-layer/db.js +0 -14
- package/out/database-layer/db.js.map +1 -1
- package/out/migrator/utils.js +2 -2
- package/out/migrator/utils.js.map +1 -1
- package/out/sbvr-api/sbvr-utils.d.ts +0 -1
- package/out/sbvr-api/sbvr-utils.js +1 -6
- package/out/sbvr-api/sbvr-utils.js.map +1 -1
- package/out/server-glue/module.d.ts +0 -1
- package/out/server-glue/module.js +3 -2
- package/out/server-glue/module.js.map +1 -1
- package/out/webresource-handler/handlers/NoopHandler.d.ts +3 -0
- package/out/webresource-handler/handlers/NoopHandler.js +6 -0
- package/out/webresource-handler/handlers/NoopHandler.js.map +1 -1
- package/out/webresource-handler/handlers/S3Handler.d.ts +7 -0
- package/out/webresource-handler/handlers/S3Handler.js +68 -2
- package/out/webresource-handler/handlers/S3Handler.js.map +1 -1
- package/out/webresource-handler/index.d.ts +10 -1
- package/out/webresource-handler/index.js +25 -10
- package/out/webresource-handler/index.js.map +1 -1
- package/out/webresource-handler/multipartUpload.d.ts +39 -0
- package/out/webresource-handler/multipartUpload.js +187 -0
- package/out/webresource-handler/multipartUpload.js.map +1 -0
- package/out/webresource-handler/webresource.sbvr +62 -0
- package/package.json +4 -7
- package/src/config-loader/env.ts +1 -6
- package/src/database-layer/db.ts +0 -24
- package/src/migrator/utils.ts +1 -1
- package/src/sbvr-api/sbvr-utils.ts +1 -5
- package/src/server-glue/module.ts +2 -1
- package/src/webresource-handler/handlers/NoopHandler.ts +21 -0
- package/src/webresource-handler/handlers/S3Handler.ts +130 -4
- package/src/webresource-handler/index.ts +46 -11
- package/src/webresource-handler/multipartUpload.ts +286 -0
- package/src/webresource-handler/webresource.sbvr +62 -0
- package/out/tasks/common.d.ts +0 -4
- package/out/tasks/common.js +0 -13
- package/out/tasks/common.js.map +0 -1
- package/out/tasks/index.d.ts +0 -10
- package/out/tasks/index.js +0 -139
- package/out/tasks/index.js.map +0 -1
- package/out/tasks/model.sbvr +0 -60
- package/out/tasks/types.d.ts +0 -38
- package/out/tasks/types.js +0 -10
- package/out/tasks/types.js.map +0 -1
- package/out/tasks/worker.d.ts +0 -16
- package/out/tasks/worker.js +0 -191
- package/out/tasks/worker.js.map +0 -1
- package/src/tasks/common.ts +0 -14
- package/src/tasks/index.ts +0 -158
- package/src/tasks/model.sbvr +0 -60
- package/src/tasks/types.ts +0 -58
- package/src/tasks/worker.ts +0 -246
@@ -0,0 +1,286 @@
|
|
1
|
+
import type { WebResourceType as WebResource } from '@balena/sbvr-types';
|
2
|
+
import { randomUUID } from 'node:crypto';
|
3
|
+
import type { AnyObject } from 'pinejs-client-core';
|
4
|
+
import type { WebResourceHandler } from '.';
|
5
|
+
import { getWebResourceFields } from '.';
|
6
|
+
import { api } from '../sbvr-api/sbvr-utils';
|
7
|
+
import type { ODataRequest } from '../sbvr-api/uri-parser';
|
8
|
+
import { errors, permissions, sbvrUtils } from '../server-glue/module';
|
9
|
+
|
10
|
+
export interface BeginUploadPayload {
|
11
|
+
filename: string;
|
12
|
+
content_type: string;
|
13
|
+
size: number;
|
14
|
+
chunk_size: number;
|
15
|
+
}
|
16
|
+
|
17
|
+
type BeginUploadDbCheck = BeginUploadPayload & WebResource;
|
18
|
+
|
19
|
+
export interface UploadUrl {
|
20
|
+
url: string;
|
21
|
+
chunkSize: number;
|
22
|
+
partNumber: number;
|
23
|
+
}
|
24
|
+
|
25
|
+
export interface BeginUploadHandlerResponse {
|
26
|
+
uploadUrls: UploadUrl[];
|
27
|
+
fileKey: string;
|
28
|
+
uploadId: string;
|
29
|
+
}
|
30
|
+
|
31
|
+
export interface PendingUpload extends BeginUploadPayload {
|
32
|
+
fieldName: string;
|
33
|
+
fileKey: string;
|
34
|
+
uploadId: string;
|
35
|
+
}
|
36
|
+
|
37
|
+
export interface BeginUploadResponse {
|
38
|
+
[fieldName: string]: {
|
39
|
+
key: string;
|
40
|
+
uploadUrls: UploadUrl[];
|
41
|
+
};
|
42
|
+
}
|
43
|
+
export interface CommitUploadHandlerPayload {
|
44
|
+
fileKey: string;
|
45
|
+
uploadId: string;
|
46
|
+
filename: string;
|
47
|
+
multipartUploadChecksums?: AnyObject;
|
48
|
+
}
|
49
|
+
|
50
|
+
const MB = 1024 * 1024;
|
51
|
+
|
52
|
+
export const multipartUploadHooks = (
|
53
|
+
webResourceHandler: WebResourceHandler,
|
54
|
+
): sbvrUtils.Hooks => {
|
55
|
+
return {
|
56
|
+
POSTPARSE: async ({ req, request, tx }) => {
|
57
|
+
if (request.odataQuery.property?.resource === 'beginUpload') {
|
58
|
+
const uploadParams = parseBeginUpload(request);
|
59
|
+
|
60
|
+
await sbvrUtils.api[request.vocabulary].post({
|
61
|
+
url: request.url.substring(1).replace('beginUpload', 'canAccess'),
|
62
|
+
body: { method: 'PATCH' },
|
63
|
+
});
|
64
|
+
|
65
|
+
// This transaction is necessary because beginUpload requests
|
66
|
+
// will rollback the transaction (in order to first validate)
|
67
|
+
// The metadata requested. If we don't pass any transaction
|
68
|
+
// It will use the default transaction handler which will error out
|
69
|
+
// on any rollback.
|
70
|
+
tx = await sbvrUtils.db.transaction();
|
71
|
+
req.tx = tx;
|
72
|
+
request.tx = tx;
|
73
|
+
|
74
|
+
request.method = 'PATCH';
|
75
|
+
request.values = uploadParams;
|
76
|
+
request.odataQuery.resource = request.resourceName;
|
77
|
+
delete request.odataQuery.property;
|
78
|
+
request.custom.isAction = 'beginUpload';
|
79
|
+
} else if (request.odataQuery.property?.resource === 'commitUpload') {
|
80
|
+
const commitPayload = await parseCommitUpload(request);
|
81
|
+
|
82
|
+
await sbvrUtils.api[request.vocabulary].post({
|
83
|
+
url: request.url.substring(1).replace('commitUpload', 'canAccess'),
|
84
|
+
body: { method: 'PATCH' },
|
85
|
+
});
|
86
|
+
|
87
|
+
const webresource = await webResourceHandler.commitUpload({
|
88
|
+
fileKey: commitPayload.metadata.fileKey,
|
89
|
+
uploadId: commitPayload.metadata.uploadId,
|
90
|
+
filename: commitPayload.metadata.filename,
|
91
|
+
multipartUploadChecksums: commitPayload.additionalCommitInfo,
|
92
|
+
});
|
93
|
+
|
94
|
+
await api.webresource.patch({
|
95
|
+
resource: 'multipart_upload',
|
96
|
+
body: {
|
97
|
+
status: 'completed',
|
98
|
+
},
|
99
|
+
options: {
|
100
|
+
$filter: {
|
101
|
+
uuid: commitPayload.key,
|
102
|
+
},
|
103
|
+
},
|
104
|
+
passthrough: {
|
105
|
+
req: permissions.root,
|
106
|
+
tx: tx,
|
107
|
+
},
|
108
|
+
});
|
109
|
+
|
110
|
+
request.method = 'PATCH';
|
111
|
+
request.values = {
|
112
|
+
[commitPayload.metadata.fieldName]: webresource,
|
113
|
+
};
|
114
|
+
request.odataQuery.resource = request.resourceName;
|
115
|
+
delete request.odataQuery.property;
|
116
|
+
request.custom.isAction = 'commitUpload';
|
117
|
+
request.custom.commitUploadPayload = webresource;
|
118
|
+
}
|
119
|
+
},
|
120
|
+
PRERESPOND: async ({ req, request, response, tx }) => {
|
121
|
+
if (request.custom.isAction === 'beginUpload') {
|
122
|
+
await tx.rollback();
|
123
|
+
|
124
|
+
response.statusCode = 200;
|
125
|
+
response.body = await beginUpload(
|
126
|
+
webResourceHandler,
|
127
|
+
request,
|
128
|
+
req.user?.actor,
|
129
|
+
);
|
130
|
+
} else if (request.custom.isAction === 'commitUpload') {
|
131
|
+
response.body = await webResourceHandler.onPreRespond(
|
132
|
+
request.custom.commitUploadPayload,
|
133
|
+
);
|
134
|
+
}
|
135
|
+
},
|
136
|
+
};
|
137
|
+
};
|
138
|
+
|
139
|
+
export const beginUpload = async (
|
140
|
+
webResourceHandler: WebResourceHandler,
|
141
|
+
odataRequest: ODataRequest,
|
142
|
+
actorId?: number,
|
143
|
+
): Promise<BeginUploadResponse> => {
|
144
|
+
const payload = odataRequest.values as { [x: string]: BeginUploadPayload };
|
145
|
+
const fieldName = Object.keys(payload)[0];
|
146
|
+
const metadata = payload[fieldName];
|
147
|
+
|
148
|
+
const { fileKey, uploadId, uploadUrls } =
|
149
|
+
await webResourceHandler.beginUpload(fieldName, metadata);
|
150
|
+
const uuid = randomUUID();
|
151
|
+
|
152
|
+
try {
|
153
|
+
await api.webresource.post({
|
154
|
+
resource: 'multipart_upload',
|
155
|
+
body: {
|
156
|
+
uuid,
|
157
|
+
resource_name: odataRequest.resourceName,
|
158
|
+
field_name: fieldName,
|
159
|
+
resource_id: odataRequest.affectedIds?.[0],
|
160
|
+
upload_id: uploadId,
|
161
|
+
file_key: fileKey,
|
162
|
+
status: 'pending',
|
163
|
+
filename: metadata.filename,
|
164
|
+
content_type: metadata.content_type,
|
165
|
+
size: metadata.size,
|
166
|
+
chunk_size: metadata.chunk_size,
|
167
|
+
expiry_date: Date.now() + 7 * 24 * 60 * 60 * 1000, // 7 days in ms
|
168
|
+
is_created_by__actor: actorId,
|
169
|
+
},
|
170
|
+
passthrough: {
|
171
|
+
req: permissions.root,
|
172
|
+
},
|
173
|
+
});
|
174
|
+
} catch (err) {
|
175
|
+
console.error('failed to start multipart upload', err);
|
176
|
+
throw new errors.BadRequestError('Failed to start multipart upload');
|
177
|
+
}
|
178
|
+
|
179
|
+
return { [fieldName]: { key: uuid, uploadUrls } };
|
180
|
+
};
|
181
|
+
|
182
|
+
const parseBeginUpload = (request: ODataRequest) => {
|
183
|
+
if (request.odataQuery.key == null) {
|
184
|
+
throw new errors.BadRequestError();
|
185
|
+
}
|
186
|
+
|
187
|
+
const fieldNames = Object.keys(request.values);
|
188
|
+
if (fieldNames.length !== 1) {
|
189
|
+
throw new errors.BadRequestError(
|
190
|
+
'You can only get upload url for one field at a time',
|
191
|
+
);
|
192
|
+
}
|
193
|
+
|
194
|
+
const [fieldName] = fieldNames;
|
195
|
+
const webResourceFields = getWebResourceFields(request, false);
|
196
|
+
if (!webResourceFields.includes(fieldName)) {
|
197
|
+
throw new errors.BadRequestError(
|
198
|
+
`You must provide a valid webresource field from: ${JSON.stringify(webResourceFields)}`,
|
199
|
+
);
|
200
|
+
}
|
201
|
+
|
202
|
+
const beginUploadPayload = parseBeginUploadPayload(request.values[fieldName]);
|
203
|
+
if (beginUploadPayload == null) {
|
204
|
+
throw new errors.BadRequestError('Invalid file metadata');
|
205
|
+
}
|
206
|
+
|
207
|
+
const uploadMetadataCheck: BeginUploadDbCheck = {
|
208
|
+
...beginUploadPayload,
|
209
|
+
href: 'metadata_check',
|
210
|
+
};
|
211
|
+
|
212
|
+
return { [fieldName]: uploadMetadataCheck };
|
213
|
+
};
|
214
|
+
|
215
|
+
const parseBeginUploadPayload = (
|
216
|
+
payload: AnyObject,
|
217
|
+
): BeginUploadPayload | null => {
|
218
|
+
if (typeof payload !== 'object') {
|
219
|
+
return null;
|
220
|
+
}
|
221
|
+
|
222
|
+
let { filename, content_type, size, chunk_size } = payload;
|
223
|
+
if (
|
224
|
+
typeof filename !== 'string' ||
|
225
|
+
typeof content_type !== 'string' ||
|
226
|
+
typeof size !== 'number' ||
|
227
|
+
(chunk_size != null && typeof chunk_size !== 'number') ||
|
228
|
+
(chunk_size != null && chunk_size < 5 * MB)
|
229
|
+
) {
|
230
|
+
return null;
|
231
|
+
}
|
232
|
+
|
233
|
+
if (chunk_size == null) {
|
234
|
+
chunk_size = 5 * MB;
|
235
|
+
}
|
236
|
+
return { filename, content_type, size, chunk_size };
|
237
|
+
};
|
238
|
+
|
239
|
+
const parseCommitUpload = async (request: ODataRequest) => {
|
240
|
+
if (request.odataQuery.key == null) {
|
241
|
+
throw new errors.BadRequestError();
|
242
|
+
}
|
243
|
+
|
244
|
+
const { key, additionalCommitInfo } = request.values;
|
245
|
+
if (typeof key !== 'string') {
|
246
|
+
throw new errors.BadRequestError('Invalid key type');
|
247
|
+
}
|
248
|
+
|
249
|
+
// TODO: actor permissions
|
250
|
+
const [multipartUpload] = (await api.webresource.get({
|
251
|
+
resource: 'multipart_upload',
|
252
|
+
options: {
|
253
|
+
$select: ['id', 'file_key', 'upload_id', 'field_name', 'filename'],
|
254
|
+
$filter: {
|
255
|
+
uuid: key,
|
256
|
+
status: 'pending',
|
257
|
+
expiry_date: { $gt: { $now: {} } },
|
258
|
+
},
|
259
|
+
},
|
260
|
+
passthrough: {
|
261
|
+
req: permissions.root,
|
262
|
+
tx: request.tx,
|
263
|
+
},
|
264
|
+
})) as [
|
265
|
+
{
|
266
|
+
id: number;
|
267
|
+
file_key: string;
|
268
|
+
upload_id: string;
|
269
|
+
field_name: string;
|
270
|
+
filename: string;
|
271
|
+
}?,
|
272
|
+
];
|
273
|
+
|
274
|
+
if (multipartUpload == null) {
|
275
|
+
throw new errors.BadRequestError(`Invalid upload for key ${key}`);
|
276
|
+
}
|
277
|
+
|
278
|
+
const metadata = {
|
279
|
+
fileKey: multipartUpload.file_key,
|
280
|
+
uploadId: multipartUpload.upload_id,
|
281
|
+
filename: multipartUpload.filename,
|
282
|
+
fieldName: multipartUpload.field_name,
|
283
|
+
};
|
284
|
+
|
285
|
+
return { key, additionalCommitInfo, metadata };
|
286
|
+
};
|
@@ -0,0 +1,62 @@
|
|
1
|
+
Vocabulary: Auth
|
2
|
+
|
3
|
+
Term: actor
|
4
|
+
Term: expiry date
|
5
|
+
Concept Type: Date Time (Type)
|
6
|
+
|
7
|
+
Vocabulary: webresource
|
8
|
+
|
9
|
+
Term: uuid
|
10
|
+
Concept Type: Short Text (Type)
|
11
|
+
Term: resource name
|
12
|
+
Concept Type: Short Text (Type)
|
13
|
+
Term: field name
|
14
|
+
Concept Type: Short Text (Type)
|
15
|
+
Term: resource id
|
16
|
+
Concept Type: Integer (Type)
|
17
|
+
Term: upload id
|
18
|
+
Concept Type: Short Text (Type)
|
19
|
+
Term: file key
|
20
|
+
Concept Type: Short Text (Type)
|
21
|
+
Term: status
|
22
|
+
Concept Type: Short Text (Type)
|
23
|
+
Term: filename
|
24
|
+
Concept Type: Short Text (Type)
|
25
|
+
Term: content type
|
26
|
+
Concept Type: Short Text (Type)
|
27
|
+
Term: size
|
28
|
+
Concept Type: Integer (Type)
|
29
|
+
Term: chunk size
|
30
|
+
Concept Type: Integer (Type)
|
31
|
+
Term: valid until date
|
32
|
+
Concept Type: Date Time (Type)
|
33
|
+
|
34
|
+
Term: multipart upload
|
35
|
+
Fact type: multipart upload has uuid
|
36
|
+
Necessity: each multipart upload has exactly one uuid
|
37
|
+
Necessity: each uuid is of exactly one multipart upload
|
38
|
+
Fact type: multipart upload has resource name
|
39
|
+
Necessity: each multipart upload has exactly one resource name
|
40
|
+
Fact type: multipart upload has field name
|
41
|
+
Necessity: each multipart upload has exactly one field name
|
42
|
+
Fact type: multipart upload has resource id
|
43
|
+
Necessity: each multipart upload has exactly one resource id
|
44
|
+
Fact type: multipart upload has upload id
|
45
|
+
Necessity: each multipart upload has exactly one upload id
|
46
|
+
Fact type: multipart upload has file key
|
47
|
+
Necessity: each multipart upload has exactly one file key
|
48
|
+
Fact type: multipart upload has status
|
49
|
+
Necessity: each multipart upload has exactly one status
|
50
|
+
Definition: "pending" or "completed" or "cancelled"
|
51
|
+
Fact type: multipart upload has filename
|
52
|
+
Necessity: each multipart upload has exactly one filename
|
53
|
+
Fact type: multipart upload has content type
|
54
|
+
Necessity: each multipart upload has exactly one content type
|
55
|
+
Fact type: multipart upload has size
|
56
|
+
Necessity: each multipart upload has exactly one size
|
57
|
+
Fact type: multipart upload has chunk size
|
58
|
+
Necessity: each multipart upload has exactly one chunk size
|
59
|
+
Fact type: multipart upload has expiry date (Auth)
|
60
|
+
Necessity: each multipart upload has exactly one expiry date (Auth)
|
61
|
+
Fact type: multipart upload is created by actor (Auth)
|
62
|
+
Necessity: each multipart upload is created by at most one actor (Auth)
|
package/out/tasks/common.d.ts
DELETED
package/out/tasks/common.js
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
|
-
};
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
-
exports.ajv = exports.channel = exports.apiRoot = void 0;
|
7
|
-
const ajv_1 = __importDefault(require("ajv"));
|
8
|
-
exports.apiRoot = 'tasks';
|
9
|
-
exports.channel = 'task_insert';
|
10
|
-
exports.ajv = new ajv_1.default({
|
11
|
-
inlineRefs: false,
|
12
|
-
});
|
13
|
-
//# sourceMappingURL=common.js.map
|
package/out/tasks/common.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"common.js","sourceRoot":"","sources":["../../src/tasks/common.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAsB;AAGT,QAAA,OAAO,GAAG,OAAO,CAAC;AAGlB,QAAA,OAAO,GAAG,aAAa,CAAC;AAKxB,QAAA,GAAG,GAAG,IAAI,aAAG,CAAC;IAC1B,UAAU,EAAE,KAAK;CACjB,CAAC,CAAC"}
|
package/out/tasks/index.d.ts
DELETED
@@ -1,10 +0,0 @@
|
|
1
|
-
import type { Schema } from 'ajv';
|
2
|
-
import type * as Db from '../database-layer/db';
|
3
|
-
import type { sbvrUtils } from '../server-glue/module';
|
4
|
-
import type { TaskHandler } from './types';
|
5
|
-
export * from './types';
|
6
|
-
export declare const config: {
|
7
|
-
models: sbvrUtils.ExecutableModel[];
|
8
|
-
};
|
9
|
-
export declare function setup(db: Db.Database, tx: Db.Tx): Promise<void>;
|
10
|
-
export declare function addTaskHandler(name: string, fn: TaskHandler['fn'], schema?: Schema): void;
|
package/out/tasks/index.js
DELETED
@@ -1,139 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
-
if (k2 === undefined) k2 = k;
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
-
}
|
8
|
-
Object.defineProperty(o, k2, desc);
|
9
|
-
}) : (function(o, m, k, k2) {
|
10
|
-
if (k2 === undefined) k2 = k;
|
11
|
-
o[k2] = m[k];
|
12
|
-
}));
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
15
|
-
}) : function(o, v) {
|
16
|
-
o["default"] = v;
|
17
|
-
});
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
19
|
-
if (mod && mod.__esModule) return mod;
|
20
|
-
var result = {};
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
22
|
-
__setModuleDefault(result, mod);
|
23
|
-
return result;
|
24
|
-
};
|
25
|
-
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
26
|
-
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
27
|
-
};
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
29
|
-
exports.addTaskHandler = exports.setup = exports.config = void 0;
|
30
|
-
const cronParser = __importStar(require("cron-parser"));
|
31
|
-
const env_1 = require("../config-loader/env");
|
32
|
-
const errors_1 = require("../sbvr-api/errors");
|
33
|
-
const hooks_1 = require("../sbvr-api/hooks");
|
34
|
-
const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
|
35
|
-
const common_1 = require("./common");
|
36
|
-
const worker_1 = require("./worker");
|
37
|
-
__exportStar(require("./types"), exports);
|
38
|
-
const modelText = require('./model.sbvr');
|
39
|
-
exports.config = {
|
40
|
-
models: [
|
41
|
-
{
|
42
|
-
modelName: common_1.apiRoot,
|
43
|
-
apiRoot: common_1.apiRoot,
|
44
|
-
modelText,
|
45
|
-
customServerCode: exports,
|
46
|
-
},
|
47
|
-
],
|
48
|
-
};
|
49
|
-
async function createTrigger(tx) {
|
50
|
-
await tx.executeSql(`
|
51
|
-
CREATE OR REPLACE FUNCTION notify_task_insert()
|
52
|
-
RETURNS TRIGGER AS $$
|
53
|
-
BEGIN
|
54
|
-
PERFORM pg_notify('${common_1.channel}', NEW.id::text);
|
55
|
-
RETURN NEW;
|
56
|
-
END;
|
57
|
-
$$ LANGUAGE plpgsql;
|
58
|
-
`);
|
59
|
-
await tx.executeSql(`
|
60
|
-
CREATE OR REPLACE TRIGGER task_insert_trigger
|
61
|
-
AFTER INSERT ON task
|
62
|
-
FOR EACH ROW WHEN (NEW.status = 'pending' AND NEW."is scheduled to execute on-time" IS NULL)
|
63
|
-
EXECUTE FUNCTION notify_task_insert();
|
64
|
-
`);
|
65
|
-
}
|
66
|
-
let worker = null;
|
67
|
-
async function setup(db, tx) {
|
68
|
-
if (db.engine !== 'postgres') {
|
69
|
-
return;
|
70
|
-
}
|
71
|
-
await createTrigger(tx);
|
72
|
-
const client = new sbvr_utils_1.PinejsClient({
|
73
|
-
apiPrefix: `/${common_1.apiRoot}/`,
|
74
|
-
});
|
75
|
-
worker = new worker_1.Worker(client);
|
76
|
-
(0, hooks_1.addPureHook)('POST', common_1.apiRoot, 'task', {
|
77
|
-
POSTPARSE: async ({ req, request }) => {
|
78
|
-
request.values.is_created_by__actor =
|
79
|
-
req.user?.actor ?? req.apiKey?.actor;
|
80
|
-
if (request.values.is_created_by__actor == null) {
|
81
|
-
throw new errors_1.BadRequestError('Creating tasks with missing actor on req is not allowed');
|
82
|
-
}
|
83
|
-
request.values.status = 'pending';
|
84
|
-
request.values.attempt_count = 0;
|
85
|
-
request.values.priority ??= 1;
|
86
|
-
request.values.attempt_limit ??= 1;
|
87
|
-
if (request.values.is_scheduled_with__cron_expression != null &&
|
88
|
-
request.values.is_scheduled_to_execute_on__time == null) {
|
89
|
-
try {
|
90
|
-
request.values.is_scheduled_to_execute_on__time = cronParser
|
91
|
-
.parseExpression(request.values.is_scheduled_with__cron_expression)
|
92
|
-
.next()
|
93
|
-
.toDate()
|
94
|
-
.toISOString();
|
95
|
-
}
|
96
|
-
catch (_) {
|
97
|
-
throw new errors_1.BadRequestError(`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`);
|
98
|
-
}
|
99
|
-
}
|
100
|
-
if (request.values.is_scheduled_to_execute_on__time != null) {
|
101
|
-
const now = new Date(new Date().getTime() + env_1.tasks.queueIntervalMS);
|
102
|
-
const startTime = new Date(request.values.is_scheduled_to_execute_on__time);
|
103
|
-
if (startTime < now) {
|
104
|
-
throw new errors_1.BadRequestError(`Task scheduled start time must be greater than ${env_1.tasks.queueIntervalMS} milliseconds in the future`);
|
105
|
-
}
|
106
|
-
}
|
107
|
-
const handlerName = request.values.is_executed_by__handler;
|
108
|
-
if (handlerName == null) {
|
109
|
-
throw new errors_1.BadRequestError(`Must specify a task handler to execute`);
|
110
|
-
}
|
111
|
-
const handler = worker?.handlers[handlerName];
|
112
|
-
if (handler == null) {
|
113
|
-
throw new errors_1.BadRequestError(`No task handler with name '${handlerName}' registered`);
|
114
|
-
}
|
115
|
-
if (handler.validate != null) {
|
116
|
-
if (!handler.validate(request.values.is_executed_with__parameter_set)) {
|
117
|
-
throw new errors_1.BadRequestError(`Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
|
118
|
-
}
|
119
|
-
}
|
120
|
-
},
|
121
|
-
});
|
122
|
-
worker.start();
|
123
|
-
}
|
124
|
-
exports.setup = setup;
|
125
|
-
function addTaskHandler(name, fn, schema) {
|
126
|
-
if (worker == null) {
|
127
|
-
return;
|
128
|
-
}
|
129
|
-
if (worker.handlers[name] != null) {
|
130
|
-
throw new Error(`Task handler with name '${name}' already registered`);
|
131
|
-
}
|
132
|
-
worker.handlers[name] = {
|
133
|
-
name,
|
134
|
-
fn,
|
135
|
-
validate: schema != null ? common_1.ajv.compile(schema) : undefined,
|
136
|
-
};
|
137
|
-
}
|
138
|
-
exports.addTaskHandler = addTaskHandler;
|
139
|
-
//# sourceMappingURL=index.js.map
|
package/out/tasks/index.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,uDAAsD;AAEtD,qCAAiD;AAEjD,qCAAkC;AAElC,0CAAwB;AAGxB,MAAM,SAAS,GAAW,OAAO,CAAC,cAAc,CAAC,CAAC;AAErC,QAAA,MAAM,GAAG;IACrB,MAAM,EAAE;QACP;YACC,SAAS,EAAE,gBAAO;YAClB,OAAO,EAAP,gBAAO;YACP,SAAS;YACT,gBAAgB,EAAE,OAAO;SACzB;KAC8B;CAChC,CAAC;AAIF,KAAK,UAAU,aAAa,CAAC,EAAS;IACrC,MAAM,EAAE,CAAC,UAAU,CAAC;;;;iCAIY,gBAAO;;;;KAInC,CAAC,CAAC;IAGN,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;KAKhB,CAAC,CAAC;AACP,CAAC;AAED,IAAI,MAAM,GAAkB,IAAI,CAAC;AAC1B,KAAK,UAAU,KAAK,CAAC,EAAe,EAAE,EAAS;IAErD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO;IACR,CAAC;IAGD,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAExB,MAAM,MAAM,GAAG,IAAI,yBAAY,CAAC;QAC/B,SAAS,EAAE,IAAI,gBAAO,GAAG;KACzB,CAAC,CAAC;IACH,MAAM,GAAG,IAAI,eAAM,CAAC,MAAM,CAAC,CAAC;IAG5B,IAAA,mBAAW,EAAC,MAAM,EAAE,gBAAO,EAAE,MAAM,EAAE;QACpC,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;YAErC,OAAO,CAAC,MAAM,CAAC,oBAAoB;gBAClC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAI,wBAAe,CACxB,yDAAyD,CACzD,CAAC;YACH,CAAC;YAGD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC;YAGnC,IACC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAI,IAAI;gBACzD,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EACtD,CAAC;gBACF,IAAI,CAAC;oBACJ,OAAO,CAAC,MAAM,CAAC,gCAAgC,GAAG,UAAU;yBAC1D,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,CAAC;yBAClE,IAAI,EAAE;yBACN,MAAM,EAAE;yBACR,WAAW,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,MAAM,IAAI,wBAAe,CACxB,4BAA4B,OAAO,CAAC,MAAM,CAAC,kCAAkC,EAAE,CAC/E,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,IAAI,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,WAAQ,CAAC,eAAe,CAAC,CAAC;gBACtE,MAAM,SAAS,GAAG,IAAI,IAAI,CACzB,OAAO,CAAC,MAAM,CAAC,gCAAgC,CAC/C,CAAC;gBACF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBACrB,MAAM,IAAI,wBAAe,CACxB,kDAAkD,WAAQ,CAAC,eAAe,6BAA6B,CACvG,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC;YAC3D,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,wBAAe,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,wBAAe,CACxB,8BAA8B,WAAW,cAAc,CACvD,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,wBAAe,CACxB,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAtFD,sBAsFC;AAGD,SAAgB,cAAc,CAC7B,IAAY,EACZ,EAAqB,EACrB,MAAe;IAEf,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO;IACR,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,sBAAsB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;QACvB,IAAI;QACJ,EAAE;QACF,QAAQ,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,YAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC;AACH,CAAC;AAjBD,wCAiBC"}
|
package/out/tasks/model.sbvr
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
Vocabulary: tasks
|
2
|
-
|
3
|
-
Term: id
|
4
|
-
Concept Type: Big Serial (Type)
|
5
|
-
Term: actor
|
6
|
-
Concept Type: Integer (Type)
|
7
|
-
Term: attempt count
|
8
|
-
Concept Type: Integer (Type)
|
9
|
-
Term: attempt limit
|
10
|
-
Concept Type: Integer (Type)
|
11
|
-
Term: cron expression
|
12
|
-
Concept Type: Short Text (Type)
|
13
|
-
Term: error message
|
14
|
-
Concept Type: Short Text (Type)
|
15
|
-
Term: handler
|
16
|
-
Concept Type: Short Text (Type)
|
17
|
-
Term: key
|
18
|
-
Concept Type: Short Text (Type)
|
19
|
-
Term: parameter set
|
20
|
-
Concept Type: JSON (Type)
|
21
|
-
Term: priority
|
22
|
-
Concept Type: Integer (Type)
|
23
|
-
Term: status
|
24
|
-
Concept Type: Short Text (Type)
|
25
|
-
Term: time
|
26
|
-
Concept Type: Date Time (Type)
|
27
|
-
|
28
|
-
Term: task
|
29
|
-
Fact type: task has id
|
30
|
-
Necessity: each task has exactly one id
|
31
|
-
Fact type: task has key
|
32
|
-
Necessity: each task has at most one key
|
33
|
-
Fact type: task is created by actor
|
34
|
-
Necessity: each task is created by exactly one actor
|
35
|
-
Fact type: task is executed by handler
|
36
|
-
Necessity: each task is executed by exactly one handler
|
37
|
-
Fact type: task is executed with parameter set
|
38
|
-
Necessity: each task is executed with at most one parameter set
|
39
|
-
Fact type: task has priority
|
40
|
-
Necessity: each task has exactly one priority
|
41
|
-
Necessity: each task has a priority that is greater than or equal to 0
|
42
|
-
Fact type: task is scheduled with cron expression
|
43
|
-
Necessity: each task is scheduled with at most one cron expression
|
44
|
-
Fact type: task is scheduled to execute on time
|
45
|
-
Necessity: each task is scheduled to execute on at most one time
|
46
|
-
Fact type: task has status
|
47
|
-
Necessity: each task has exactly one status
|
48
|
-
Definition: "pending" or "cancelled" or "success" or "failed"
|
49
|
-
Fact type: task started on time
|
50
|
-
Necessity: each task started on at most one time
|
51
|
-
Fact type: task ended on time
|
52
|
-
Necessity: each task ended on at most one time
|
53
|
-
Fact type: task has error message
|
54
|
-
Necessity: each task has at most one error message
|
55
|
-
Fact type: task has attempt count
|
56
|
-
Necessity: each task has exactly one attempt count
|
57
|
-
Fact type: task has attempt limit
|
58
|
-
Necessity: each task has exactly one attempt limit
|
59
|
-
Necessity: each task has an attempt limit that is greater than or equal to 1
|
60
|
-
|
package/out/tasks/types.d.ts
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
import type { ValidateFunction } from 'ajv';
|
2
|
-
import type { AnyObject } from 'pinejs-client-core';
|
3
|
-
import type * as Db from '../database-layer/db';
|
4
|
-
import type { PinejsClient } from '../sbvr-api/sbvr-utils';
|
5
|
-
export declare const taskStatuses: readonly ["pending", "cancelled", "success", "failed"];
|
6
|
-
export type TaskStatus = (typeof taskStatuses)[number];
|
7
|
-
export interface Task {
|
8
|
-
id: number;
|
9
|
-
created_at: Date;
|
10
|
-
modified_at: Date;
|
11
|
-
is_created_by__actor: number;
|
12
|
-
is_executed_by__handler: string;
|
13
|
-
is_executed_with__parameter_set: object | null;
|
14
|
-
is_scheduled_with__cron_expression: string | null;
|
15
|
-
is_scheduled_to_execute_on__time: Date | null;
|
16
|
-
priority: number;
|
17
|
-
status: TaskStatus;
|
18
|
-
started_on__time: Date | null;
|
19
|
-
ended_on__time: Date | null;
|
20
|
-
error_message: string | null;
|
21
|
-
attempt_count: number;
|
22
|
-
attempt_limit: number;
|
23
|
-
}
|
24
|
-
export type PartialTask = Pick<Task, 'id' | 'is_created_by__actor' | 'is_executed_by__handler' | 'is_executed_with__parameter_set' | 'is_scheduled_with__cron_expression' | 'priority' | 'attempt_count' | 'attempt_limit'>;
|
25
|
-
export interface TaskArgs {
|
26
|
-
api: PinejsClient;
|
27
|
-
params: AnyObject;
|
28
|
-
tx: Db.Tx;
|
29
|
-
}
|
30
|
-
export type TaskResponse = Promise<{
|
31
|
-
status: TaskStatus;
|
32
|
-
error?: string;
|
33
|
-
}>;
|
34
|
-
export interface TaskHandler {
|
35
|
-
name: string;
|
36
|
-
fn: (options: TaskArgs) => TaskResponse;
|
37
|
-
validate?: ValidateFunction;
|
38
|
-
}
|
package/out/tasks/types.js
DELETED
package/out/tasks/types.js.map
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,SAAS;IACT,WAAW;IACX,SAAS;IACT,QAAQ;CACC,CAAC"}
|
package/out/tasks/worker.d.ts
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
import { PinejsClient } from '../sbvr-api/sbvr-utils';
|
2
|
-
import type { TaskHandler } from './types';
|
3
|
-
export declare class Worker {
|
4
|
-
handlers: Record<string, TaskHandler>;
|
5
|
-
private readonly concurrency;
|
6
|
-
private readonly interval;
|
7
|
-
private client;
|
8
|
-
private executing;
|
9
|
-
constructor(client: PinejsClient);
|
10
|
-
private canExecute;
|
11
|
-
private execute;
|
12
|
-
private finalize;
|
13
|
-
private getNextAttemptTime;
|
14
|
-
private poll;
|
15
|
-
start(): void;
|
16
|
-
}
|