@balena/pinejs 16.2.0-build-joshbwlng-tasks-56c035f7e142216b5cf56fc7d861ce4b51157a41-1 → 16.2.0-build-joshbwlng-tasks-009b08b1f157611c22f2425c25f905d5a59aaabe-1
Sign up to get free protection for your applications and to get access to all the features.
- package/.pinejs-cache.json +1 -1
- package/.versionbot/CHANGELOG.yml +10 -2
- package/CHANGELOG.md +2 -1
- package/out/tasks/index.d.ts +1 -1
- package/out/tasks/index.js +51 -14
- package/out/tasks/index.js.map +1 -1
- package/out/tasks/model.sbvr +1 -1
- package/out/tasks/types.d.ts +1 -1
- package/out/tasks/types.js +2 -2
- package/out/tasks/types.js.map +1 -1
- package/out/tasks/worker.js +44 -32
- package/out/tasks/worker.js.map +1 -1
- package/package.json +2 -2
- package/src/tasks/index.ts +89 -17
- package/src/tasks/model.sbvr +1 -1
- package/src/tasks/types.ts +2 -2
- package/src/tasks/worker.ts +45 -34
|
@@ -1,6 +1,14 @@
|
|
|
1
1
|
- commits:
|
|
2
|
+
- subject: Cleanup
|
|
3
|
+
hash: 009b08b1f157611c22f2425c25f905d5a59aaabe
|
|
4
|
+
body: ""
|
|
5
|
+
footer:
|
|
6
|
+
Change-type: patch
|
|
7
|
+
change-type: patch
|
|
8
|
+
author: Josh Bowling
|
|
9
|
+
nested: []
|
|
2
10
|
- subject: Add async tasks
|
|
3
|
-
hash:
|
|
11
|
+
hash: 1297814070f10e99d7bd475c28c365cc03081096
|
|
4
12
|
body: ""
|
|
5
13
|
footer:
|
|
6
14
|
Change-type: minor
|
|
@@ -9,7 +17,7 @@
|
|
|
9
17
|
nested: []
|
|
10
18
|
version: 16.2.0
|
|
11
19
|
title: ""
|
|
12
|
-
date: 2024-04-
|
|
20
|
+
date: 2024-04-30T05:37:15.944Z
|
|
13
21
|
- commits:
|
|
14
22
|
- subject: Update @balena/odata-to-abstract-sql to 6.2.7
|
|
15
23
|
hash: 76b2f842af0b49ef2956db23a22758fc0d1e50a0
|
package/CHANGELOG.md
CHANGED
|
@@ -5,8 +5,9 @@ automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
|
|
|
5
5
|
This project adheres to [Semantic Versioning](http://semver.org/).
|
|
6
6
|
|
|
7
7
|
# v16.2.0
|
|
8
|
-
## (2024-04-
|
|
8
|
+
## (2024-04-30)
|
|
9
9
|
|
|
10
|
+
* Cleanup [Josh Bowling]
|
|
10
11
|
* Add async tasks [Josh Bowling]
|
|
11
12
|
|
|
12
13
|
# v16.1.3
|
package/out/tasks/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Schema } from 'ajv';
|
|
2
2
|
import type * as Db from '../database-layer/db';
|
|
3
|
-
import
|
|
3
|
+
import * as sbvrUtils from '../sbvr-api/sbvr-utils';
|
|
4
4
|
import type { TaskHandler } from './types';
|
|
5
5
|
export * from './types';
|
|
6
6
|
export declare const config: {
|
package/out/tasks/index.js
CHANGED
|
@@ -32,6 +32,7 @@ const env_1 = require("../config-loader/env");
|
|
|
32
32
|
const errors_1 = require("../sbvr-api/errors");
|
|
33
33
|
const hooks_1 = require("../sbvr-api/hooks");
|
|
34
34
|
const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
|
|
35
|
+
const sbvrUtils = __importStar(require("../sbvr-api/sbvr-utils"));
|
|
35
36
|
const common_1 = require("./common");
|
|
36
37
|
const worker_1 = require("./worker");
|
|
37
38
|
__exportStar(require("./types"), exports);
|
|
@@ -48,20 +49,54 @@ exports.config = {
|
|
|
48
49
|
};
|
|
49
50
|
async function createTrigger(tx) {
|
|
50
51
|
await tx.executeSql(`
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
52
|
+
CREATE OR REPLACE FUNCTION notify_task_insert()
|
|
53
|
+
RETURNS TRIGGER AS $$
|
|
54
|
+
BEGIN
|
|
55
|
+
PERFORM pg_notify('${common_1.channel}', NEW.id::text);
|
|
56
|
+
RETURN NEW;
|
|
57
|
+
END;
|
|
58
|
+
$$ LANGUAGE plpgsql;
|
|
59
|
+
`);
|
|
59
60
|
await tx.executeSql(`
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
CREATE OR REPLACE TRIGGER task_insert_trigger
|
|
62
|
+
AFTER INSERT ON task
|
|
63
|
+
FOR EACH ROW WHEN (NEW.status = 'queued' AND NEW."is scheduled to execute on-time" IS NULL)
|
|
64
|
+
EXECUTE FUNCTION notify_task_insert();
|
|
65
|
+
`);
|
|
66
|
+
}
|
|
67
|
+
async function createIndexes(tx) {
|
|
68
|
+
await tx.executeSql(`
|
|
69
|
+
CREATE INDEX IF NOT EXISTS idx_task_poll ON task USING btree (
|
|
70
|
+
"is executed by-handler",
|
|
71
|
+
"is scheduled to execute on-time" ASC,
|
|
72
|
+
"priority" DESC,
|
|
73
|
+
"id" ASC
|
|
74
|
+
) WHERE status = 'queued';
|
|
75
|
+
`);
|
|
76
|
+
await tx.executeSql(`
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_task_queued ON task (
|
|
78
|
+
"id",
|
|
79
|
+
"is created by-actor"
|
|
80
|
+
) WHERE status = 'queued';
|
|
81
|
+
`);
|
|
82
|
+
}
|
|
83
|
+
async function adjustPriority(actor, priority) {
|
|
84
|
+
const result = await sbvrUtils.db.executeSql(`SELECT "is created by-actor", COUNT("id") AS task_count
|
|
85
|
+
FROM (
|
|
86
|
+
SELECT "id", "is created by-actor"
|
|
87
|
+
FROM task
|
|
88
|
+
WHERE "status" = 'queued'
|
|
89
|
+
ORDER BY "id" DESC
|
|
90
|
+
LIMIT 100
|
|
91
|
+
) AS recent_tasks
|
|
92
|
+
GROUP BY "is created by-actor"
|
|
93
|
+
ORDER BY task_count DESC;
|
|
94
|
+
`);
|
|
95
|
+
if (result.rows.some((row) => parseInt(row.task_count, 10) >= 49 &&
|
|
96
|
+
row['is created by-actor'] !== actor)) {
|
|
97
|
+
return priority + 1;
|
|
98
|
+
}
|
|
99
|
+
return priority;
|
|
65
100
|
}
|
|
66
101
|
let worker = null;
|
|
67
102
|
async function setup(db, tx) {
|
|
@@ -69,6 +104,7 @@ async function setup(db, tx) {
|
|
|
69
104
|
return;
|
|
70
105
|
}
|
|
71
106
|
await createTrigger(tx);
|
|
107
|
+
await createIndexes(tx);
|
|
72
108
|
const client = new sbvr_utils_1.PinejsClient({
|
|
73
109
|
apiPrefix: `/${common_1.apiRoot}/`,
|
|
74
110
|
});
|
|
@@ -80,10 +116,11 @@ async function setup(db, tx) {
|
|
|
80
116
|
if (request.values.is_created_by__actor == null) {
|
|
81
117
|
throw new errors_1.BadRequestError('Creating tasks with missing actor on req is not allowed');
|
|
82
118
|
}
|
|
83
|
-
request.values.status = '
|
|
119
|
+
request.values.status = 'queued';
|
|
84
120
|
request.values.attempt_count = 0;
|
|
85
121
|
request.values.priority ??= 1;
|
|
86
122
|
request.values.attempt_limit ??= 1;
|
|
123
|
+
request.values.priority = await adjustPriority(request.values.is_created_by__actor, request.values.priority);
|
|
87
124
|
if (request.values.is_scheduled_with__cron_expression != null &&
|
|
88
125
|
request.values.is_scheduled_to_execute_on__time == null) {
|
|
89
126
|
try {
|
package/out/tasks/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,uDAAsD;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,uDAAsD;AACtD,kEAAoD;AACpD,qCAAiD;AAEjD,qCAAkC;AAElC,0CAAwB;AAGxB,MAAM,SAAS,GAAW,OAAO,CAAC,cAAc,CAAC,CAAC;AAErC,QAAA,MAAM,GAAG;IACrB,MAAM,EAAE;QACP;YACC,SAAS,EAAE,gBAAO;YAClB,OAAO,EAAP,gBAAO;YACP,SAAS;YACT,gBAAgB,EAAE,OAAO;SACzB;KAC8B;CAChC,CAAC;AAIF,KAAK,UAAU,aAAa,CAAC,EAAS;IACrC,MAAM,EAAE,CAAC,UAAU,CAAC;;;;wBAIG,gBAAO;;;;EAI7B,CAAC,CAAC;IAGH,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;EAKnB,CAAC,CAAC;AACJ,CAAC;AAGD,KAAK,UAAU,aAAa,CAAC,EAAS;IAErC,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;;;EAOnB,CAAC,CAAC;IAGH,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;EAKnB,CAAC,CAAC;AACJ,CAAC;AAOD,KAAK,UAAU,cAAc,CAC5B,KAAa,EACb,QAAgB;IAEhB,MAAM,MAAM,GAAG,MAAO,SAAS,CAAC,EAAE,CAAC,UAAU,CAC5C;;;;;;;;;;EAUA,CAOE,CAAC;IAGJ,IACC,MAAM,CAAC,IAAI,CAAC,IAAI,CACf,CAAC,GAAG,EAAE,EAAE,CACP,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,EAAE;QAClC,GAAG,CAAC,qBAAqB,CAAC,KAAK,KAAK,CACrC,EACA,CAAC;QACF,OAAO,QAAQ,GAAG,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,IAAI,MAAM,GAAkB,IAAI,CAAC;AAC1B,KAAK,UAAU,KAAK,CAAC,EAAe,EAAE,EAAS;IAErD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO;IACR,CAAC;IAGD,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAGxB,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAExB,MAAM,MAAM,GAAG,IAAI,yBAAY,CAAC;QAC/B,SAAS,EAAE,IAAI,gBAAO,GAAG;KACzB,CAAC,CAAC;IACH,MAAM,GAAG,IAAI,eAAM,CAAC,MAAM,CAAC,CAAC;IAG5B,IAAA,mBAAW,EAAC,MAAM,EAAE,gBAAO,EAAE,MAAM,EAAE;QACpC,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;YAErC,OAAO,CAAC,MAAM,CAAC,oBAAoB;gBAClC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAI,wBAAe,CACxB,yDAAyD,CACzD,CAAC;YACH,CAAC;YAGD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC;YAGnC,OAAO,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,cAAc,CAC7C,OAAO,CAAC,MAAM,CAAC,oBAAoB,EACnC,OAAO,CAAC,MAAM,CAAC,QAAQ,CACvB,CAAC;YAGF,IACC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAI,IAAI;gBACzD,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EACtD,CAAC;gBACF,IAAI,CAAC;oBACJ,OAAO,CAAC,MAAM,CAAC,gCAAgC,GAAG,UAAU;yBAC1D,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,CAAC;yBAClE,IAAI,EAAE;yBACN,MAAM,EAAE;yBACR,WAAW,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,MAAM,IAAI,wBAAe,CACxB,4BAA4B,OAAO,CAAC,MAAM,CAAC,kCAAkC,EAAE,CAC/E,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,IAAI,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,WAAQ,CAAC,eAAe,CAAC,CAAC;gBACtE,MAAM,SAAS,GAAG,IAAI,IAAI,CACzB,OAAO,CAAC,MAAM,CAAC,gCAAgC,CAC/C,CAAC;gBACF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBACrB,MAAM,IAAI,wBAAe,CACxB,kDAAkD,WAAQ,CAAC,eAAe,6BAA6B,CACvG,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC;YAC3D,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,wBAAe,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,wBAAe,CACxB,8BAA8B,WAAW,cAAc,CACvD,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,wBAAe,CACxB,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AA/FD,sBA+FC;AAGD,SAAgB,cAAc,CAC7B,IAAY,EACZ,EAAqB,EACrB,MAAe;IAEf,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO;IACR,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,sBAAsB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;QACvB,IAAI;QACJ,EAAE;QACF,QAAQ,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,YAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC;AACH,CAAC;AAjBD,wCAiBC"}
|
package/out/tasks/model.sbvr
CHANGED
|
@@ -45,7 +45,7 @@ Fact type: task is scheduled to execute on time
|
|
|
45
45
|
Necessity: each task is scheduled to execute on at most one time
|
|
46
46
|
Fact type: task has status
|
|
47
47
|
Necessity: each task has exactly one status
|
|
48
|
-
Definition: "
|
|
48
|
+
Definition: "queued" or "cancelled" or "succeeded" or "failed"
|
|
49
49
|
Fact type: task started on time
|
|
50
50
|
Necessity: each task started on at most one time
|
|
51
51
|
Fact type: task ended on time
|
package/out/tasks/types.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import type { ValidateFunction } from 'ajv';
|
|
|
2
2
|
import type { AnyObject } from 'pinejs-client-core';
|
|
3
3
|
import type * as Db from '../database-layer/db';
|
|
4
4
|
import type { PinejsClient } from '../sbvr-api/sbvr-utils';
|
|
5
|
-
export declare const taskStatuses: readonly ["
|
|
5
|
+
export declare const taskStatuses: readonly ["queued", "cancelled", "succeeded", "failed"];
|
|
6
6
|
export type TaskStatus = (typeof taskStatuses)[number];
|
|
7
7
|
export interface Task {
|
|
8
8
|
id: number;
|
package/out/tasks/types.js
CHANGED
package/out/tasks/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;CACC,CAAC"}
|
package/out/tasks/worker.js
CHANGED
|
@@ -25,6 +25,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
26
|
exports.Worker = void 0;
|
|
27
27
|
const env_1 = require("../config-loader/env");
|
|
28
|
+
const db_1 = require("../database-layer/db");
|
|
28
29
|
const permissions = __importStar(require("../sbvr-api/permissions"));
|
|
29
30
|
const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
|
|
30
31
|
const module_1 = require("../server-glue/module");
|
|
@@ -66,23 +67,34 @@ class Worker {
|
|
|
66
67
|
await this.finalize(tx, task, startedOnTime, 'failed', `Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
|
|
67
68
|
return;
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
70
|
+
let status = 'queued';
|
|
71
|
+
let error;
|
|
72
|
+
try {
|
|
73
|
+
await module_1.sbvrUtils.db.transaction(async (handlerTx) => {
|
|
74
|
+
const results = await handler.fn({
|
|
75
|
+
api: new sbvr_utils_1.PinejsClient({
|
|
76
|
+
passthrough: {
|
|
77
|
+
tx: handlerTx,
|
|
78
|
+
},
|
|
79
|
+
}),
|
|
80
|
+
params: task.is_executed_with__parameter_set ?? {},
|
|
81
|
+
tx: handlerTx,
|
|
82
|
+
});
|
|
83
|
+
status = results.status;
|
|
84
|
+
error = results.error;
|
|
85
|
+
if (results.status !== 'succeeded' && !handlerTx.isClosed()) {
|
|
86
|
+
await handlerTx.rollback();
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
if (!(err instanceof db_1.TransactionClosedError)) {
|
|
92
|
+
throw err;
|
|
93
|
+
}
|
|
81
94
|
}
|
|
82
|
-
|
|
83
|
-
await
|
|
95
|
+
finally {
|
|
96
|
+
await this.finalize(tx, task, startedOnTime, status, error);
|
|
84
97
|
}
|
|
85
|
-
await this.finalize(tx, task, startedOnTime, result.status, result.error);
|
|
86
98
|
}
|
|
87
99
|
catch (err) {
|
|
88
100
|
console.error('Task execution failed:', err);
|
|
@@ -102,7 +114,7 @@ class Worker {
|
|
|
102
114
|
...(errorMessage != null && { error_message: errorMessage }),
|
|
103
115
|
};
|
|
104
116
|
if (status === 'failed' && attemptCount < task.attempt_limit) {
|
|
105
|
-
body.status = '
|
|
117
|
+
body.status = 'queued';
|
|
106
118
|
body.is_scheduled_to_execute_on__time =
|
|
107
119
|
this.getNextAttemptTime(attemptCount);
|
|
108
120
|
}
|
|
@@ -115,7 +127,7 @@ class Worker {
|
|
|
115
127
|
id: task.id,
|
|
116
128
|
body,
|
|
117
129
|
});
|
|
118
|
-
if (['failed', '
|
|
130
|
+
if (['failed', 'succeeded'].includes(body.status) &&
|
|
119
131
|
task.is_scheduled_with__cron_expression != null) {
|
|
120
132
|
await this.client.post({
|
|
121
133
|
resource: 'task',
|
|
@@ -148,21 +160,21 @@ class Worker {
|
|
|
148
160
|
return;
|
|
149
161
|
}
|
|
150
162
|
const result = await tx.executeSql(`SELECT ${selectColumns}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
FROM task AS t
|
|
164
|
+
WHERE
|
|
165
|
+
t."is executed by-handler" IN (${binds}) AND
|
|
166
|
+
t."status" = 'queued' AND
|
|
167
|
+
t."attempt count" <= t."attempt limit" AND
|
|
168
|
+
(
|
|
169
|
+
t."is scheduled to execute on-time" IS NULL OR
|
|
170
|
+
t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
|
|
171
|
+
)
|
|
172
|
+
ORDER BY
|
|
173
|
+
t."is scheduled to execute on-time" ASC,
|
|
174
|
+
t."priority" DESC,
|
|
175
|
+
t."id" ASC
|
|
176
|
+
LIMIT ${Math.max(this.concurrency - this.executing, 0)}
|
|
177
|
+
FOR UPDATE SKIP LOCKED`, handlerNames);
|
|
166
178
|
if (result.rows.length === 0) {
|
|
167
179
|
return;
|
|
168
180
|
}
|
package/out/tasks/worker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,8CAAyD;AAEzD,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IAOlB,YAAY,MAAoB;QANzB,aAAQ,GAAgC,EAAE,CAAC;QAI1C,cAAS,GAAG,CAAC,CAAC;QAGrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,OAAO,CAAC,EAAS,EAAE,IAAiB;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,MAAM,
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,8CAAyD;AAEzD,6CAA8D;AAC9D,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IAOlB,YAAY,MAAoB;QANzB,aAAQ,GAAgC,EAAE,CAAC;QAI1C,cAAS,GAAG,CAAC,CAAC;QAGrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,OAAO,CAAC,EAAS,EAAE,IAAiB;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,IAAI,MAAM,GAAe,QAAQ,CAAC;YAClC,IAAI,KAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;oBAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC;wBAChC,GAAG,EAAE,IAAI,yBAAY,CAAC;4BACrB,WAAW,EAAE;gCACZ,EAAE,EAAE,SAAS;6BACb;yBACD,CAAC;wBACF,MAAM,EAAE,IAAI,CAAC,+BAA+B,IAAI,EAAE;wBAClD,EAAE,EAAE,SAAS;qBACb,CAAC,CAAC;oBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBACxB,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;oBACtB,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC;wBAC7D,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;oBAC5B,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBAEd,IAAI,CAAC,CAAC,GAAG,YAAY,2BAAsB,CAAC,EAAE,CAAC;oBAC9C,MAAM,GAAG,CAAC;gBACX,CAAC;YACF,CAAC;oBAAS,CAAC;gBAEV,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAEd,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,QAAQ,CACrB,EAAS,EACT,IAAiB,EACjB,aAAmB,EACnB,MAAkB,EAClB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAc;YACvB,gBAAgB,EAAE,aAAa;YAC/B,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;SAC5D,CAAC;QAIF,IAAI,MAAM,KAAK,QAAQ,IAAI,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;YAGvB,IAAI,CAAC,gCAAgC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAGD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE;gBACZ,EAAE;gBACF,GAAG,EAAE,WAAW,CAAC,IAAI;aACrB;YACD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;SACJ,CAAC,CAAC;QAIH,IACC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7C,IAAI,CAAC,kCAAkC,IAAI,IAAI,EAC9C,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,MAAM;gBAChB,WAAW,EAAE;oBACZ,EAAE;oBACF,GAAG,EAAE,WAAW,CAAC,IAAI;iBACrB;gBACD,IAAI,EAAE;oBACL,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,+BAA+B,EAAE,IAAI,CAAC,+BAA+B;oBACrE,kCAAkC,EACjC,IAAI,CAAC,kCAAkC;oBACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACvB;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAGO,kBAAkB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAChE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACrC,CAAC;IAGO,IAAI;QACX,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,kBAAS,CAAC,EAAE;aACV,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxB,OAAO;YACR,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa;;;uCAGW,KAAK;;;;;8EAKkC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;;;;;;aAMhG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;4BAC/B,EACvB,YAAY,CACZ,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACR,CAAC;YAGD,MAAM,OAAO,CAAC,GAAG,CAChB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,GAAkB,CAAC,CAAC;YAC5C,CAAC,CAAC,CACF,CAAC;YACF,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACb,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACL,CAAC;IAGM,KAAK;QACX,kBAAS,CAAC,EAAE,CAAC,EAAE,EAAE,CAChB,cAAc,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACb,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa,sDAAsD,EAC7E,CAAC,GAAG,CAAC,OAAO,CAAC,CACb,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAgB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,EACD;YACC,OAAO,EAAP,gBAAO;SACP,CACD,CAAC;QACF,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;CACD;AA3OD,wBA2OC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@balena/pinejs",
|
|
3
|
-
"version": "16.2.0-build-joshbwlng-tasks-
|
|
3
|
+
"version": "16.2.0-build-joshbwlng-tasks-009b08b1f157611c22f2425c25f905d5a59aaabe-1",
|
|
4
4
|
"main": "out/server-glue/module",
|
|
5
5
|
"type": "commonjs",
|
|
6
6
|
"repository": "git@github.com:balena-io/pinejs.git",
|
|
@@ -147,6 +147,6 @@
|
|
|
147
147
|
"recursive": true
|
|
148
148
|
},
|
|
149
149
|
"versionist": {
|
|
150
|
-
"publishedAt": "2024-04-
|
|
150
|
+
"publishedAt": "2024-04-30T05:37:16.724Z"
|
|
151
151
|
}
|
|
152
152
|
}
|
package/src/tasks/index.ts
CHANGED
|
@@ -5,7 +5,7 @@ import type * as Db from '../database-layer/db';
|
|
|
5
5
|
import { BadRequestError } from '../sbvr-api/errors';
|
|
6
6
|
import { addPureHook } from '../sbvr-api/hooks';
|
|
7
7
|
import { PinejsClient } from '../sbvr-api/sbvr-utils';
|
|
8
|
-
import
|
|
8
|
+
import * as sbvrUtils from '../sbvr-api/sbvr-utils';
|
|
9
9
|
import { ajv, apiRoot, channel } from './common';
|
|
10
10
|
import type { TaskHandler } from './types';
|
|
11
11
|
import { Worker } from './worker';
|
|
@@ -30,22 +30,85 @@ export const config = {
|
|
|
30
30
|
// Only poll or execute on triggers if worker is not already at max concurrency
|
|
31
31
|
async function createTrigger(tx: Db.Tx): Promise<void> {
|
|
32
32
|
await tx.executeSql(`
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
// Only trigger if task is
|
|
33
|
+
CREATE OR REPLACE FUNCTION notify_task_insert()
|
|
34
|
+
RETURNS TRIGGER AS $$
|
|
35
|
+
BEGIN
|
|
36
|
+
PERFORM pg_notify('${channel}', NEW.id::text);
|
|
37
|
+
RETURN NEW;
|
|
38
|
+
END;
|
|
39
|
+
$$ LANGUAGE plpgsql;
|
|
40
|
+
`);
|
|
41
|
+
|
|
42
|
+
// Only trigger if task is queued and not scheduled
|
|
43
43
|
await tx.executeSql(`
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
44
|
+
CREATE OR REPLACE TRIGGER task_insert_trigger
|
|
45
|
+
AFTER INSERT ON task
|
|
46
|
+
FOR EACH ROW WHEN (NEW.status = 'queued' AND NEW."is scheduled to execute on-time" IS NULL)
|
|
47
|
+
EXECUTE FUNCTION notify_task_insert();
|
|
48
|
+
`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Create indexes
|
|
52
|
+
async function createIndexes(tx: Db.Tx): Promise<void> {
|
|
53
|
+
// Partial index for polling
|
|
54
|
+
await tx.executeSql(`
|
|
55
|
+
CREATE INDEX IF NOT EXISTS idx_task_poll ON task USING btree (
|
|
56
|
+
"is executed by-handler",
|
|
57
|
+
"is scheduled to execute on-time" ASC,
|
|
58
|
+
"priority" DESC,
|
|
59
|
+
"id" ASC
|
|
60
|
+
) WHERE status = 'queued';
|
|
61
|
+
`);
|
|
62
|
+
|
|
63
|
+
// Partial index used to adjust priority based on actor
|
|
64
|
+
await tx.executeSql(`
|
|
65
|
+
CREATE INDEX IF NOT EXISTS idx_task_queued ON task (
|
|
66
|
+
"id",
|
|
67
|
+
"is created by-actor"
|
|
68
|
+
) WHERE status = 'queued';
|
|
69
|
+
`);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Check if a task should be given priority
|
|
73
|
+
// This is used to balance the load across actors
|
|
74
|
+
// by giving priority to actors with less tasks in the queue
|
|
75
|
+
// when another actor is crossing a percentage threshold of queued tasks
|
|
76
|
+
// This is to prevent a single actor from hogging the queue
|
|
77
|
+
async function adjustPriority(
|
|
78
|
+
actor: number,
|
|
79
|
+
priority: number,
|
|
80
|
+
): Promise<number> {
|
|
81
|
+
const result = await (sbvrUtils.db.executeSql(
|
|
82
|
+
`SELECT "is created by-actor", COUNT("id") AS task_count
|
|
83
|
+
FROM (
|
|
84
|
+
SELECT "id", "is created by-actor"
|
|
85
|
+
FROM task
|
|
86
|
+
WHERE "status" = 'queued'
|
|
87
|
+
ORDER BY "id" DESC
|
|
88
|
+
LIMIT 100
|
|
89
|
+
) AS recent_tasks
|
|
90
|
+
GROUP BY "is created by-actor"
|
|
91
|
+
ORDER BY task_count DESC;
|
|
92
|
+
`,
|
|
93
|
+
) as Promise<{
|
|
94
|
+
rowsAffected: number;
|
|
95
|
+
rows: Array<{
|
|
96
|
+
'is created by-actor': number;
|
|
97
|
+
task_count: string;
|
|
98
|
+
}>;
|
|
99
|
+
}>);
|
|
100
|
+
|
|
101
|
+
// Increase the priority of this task if another actor is filling up the queue
|
|
102
|
+
if (
|
|
103
|
+
result.rows.some(
|
|
104
|
+
(row) =>
|
|
105
|
+
parseInt(row.task_count, 10) >= 49 &&
|
|
106
|
+
row['is created by-actor'] !== actor,
|
|
107
|
+
)
|
|
108
|
+
) {
|
|
109
|
+
return priority + 1;
|
|
110
|
+
}
|
|
111
|
+
return priority;
|
|
49
112
|
}
|
|
50
113
|
|
|
51
114
|
let worker: Worker | null = null;
|
|
@@ -58,6 +121,9 @@ export async function setup(db: Db.Database, tx: Db.Tx): Promise<void> {
|
|
|
58
121
|
// Create trigger function if it doesn't exist
|
|
59
122
|
await createTrigger(tx);
|
|
60
123
|
|
|
124
|
+
// Create indexes if they don't exist
|
|
125
|
+
await createIndexes(tx);
|
|
126
|
+
|
|
61
127
|
const client = new PinejsClient({
|
|
62
128
|
apiPrefix: `/${apiRoot}/`,
|
|
63
129
|
});
|
|
@@ -76,11 +142,17 @@ export async function setup(db: Db.Database, tx: Db.Tx): Promise<void> {
|
|
|
76
142
|
}
|
|
77
143
|
|
|
78
144
|
// Set defaults
|
|
79
|
-
request.values.status = '
|
|
145
|
+
request.values.status = 'queued';
|
|
80
146
|
request.values.attempt_count = 0;
|
|
81
147
|
request.values.priority ??= 1;
|
|
82
148
|
request.values.attempt_limit ??= 1;
|
|
83
149
|
|
|
150
|
+
// Possibly adjust priority based on actor
|
|
151
|
+
request.values.priority = await adjustPriority(
|
|
152
|
+
request.values.is_created_by__actor,
|
|
153
|
+
request.values.priority,
|
|
154
|
+
);
|
|
155
|
+
|
|
84
156
|
// Set scheduled start time using cron expression if provided
|
|
85
157
|
if (
|
|
86
158
|
request.values.is_scheduled_with__cron_expression != null &&
|
package/src/tasks/model.sbvr
CHANGED
|
@@ -45,7 +45,7 @@ Fact type: task is scheduled to execute on time
|
|
|
45
45
|
Necessity: each task is scheduled to execute on at most one time
|
|
46
46
|
Fact type: task has status
|
|
47
47
|
Necessity: each task has exactly one status
|
|
48
|
-
Definition: "
|
|
48
|
+
Definition: "queued" or "cancelled" or "succeeded" or "failed"
|
|
49
49
|
Fact type: task started on time
|
|
50
50
|
Necessity: each task started on at most one time
|
|
51
51
|
Fact type: task ended on time
|
package/src/tasks/types.ts
CHANGED
|
@@ -4,9 +4,9 @@ import type * as Db from '../database-layer/db';
|
|
|
4
4
|
import type { PinejsClient } from '../sbvr-api/sbvr-utils';
|
|
5
5
|
|
|
6
6
|
export const taskStatuses = [
|
|
7
|
-
'
|
|
7
|
+
'queued',
|
|
8
8
|
'cancelled',
|
|
9
|
-
'
|
|
9
|
+
'succeeded',
|
|
10
10
|
'failed',
|
|
11
11
|
] as const;
|
|
12
12
|
export type TaskStatus = (typeof taskStatuses)[number];
|
package/src/tasks/worker.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AnyObject } from 'pinejs-client-core';
|
|
2
2
|
import { tasks as tasksEnv } from '../config-loader/env';
|
|
3
3
|
import type * as Db from '../database-layer/db';
|
|
4
|
+
import { TransactionClosedError } from '../database-layer/db';
|
|
4
5
|
import * as permissions from '../sbvr-api/permissions';
|
|
5
6
|
import { PinejsClient } from '../sbvr-api/sbvr-utils';
|
|
6
7
|
import { sbvrUtils } from '../server-glue/module';
|
|
@@ -80,24 +81,34 @@ export class Worker {
|
|
|
80
81
|
}
|
|
81
82
|
|
|
82
83
|
// Execute handler
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
84
|
+
let status: TaskStatus = 'queued';
|
|
85
|
+
let error: string | undefined;
|
|
86
|
+
try {
|
|
87
|
+
await sbvrUtils.db.transaction(async (handlerTx) => {
|
|
88
|
+
const results = await handler.fn({
|
|
89
|
+
api: new PinejsClient({
|
|
90
|
+
passthrough: {
|
|
91
|
+
tx: handlerTx,
|
|
92
|
+
},
|
|
93
|
+
}),
|
|
94
|
+
params: task.is_executed_with__parameter_set ?? {},
|
|
95
|
+
tx: handlerTx,
|
|
96
|
+
});
|
|
97
|
+
status = results.status;
|
|
98
|
+
error = results.error;
|
|
99
|
+
if (results.status !== 'succeeded' && !handlerTx.isClosed()) {
|
|
100
|
+
await handlerTx.rollback();
|
|
101
|
+
}
|
|
102
|
+
});
|
|
103
|
+
} catch (err) {
|
|
104
|
+
// Ignore closed/rollback errors
|
|
105
|
+
if (!(err instanceof TransactionClosedError)) {
|
|
106
|
+
throw err;
|
|
107
|
+
}
|
|
108
|
+
} finally {
|
|
109
|
+
// Update task with results
|
|
110
|
+
await this.finalize(tx, task, startedOnTime, status, error);
|
|
97
111
|
}
|
|
98
|
-
|
|
99
|
-
// Update task with results
|
|
100
|
-
await this.finalize(tx, task, startedOnTime, result.status, result.error);
|
|
101
112
|
} catch (err) {
|
|
102
113
|
// This shouldn't happen, but if it does we want to log and kill the process
|
|
103
114
|
console.error('Task execution failed:', err);
|
|
@@ -127,7 +138,7 @@ export class Worker {
|
|
|
127
138
|
// Re-enqueue if the task failed but has retries left, remember that
|
|
128
139
|
// attemptCount includes the initial attempt while attempt_limit does not
|
|
129
140
|
if (status === 'failed' && attemptCount < task.attempt_limit) {
|
|
130
|
-
body.status = '
|
|
141
|
+
body.status = 'queued';
|
|
131
142
|
|
|
132
143
|
// Schedule next attempt using exponential backoff
|
|
133
144
|
body.is_scheduled_to_execute_on__time =
|
|
@@ -148,7 +159,7 @@ export class Worker {
|
|
|
148
159
|
// Create new task with same configuration if previous
|
|
149
160
|
// iteration completed and has a cron expression
|
|
150
161
|
if (
|
|
151
|
-
['failed', '
|
|
162
|
+
['failed', 'succeeded'].includes(body.status) &&
|
|
152
163
|
task.is_scheduled_with__cron_expression != null
|
|
153
164
|
) {
|
|
154
165
|
await this.client.post({
|
|
@@ -189,21 +200,21 @@ export class Worker {
|
|
|
189
200
|
|
|
190
201
|
const result = await tx.executeSql(
|
|
191
202
|
`SELECT ${selectColumns}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
203
|
+
FROM task AS t
|
|
204
|
+
WHERE
|
|
205
|
+
t."is executed by-handler" IN (${binds}) AND
|
|
206
|
+
t."status" = 'queued' AND
|
|
207
|
+
t."attempt count" <= t."attempt limit" AND
|
|
208
|
+
(
|
|
209
|
+
t."is scheduled to execute on-time" IS NULL OR
|
|
210
|
+
t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
|
|
211
|
+
)
|
|
212
|
+
ORDER BY
|
|
213
|
+
t."is scheduled to execute on-time" ASC,
|
|
214
|
+
t."priority" DESC,
|
|
215
|
+
t."id" ASC
|
|
216
|
+
LIMIT ${Math.max(this.concurrency - this.executing, 0)}
|
|
217
|
+
FOR UPDATE SKIP LOCKED`,
|
|
207
218
|
handlerNames,
|
|
208
219
|
);
|
|
209
220
|
if (result.rows.length === 0) {
|