@balena/pinejs 17.0.1-build-renovate-major-commander-603807638f3daefd47c7438b099d6badd5384356-1 → 17.1.0-build-joshbwlng-tasks-1c3c1def048695bf69b256a3c560f6777c62c901-1
Sign up to get free protection for your applications and to get access to all the features.
- package/.pinejs-cache.json +1 -1
- package/.versionbot/CHANGELOG.yml +14 -7
- package/CHANGELOG.md +4 -3
- package/VERSION +1 -1
- package/out/config-loader/env.d.ts +4 -0
- package/out/config-loader/env.js +5 -1
- package/out/config-loader/env.js.map +1 -1
- package/out/database-layer/db.d.ts +3 -0
- package/out/database-layer/db.js +17 -0
- package/out/database-layer/db.js.map +1 -1
- package/out/sbvr-api/sbvr-utils.d.ts +1 -0
- package/out/sbvr-api/sbvr-utils.js +5 -1
- package/out/sbvr-api/sbvr-utils.js.map +1 -1
- package/out/server-glue/module.d.ts +1 -0
- package/out/server-glue/module.js +4 -1
- package/out/server-glue/module.js.map +1 -1
- package/out/tasks/common.d.ts +4 -0
- package/out/tasks/common.js +13 -0
- package/out/tasks/common.js.map +1 -0
- package/out/tasks/index.d.ts +8 -0
- package/out/tasks/index.js +142 -0
- package/out/tasks/index.js.map +1 -0
- package/out/tasks/tasks.sbvr +60 -0
- package/out/tasks/types.d.ts +38 -0
- package/out/tasks/types.js +10 -0
- package/out/tasks/types.js.map +1 -0
- package/out/tasks/worker.d.ts +16 -0
- package/out/tasks/worker.js +228 -0
- package/out/tasks/worker.js.map +1 -0
- package/package.json +7 -4
- package/src/config-loader/env.ts +6 -1
- package/src/database-layer/db.ts +25 -0
- package/src/sbvr-api/sbvr-utils.ts +6 -1
- package/src/server-glue/module.ts +3 -0
- package/src/tasks/common.ts +14 -0
- package/src/tasks/index.ts +158 -0
- package/src/tasks/tasks.sbvr +60 -0
- package/src/tasks/types.ts +58 -0
- package/src/tasks/worker.ts +281 -0
@@ -0,0 +1,142 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
+
if (k2 === undefined) k2 = k;
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
+
}
|
8
|
+
Object.defineProperty(o, k2, desc);
|
9
|
+
}) : (function(o, m, k, k2) {
|
10
|
+
if (k2 === undefined) k2 = k;
|
11
|
+
o[k2] = m[k];
|
12
|
+
}));
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
15
|
+
}) : function(o, v) {
|
16
|
+
o["default"] = v;
|
17
|
+
});
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
19
|
+
if (mod && mod.__esModule) return mod;
|
20
|
+
var result = {};
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
22
|
+
__setModuleDefault(result, mod);
|
23
|
+
return result;
|
24
|
+
};
|
25
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
26
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
27
|
+
};
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
29
|
+
exports.addTaskHandler = exports.setup = exports.config = void 0;
|
30
|
+
const cronParser = __importStar(require("cron-parser"));
|
31
|
+
const env_1 = require("../config-loader/env");
|
32
|
+
const errors_1 = require("../sbvr-api/errors");
|
33
|
+
const hooks_1 = require("../sbvr-api/hooks");
|
34
|
+
const sbvrUtils = __importStar(require("../sbvr-api/sbvr-utils"));
|
35
|
+
const common_1 = require("./common");
|
36
|
+
const worker_1 = require("./worker");
|
37
|
+
__exportStar(require("./types"), exports);
|
38
|
+
const modelText = require('./tasks.sbvr');
|
39
|
+
const initSql = `
|
40
|
+
CREATE OR REPLACE FUNCTION notify_task_insert()
|
41
|
+
RETURNS TRIGGER AS $$
|
42
|
+
BEGIN
|
43
|
+
PERFORM pg_notify('${common_1.channel}', NEW.id::text);
|
44
|
+
RETURN NEW;
|
45
|
+
END;
|
46
|
+
$$ LANGUAGE plpgsql;
|
47
|
+
|
48
|
+
CREATE OR REPLACE TRIGGER task_insert_trigger
|
49
|
+
AFTER INSERT ON task
|
50
|
+
FOR EACH ROW WHEN (NEW.status = 'queued' AND NEW."is scheduled to execute on-time" IS NULL)
|
51
|
+
EXECUTE FUNCTION notify_task_insert();
|
52
|
+
|
53
|
+
CREATE INDEX IF NOT EXISTS idx_task_poll ON task USING btree (
|
54
|
+
"is executed by-handler",
|
55
|
+
"is scheduled to execute on-time" ASC,
|
56
|
+
"priority" DESC,
|
57
|
+
"id" ASC
|
58
|
+
) WHERE status = 'queued';
|
59
|
+
`;
|
60
|
+
exports.config = {
|
61
|
+
models: [
|
62
|
+
{
|
63
|
+
modelName: common_1.apiRoot,
|
64
|
+
apiRoot: common_1.apiRoot,
|
65
|
+
modelText,
|
66
|
+
customServerCode: exports,
|
67
|
+
initSql,
|
68
|
+
},
|
69
|
+
],
|
70
|
+
};
|
71
|
+
let worker = null;
|
72
|
+
async function setup(db) {
|
73
|
+
if (db.engine !== 'postgres') {
|
74
|
+
console.warn('Skipping task setup as database not supported');
|
75
|
+
return;
|
76
|
+
}
|
77
|
+
const client = sbvrUtils.api[common_1.apiRoot];
|
78
|
+
worker = new worker_1.Worker(client);
|
79
|
+
(0, hooks_1.addPureHook)('POST', common_1.apiRoot, 'task', {
|
80
|
+
POSTPARSE: async ({ req, request }) => {
|
81
|
+
request.values.is_created_by__actor =
|
82
|
+
req.user?.actor ?? req.apiKey?.actor;
|
83
|
+
if (request.values.is_created_by__actor == null) {
|
84
|
+
throw new errors_1.BadRequestError('Creating tasks with missing actor on req is not allowed');
|
85
|
+
}
|
86
|
+
request.values.status = 'queued';
|
87
|
+
request.values.attempt_count = 0;
|
88
|
+
request.values.attempt_limit ??= 1;
|
89
|
+
request.values.priority ??= 1;
|
90
|
+
if (request.values.is_scheduled_with__cron_expression != null &&
|
91
|
+
request.values.is_scheduled_to_execute_on__time == null) {
|
92
|
+
try {
|
93
|
+
request.values.is_scheduled_to_execute_on__time = cronParser
|
94
|
+
.parseExpression(request.values.is_scheduled_with__cron_expression)
|
95
|
+
.next()
|
96
|
+
.toDate()
|
97
|
+
.toISOString();
|
98
|
+
}
|
99
|
+
catch {
|
100
|
+
throw new errors_1.BadRequestError(`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`);
|
101
|
+
}
|
102
|
+
}
|
103
|
+
if (request.values.is_scheduled_to_execute_on__time != null) {
|
104
|
+
const now = new Date(Date.now() + env_1.tasks.queueIntervalMS);
|
105
|
+
const startTime = new Date(request.values.is_scheduled_to_execute_on__time);
|
106
|
+
if (startTime < now) {
|
107
|
+
throw new errors_1.BadRequestError(`Task scheduled start time must be greater than ${env_1.tasks.queueIntervalMS} milliseconds in the future`);
|
108
|
+
}
|
109
|
+
}
|
110
|
+
const handlerName = request.values.is_executed_by__handler;
|
111
|
+
if (handlerName == null) {
|
112
|
+
throw new errors_1.BadRequestError(`Must specify a task handler to execute`);
|
113
|
+
}
|
114
|
+
const handler = worker?.handlers[handlerName];
|
115
|
+
if (handler == null) {
|
116
|
+
throw new errors_1.BadRequestError(`No task handler with name '${handlerName}' registered`);
|
117
|
+
}
|
118
|
+
if (handler.validate != null) {
|
119
|
+
if (!handler.validate(request.values.is_executed_with__parameter_set)) {
|
120
|
+
throw new errors_1.BadRequestError(`Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
|
121
|
+
}
|
122
|
+
}
|
123
|
+
},
|
124
|
+
});
|
125
|
+
worker.start();
|
126
|
+
}
|
127
|
+
exports.setup = setup;
|
128
|
+
function addTaskHandler(name, fn, schema) {
|
129
|
+
if (worker == null) {
|
130
|
+
throw new Error('Database does not support tasks');
|
131
|
+
}
|
132
|
+
if (worker.handlers[name] != null) {
|
133
|
+
throw new Error(`Task handler with name '${name}' already registered`);
|
134
|
+
}
|
135
|
+
worker.handlers[name] = {
|
136
|
+
name,
|
137
|
+
fn,
|
138
|
+
validate: schema != null ? common_1.ajv.compile(schema) : undefined,
|
139
|
+
};
|
140
|
+
}
|
141
|
+
exports.addTaskHandler = addTaskHandler;
|
142
|
+
//# sourceMappingURL=index.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,kEAAoD;AAEpD,qCAAiD;AAEjD,qCAAkC;AAElC,0CAAwB;AAGxB,MAAM,SAAS,GAAW,OAAO,CAAC,cAAc,CAAC,CAAC;AAIlD,MAAM,OAAO,GAAG;;;;sBAIM,gBAAO;;;;;;;;;;;;;;;;CAgB5B,CAAC;AAEW,QAAA,MAAM,GAAwB;IAC1C,MAAM,EAAE;QACP;YACC,SAAS,EAAE,gBAAO;YAClB,OAAO,EAAP,gBAAO;YACP,SAAS;YACT,gBAAgB,EAAE,OAAO;YACzB,OAAO;SACP;KACD;CACD,CAAC;AAEF,IAAI,MAAM,GAAkB,IAAI,CAAC;AAC1B,KAAK,UAAU,KAAK,CAAC,EAAe;IAE1C,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC9D,OAAO;IACR,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,gBAAO,CAAC,CAAC;IACtC,MAAM,GAAG,IAAI,eAAM,CAAC,MAAM,CAAC,CAAC;IAG5B,IAAA,mBAAW,EAAC,MAAM,EAAE,gBAAO,EAAE,MAAM,EAAE;QACpC,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;YAErC,OAAO,CAAC,MAAM,CAAC,oBAAoB;gBAClC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAI,wBAAe,CACxB,yDAAyD,CACzD,CAAC;YACH,CAAC;YAGD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,QAAQ,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC;YAEnC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;YAG9B,IACC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAI,IAAI;gBACzD,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EACtD,CAAC;gBACF,IAAI,CAAC;oBACJ,OAAO,CAAC,MAAM,CAAC,gCAAgC,GAAG,UAAU;yBAC1D,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,CAAC;yBAClE,IAAI,EAAE;yBACN,MAAM,EAAE;yBACR,WAAW,EAAE,CAAC;gBACjB,CAAC;gBAAC,MAAM,CAAC;oBACR,MAAM,IAAI,wBAAe,CACxB,4BAA4B,OAAO,CAAC,MAAM,CAAC,kCAAkC,EAAE,CAC/E,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,IAAI,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,WAAQ,CAAC,eAAe,CAAC,CAAC;gBAC5D,MAAM,SAAS,GAAG,IAAI,IAAI,CACzB,OAAO,CAAC,MAAM,CAAC,gCAAgC,CAC/C,CAAC;gBACF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBACrB,MAAM,IAAI,wBAAe,CACxB,kDAAkD,WAAQ,CAAC,eAAe,6BAA6B,CACvG,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC;YAC3D,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,wBAAe,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,wBAAe,CACxB,8BAA8B,WAAW,cAAc,CACvD,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,wBAAe,CACxB,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAnFD,sBAmFC;AAGD,SAAgB,cAAc,CAC7B,IAAY,EACZ,EAAqB,EACrB,MAAe;IAEf,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,sBAAsB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;QACvB,IAAI;QACJ,EAAE;QACF,QAAQ,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,YAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC;AACH,CAAC;AAjBD,wCAiBC"}
|
@@ -0,0 +1,60 @@
|
|
1
|
+
Vocabulary: tasks
|
2
|
+
|
3
|
+
Term: id
|
4
|
+
Concept Type: Big Serial (Type)
|
5
|
+
Term: actor
|
6
|
+
Concept Type: Integer (Type)
|
7
|
+
Term: attempt count
|
8
|
+
Concept Type: Integer (Type)
|
9
|
+
Term: attempt limit
|
10
|
+
Concept Type: Integer (Type)
|
11
|
+
Term: cron expression
|
12
|
+
Concept Type: Short Text (Type)
|
13
|
+
Term: error message
|
14
|
+
Concept Type: Short Text (Type)
|
15
|
+
Term: handler
|
16
|
+
Concept Type: Short Text (Type)
|
17
|
+
Term: key
|
18
|
+
Concept Type: Short Text (Type)
|
19
|
+
Term: parameter set
|
20
|
+
Concept Type: JSON (Type)
|
21
|
+
Term: priority
|
22
|
+
Concept Type: Integer (Type)
|
23
|
+
Term: status
|
24
|
+
Concept Type: Short Text (Type)
|
25
|
+
Term: time
|
26
|
+
Concept Type: Date Time (Type)
|
27
|
+
|
28
|
+
Term: task
|
29
|
+
Fact type: task has id
|
30
|
+
Necessity: each task has exactly one id
|
31
|
+
Fact type: task has key
|
32
|
+
Necessity: each task has at most one key
|
33
|
+
Fact type: task is created by actor
|
34
|
+
Necessity: each task is created by exactly one actor
|
35
|
+
Fact type: task is executed by handler
|
36
|
+
Necessity: each task is executed by exactly one handler
|
37
|
+
Fact type: task is executed with parameter set
|
38
|
+
Necessity: each task is executed with at most one parameter set
|
39
|
+
Fact type: task has priority
|
40
|
+
Necessity: each task has exactly one priority
|
41
|
+
Necessity: each task has a priority that is greater than or equal to 0
|
42
|
+
Fact type: task is scheduled with cron expression
|
43
|
+
Necessity: each task is scheduled with at most one cron expression
|
44
|
+
Fact type: task is scheduled to execute on time
|
45
|
+
Necessity: each task is scheduled to execute on at most one time
|
46
|
+
Fact type: task has status
|
47
|
+
Necessity: each task has exactly one status
|
48
|
+
Definition: "queued" or "cancelled" or "succeeded" or "failed"
|
49
|
+
Fact type: task started on time
|
50
|
+
Necessity: each task started on at most one time
|
51
|
+
Fact type: task ended on time
|
52
|
+
Necessity: each task ended on at most one time
|
53
|
+
Fact type: task has error message
|
54
|
+
Necessity: each task has at most one error message
|
55
|
+
Fact type: task has attempt count
|
56
|
+
Necessity: each task has exactly one attempt count
|
57
|
+
Fact type: task has attempt limit
|
58
|
+
Necessity: each task has exactly one attempt limit
|
59
|
+
Necessity: each task has an attempt limit that is greater than or equal to 1
|
60
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
import type { ValidateFunction } from 'ajv';
|
2
|
+
import type { AnyObject } from 'pinejs-client-core';
|
3
|
+
import type * as Db from '../database-layer/db';
|
4
|
+
import type { PinejsClient } from '../sbvr-api/sbvr-utils';
|
5
|
+
export declare const taskStatuses: readonly ["queued", "cancelled", "succeeded", "failed"];
|
6
|
+
export type TaskStatus = (typeof taskStatuses)[number];
|
7
|
+
export interface Task {
|
8
|
+
id: number;
|
9
|
+
created_at: Date;
|
10
|
+
modified_at: Date;
|
11
|
+
is_created_by__actor: number;
|
12
|
+
is_executed_by__handler: string;
|
13
|
+
is_executed_with__parameter_set: object | null;
|
14
|
+
is_scheduled_with__cron_expression: string | null;
|
15
|
+
is_scheduled_to_execute_on__time: Date | null;
|
16
|
+
priority: number;
|
17
|
+
status: TaskStatus;
|
18
|
+
started_on__time: Date | null;
|
19
|
+
ended_on__time: Date | null;
|
20
|
+
error_message: string | null;
|
21
|
+
attempt_count: number;
|
22
|
+
attempt_limit: number;
|
23
|
+
}
|
24
|
+
export type PartialTask = Pick<Task, 'id' | 'is_created_by__actor' | 'is_executed_by__handler' | 'is_executed_with__parameter_set' | 'is_scheduled_with__cron_expression' | 'priority' | 'attempt_count' | 'attempt_limit'>;
|
25
|
+
export interface TaskArgs {
|
26
|
+
api: PinejsClient;
|
27
|
+
params: AnyObject;
|
28
|
+
tx: Db.Tx;
|
29
|
+
}
|
30
|
+
export type TaskResponse = Promise<{
|
31
|
+
status: TaskStatus;
|
32
|
+
error?: string;
|
33
|
+
}>;
|
34
|
+
export interface TaskHandler {
|
35
|
+
name: string;
|
36
|
+
fn: (options: TaskArgs) => TaskResponse;
|
37
|
+
validate?: ValidateFunction;
|
38
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,QAAQ;IACR,WAAW;IACX,WAAW;IACX,QAAQ;CACC,CAAC"}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
import { PinejsClient } from '../sbvr-api/sbvr-utils';
|
2
|
+
import type { TaskHandler } from './types';
|
3
|
+
export declare class Worker {
|
4
|
+
handlers: Record<string, TaskHandler>;
|
5
|
+
private readonly concurrency;
|
6
|
+
private readonly interval;
|
7
|
+
private client;
|
8
|
+
private executing;
|
9
|
+
constructor(client: PinejsClient);
|
10
|
+
private canExecute;
|
11
|
+
private execute;
|
12
|
+
private finalize;
|
13
|
+
private getNextAttemptTime;
|
14
|
+
private poll;
|
15
|
+
start(): void;
|
16
|
+
}
|
@@ -0,0 +1,228 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
+
if (k2 === undefined) k2 = k;
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
+
}
|
8
|
+
Object.defineProperty(o, k2, desc);
|
9
|
+
}) : (function(o, m, k, k2) {
|
10
|
+
if (k2 === undefined) k2 = k;
|
11
|
+
o[k2] = m[k];
|
12
|
+
}));
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
15
|
+
}) : function(o, v) {
|
16
|
+
o["default"] = v;
|
17
|
+
});
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
19
|
+
if (mod && mod.__esModule) return mod;
|
20
|
+
var result = {};
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
22
|
+
__setModuleDefault(result, mod);
|
23
|
+
return result;
|
24
|
+
};
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
26
|
+
exports.Worker = void 0;
|
27
|
+
const promises_1 = require("node:timers/promises");
|
28
|
+
const env_1 = require("../config-loader/env");
|
29
|
+
const db_1 = require("../database-layer/db");
|
30
|
+
const permissions = __importStar(require("../sbvr-api/permissions"));
|
31
|
+
const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
|
32
|
+
const module_1 = require("../server-glue/module");
|
33
|
+
const common_1 = require("./common");
|
34
|
+
const selectColumns = Object.entries({
|
35
|
+
id: 'id',
|
36
|
+
'is executed by-handler': 'is_executed_by__handler',
|
37
|
+
'is executed with-parameter set': 'is_executed_with__parameter_set',
|
38
|
+
'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
|
39
|
+
'attempt count': 'attempt_count',
|
40
|
+
'attempt limit': 'attempt_limit',
|
41
|
+
priority: 'priority',
|
42
|
+
'is created by-actor': 'is_created_by__actor',
|
43
|
+
})
|
44
|
+
.map(([key, value]) => `t."${key}" AS "${value}"`)
|
45
|
+
.join(', ');
|
46
|
+
class Worker {
|
47
|
+
handlers = {};
|
48
|
+
concurrency;
|
49
|
+
interval;
|
50
|
+
client;
|
51
|
+
executing = 0;
|
52
|
+
constructor(client) {
|
53
|
+
this.client = client;
|
54
|
+
this.concurrency = env_1.tasks.queueConcurrency;
|
55
|
+
this.interval = env_1.tasks.queueIntervalMS;
|
56
|
+
}
|
57
|
+
canExecute() {
|
58
|
+
return (this.executing < this.concurrency && Object.keys(this.handlers).length > 0);
|
59
|
+
}
|
60
|
+
async execute(tx, task) {
|
61
|
+
this.executing++;
|
62
|
+
try {
|
63
|
+
const handler = this.handlers[task.is_executed_by__handler];
|
64
|
+
const startedOnTime = new Date();
|
65
|
+
if (handler == null) {
|
66
|
+
await this.finalize(tx, task, startedOnTime, 'failed', 'Matching task handler not found');
|
67
|
+
return;
|
68
|
+
}
|
69
|
+
if (handler.validate != null &&
|
70
|
+
!handler.validate(task.is_executed_with__parameter_set)) {
|
71
|
+
await this.finalize(tx, task, startedOnTime, 'failed', `Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
|
72
|
+
return;
|
73
|
+
}
|
74
|
+
let status = 'queued';
|
75
|
+
let error;
|
76
|
+
try {
|
77
|
+
await module_1.sbvrUtils.db.transaction(async (handlerTx) => {
|
78
|
+
const results = await handler.fn({
|
79
|
+
api: new sbvr_utils_1.PinejsClient({
|
80
|
+
passthrough: {
|
81
|
+
tx: handlerTx,
|
82
|
+
},
|
83
|
+
}),
|
84
|
+
params: task.is_executed_with__parameter_set ?? {},
|
85
|
+
tx: handlerTx,
|
86
|
+
});
|
87
|
+
status = results.status;
|
88
|
+
error = results.error;
|
89
|
+
if (results.status !== 'succeeded' && !handlerTx.isClosed()) {
|
90
|
+
await handlerTx.rollback();
|
91
|
+
}
|
92
|
+
});
|
93
|
+
}
|
94
|
+
catch (err) {
|
95
|
+
if (!(err instanceof db_1.TransactionClosedError)) {
|
96
|
+
throw err;
|
97
|
+
}
|
98
|
+
}
|
99
|
+
finally {
|
100
|
+
await this.finalize(tx, task, startedOnTime, status, error);
|
101
|
+
}
|
102
|
+
}
|
103
|
+
catch (err) {
|
104
|
+
console.error(`Failed to execute task ${task.id} with handler ${task.is_executed_by__handler}:`, err);
|
105
|
+
process.exit(1);
|
106
|
+
}
|
107
|
+
finally {
|
108
|
+
this.executing--;
|
109
|
+
}
|
110
|
+
}
|
111
|
+
async finalize(tx, task, startedOnTime, status, errorMessage) {
|
112
|
+
const attemptCount = task.attempt_count + 1;
|
113
|
+
const body = {
|
114
|
+
started_on__time: startedOnTime,
|
115
|
+
ended_on__time: new Date(),
|
116
|
+
status,
|
117
|
+
attempt_count: attemptCount,
|
118
|
+
...(errorMessage != null && { error_message: errorMessage }),
|
119
|
+
};
|
120
|
+
if (status === 'failed' && attemptCount < task.attempt_limit) {
|
121
|
+
body.status = 'queued';
|
122
|
+
body.is_scheduled_to_execute_on__time =
|
123
|
+
this.getNextAttemptTime(attemptCount);
|
124
|
+
}
|
125
|
+
await this.client.patch({
|
126
|
+
resource: 'task',
|
127
|
+
passthrough: {
|
128
|
+
tx,
|
129
|
+
req: permissions.root,
|
130
|
+
},
|
131
|
+
id: task.id,
|
132
|
+
body,
|
133
|
+
});
|
134
|
+
if (['failed', 'succeeded'].includes(body.status) &&
|
135
|
+
task.is_scheduled_with__cron_expression != null) {
|
136
|
+
await this.client.post({
|
137
|
+
resource: 'task',
|
138
|
+
passthrough: {
|
139
|
+
tx,
|
140
|
+
req: permissions.root,
|
141
|
+
},
|
142
|
+
options: {
|
143
|
+
returnResource: false,
|
144
|
+
},
|
145
|
+
body: {
|
146
|
+
attempt_limit: task.attempt_limit,
|
147
|
+
is_created_by__actor: task.is_created_by__actor,
|
148
|
+
is_executed_by__handler: task.is_executed_by__handler,
|
149
|
+
is_executed_with__parameter_set: task.is_executed_with__parameter_set,
|
150
|
+
is_scheduled_with__cron_expression: task.is_scheduled_with__cron_expression,
|
151
|
+
priority: task.priority,
|
152
|
+
},
|
153
|
+
});
|
154
|
+
}
|
155
|
+
}
|
156
|
+
getNextAttemptTime(attempt) {
|
157
|
+
const delay = Math.ceil(Math.exp(Math.min(10, attempt))) * 1000;
|
158
|
+
return new Date(Date.now() + delay);
|
159
|
+
}
|
160
|
+
poll() {
|
161
|
+
let executed = false;
|
162
|
+
void (async () => {
|
163
|
+
try {
|
164
|
+
const handlerNames = Object.keys(this.handlers);
|
165
|
+
const binds = handlerNames
|
166
|
+
.map((_, index) => `$${index + 1}`)
|
167
|
+
.join(', ');
|
168
|
+
if (!this.canExecute()) {
|
169
|
+
return;
|
170
|
+
}
|
171
|
+
await module_1.sbvrUtils.db.transaction(async (tx) => {
|
172
|
+
const result = await tx.executeSql(`SELECT ${selectColumns}
|
173
|
+
FROM task AS t
|
174
|
+
WHERE
|
175
|
+
t."is executed by-handler" IN (${binds}) AND
|
176
|
+
t."status" = 'queued' AND
|
177
|
+
t."attempt count" <= t."attempt limit" AND
|
178
|
+
(
|
179
|
+
t."is scheduled to execute on-time" IS NULL OR
|
180
|
+
t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
|
181
|
+
)
|
182
|
+
ORDER BY
|
183
|
+
t."is scheduled to execute on-time" ASC,
|
184
|
+
t."priority" DESC,
|
185
|
+
t."id" ASC
|
186
|
+
LIMIT ${Math.max(this.concurrency - this.executing, 0)}
|
187
|
+
FOR UPDATE SKIP LOCKED`, handlerNames);
|
188
|
+
if (result.rows.length === 0) {
|
189
|
+
return;
|
190
|
+
}
|
191
|
+
await Promise.all(result.rows.map(async (row) => {
|
192
|
+
await this.execute(tx, row);
|
193
|
+
}));
|
194
|
+
executed = true;
|
195
|
+
});
|
196
|
+
}
|
197
|
+
catch (err) {
|
198
|
+
console.error('Failed polling for tasks:', err);
|
199
|
+
}
|
200
|
+
finally {
|
201
|
+
if (!executed) {
|
202
|
+
await (0, promises_1.setTimeout)(this.interval);
|
203
|
+
}
|
204
|
+
this.poll();
|
205
|
+
}
|
206
|
+
})();
|
207
|
+
}
|
208
|
+
start() {
|
209
|
+
if (module_1.sbvrUtils.db.engine !== 'postgres' || module_1.sbvrUtils.db.on == null) {
|
210
|
+
throw new Error('Database does not support tasks, giving up on starting worker');
|
211
|
+
}
|
212
|
+
module_1.sbvrUtils.db.on('notification', async (msg) => {
|
213
|
+
if (this.canExecute()) {
|
214
|
+
await module_1.sbvrUtils.db.transaction(async (tx) => {
|
215
|
+
const result = await tx.executeSql(`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`, [msg.payload]);
|
216
|
+
if (result.rows.length > 0) {
|
217
|
+
await this.execute(tx, result.rows[0]);
|
218
|
+
}
|
219
|
+
});
|
220
|
+
}
|
221
|
+
}, {
|
222
|
+
channel: common_1.channel,
|
223
|
+
});
|
224
|
+
this.poll();
|
225
|
+
}
|
226
|
+
}
|
227
|
+
exports.Worker = Worker;
|
228
|
+
//# sourceMappingURL=worker.js.map
|
@@ -0,0 +1 @@
|
|
1
|
+
{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mDAAkD;AAElD,8CAAyD;AAEzD,6CAA8D;AAC9D,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IACX,QAAQ,GAAgC,EAAE,CAAC;IACjC,WAAW,CAAS;IACpB,QAAQ,CAAS;IAC1B,MAAM,CAAe;IACrB,SAAS,GAAG,CAAC,CAAC;IAEtB,YAAY,MAAoB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,OAAO,CAAC,EAAS,EAAE,IAAiB;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,IAAI,MAAM,GAAe,QAAQ,CAAC;YAClC,IAAI,KAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;oBAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC;wBAChC,GAAG,EAAE,IAAI,yBAAY,CAAC;4BACrB,WAAW,EAAE;gCACZ,EAAE,EAAE,SAAS;6BACb;yBACD,CAAC;wBACF,MAAM,EAAE,IAAI,CAAC,+BAA+B,IAAI,EAAE;wBAClD,EAAE,EAAE,SAAS;qBACb,CAAC,CAAC;oBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBACxB,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;oBACtB,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC;wBAC7D,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;oBAC5B,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBAEd,IAAI,CAAC,CAAC,GAAG,YAAY,2BAAsB,CAAC,EAAE,CAAC;oBAC9C,MAAM,GAAG,CAAC;gBACX,CAAC;YACF,CAAC;oBAAS,CAAC;gBAEV,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAEd,OAAO,CAAC,KAAK,CACZ,0BAA0B,IAAI,CAAC,EAAE,iBAAiB,IAAI,CAAC,uBAAuB,GAAG,EACjF,GAAG,CACH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,QAAQ,CACrB,EAAS,EACT,IAAiB,EACjB,aAAmB,EACnB,MAAkB,EAClB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAc;YACvB,gBAAgB,EAAE,aAAa;YAC/B,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;SAC5D,CAAC;QAIF,IAAI,MAAM,KAAK,QAAQ,IAAI,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;YAGvB,IAAI,CAAC,gCAAgC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAGD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE;gBACZ,EAAE;gBACF,GAAG,EAAE,WAAW,CAAC,IAAI;aACrB;YACD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;SACJ,CAAC,CAAC;QAIH,IACC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7C,IAAI,CAAC,kCAAkC,IAAI,IAAI,EAC9C,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,MAAM;gBAChB,WAAW,EAAE;oBACZ,EAAE;oBACF,GAAG,EAAE,WAAW,CAAC,IAAI;iBACrB;gBACD,OAAO,EAAE;oBACR,cAAc,EAAE,KAAK;iBACrB;gBACD,IAAI,EAAE;oBACL,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,+BAA+B,EAAE,IAAI,CAAC,+BAA+B;oBACrE,kCAAkC,EACjC,IAAI,CAAC,kCAAkC;oBACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACvB;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAGO,kBAAkB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAChE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACrC,CAAC;IAGO,IAAI;QACX,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,YAAY;qBACxB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;qBAClC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACxB,OAAO;gBACR,CAAC;gBACD,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa;;;wCAGW,KAAK;;;;;+EAKkC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;;;;;;cAMhG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;6BAC/B,EACvB,YAAY,CACZ,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC9B,OAAO;oBACR,CAAC;oBAGD,MAAM,OAAO,CAAC,GAAG,CAChB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;wBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,GAAkB,CAAC,CAAC;oBAC5C,CAAC,CAAC,CACF,CAAC;oBACF,QAAQ,GAAG,IAAI,CAAC;gBACjB,CAAC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,MAAM,IAAA,qBAAU,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,IAAI,EAAE,CAAC;YACb,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;IAGM,KAAK;QAEX,IAAI,kBAAS,CAAC,EAAE,CAAC,MAAM,KAAK,UAAU,IAAI,kBAAS,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,+DAA+D,CAC/D,CAAC;QACH,CAAC;QACD,kBAAS,CAAC,EAAE,CAAC,EAAE,CACd,cAAc,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACb,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa,sDAAsD,EAC7E,CAAC,GAAG,CAAC,OAAO,CAAC,CACb,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAgB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,EACD;YACC,OAAO,EAAP,gBAAO;SACP,CACD,CAAC;QACF,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;CACD;AA5PD,wBA4PC"}
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@balena/pinejs",
|
3
|
-
"version": "17.0
|
3
|
+
"version": "17.1.0-build-joshbwlng-tasks-1c3c1def048695bf69b256a3c560f6777c62c901-1",
|
4
4
|
"main": "out/server-glue/module",
|
5
5
|
"type": "commonjs",
|
6
6
|
"repository": "git@github.com:balena-io/pinejs.git",
|
@@ -20,7 +20,7 @@
|
|
20
20
|
"webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
|
21
21
|
"lint": "balena-lint -t tsconfig.dev.json -e js -e ts src build typings Gruntfile.ts && npx tsc --project tsconfig.dev.json --noEmit",
|
22
22
|
"test": "npm run lint && npm run build && npm run webpack-build && npm run test:compose",
|
23
|
-
"test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 npm run mocha",
|
23
|
+
"test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 npm run mocha",
|
24
24
|
"mocha": "TS_NODE_FILES=true mocha",
|
25
25
|
"prettify": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.ts"
|
26
26
|
},
|
@@ -51,8 +51,10 @@
|
|
51
51
|
"@types/pg": "^8.11.6",
|
52
52
|
"@types/randomstring": "^1.3.0",
|
53
53
|
"@types/websql": "^0.0.30",
|
54
|
+
"ajv": "^8.12.0",
|
54
55
|
"busboy": "^1.6.0",
|
55
|
-
"commander": "^
|
56
|
+
"commander": "^11.1.0",
|
57
|
+
"cron-parser": "^4.9.0",
|
56
58
|
"deep-freeze": "^0.0.1",
|
57
59
|
"eventemitter3": "^5.0.1",
|
58
60
|
"express-session": "^1.18.0",
|
@@ -89,6 +91,7 @@
|
|
89
91
|
"grunt-ts": "^6.0.0-beta.22",
|
90
92
|
"grunt-webpack": "^6.0.0",
|
91
93
|
"husky": "^9.0.11",
|
94
|
+
"json-schema-to-ts": "^3.1.0",
|
92
95
|
"lint-staged": "^15.2.5",
|
93
96
|
"load-grunt-tasks": "^5.1.0",
|
94
97
|
"mocha": "^10.4.0",
|
@@ -144,6 +147,6 @@
|
|
144
147
|
"recursive": true
|
145
148
|
},
|
146
149
|
"versionist": {
|
147
|
-
"publishedAt": "2024-06-
|
150
|
+
"publishedAt": "2024-06-10T04:16:15.696Z"
|
148
151
|
}
|
149
152
|
}
|
package/src/config-loader/env.ts
CHANGED
@@ -49,7 +49,7 @@ export const cache = {
|
|
49
49
|
apiKeyActorId: false as CacheOpts,
|
50
50
|
};
|
51
51
|
|
52
|
-
import { boolVar } from '@balena/env-parsing';
|
52
|
+
import { boolVar, intVar } from '@balena/env-parsing';
|
53
53
|
import memoize from 'memoizee';
|
54
54
|
import memoizeWeak = require('memoizee/weak');
|
55
55
|
export const createCache = <T extends (...args: any[]) => any>(
|
@@ -146,3 +146,8 @@ export const migrator = {
|
|
146
146
|
*/
|
147
147
|
asyncMigrationIsEnabled: boolVar('PINEJS_ASYNC_MIGRATION_ENABLED', true),
|
148
148
|
};
|
149
|
+
|
150
|
+
export const tasks = {
|
151
|
+
queueConcurrency: intVar('PINEJS_QUEUE_CONCURRENCY', 0),
|
152
|
+
queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
|
153
|
+
};
|
package/src/database-layer/db.ts
CHANGED
@@ -98,6 +98,13 @@ export interface Database extends BaseDatabase {
|
|
98
98
|
) => Promise<Result>;
|
99
99
|
transaction: TransactionFn;
|
100
100
|
readTransaction: TransactionFn;
|
101
|
+
on?: (
|
102
|
+
name: 'notification',
|
103
|
+
fn: (...args: any[]) => Promise<void>,
|
104
|
+
options?: {
|
105
|
+
channel?: string;
|
106
|
+
},
|
107
|
+
) => void;
|
101
108
|
}
|
102
109
|
|
103
110
|
interface EngineParams {
|
@@ -689,6 +696,24 @@ if (maybePg != null) {
|
|
689
696
|
return {
|
690
697
|
engine: Engines.postgres,
|
691
698
|
executeSql: atomicExecuteSql,
|
699
|
+
on: async (name, fn, options) => {
|
700
|
+
if (name === 'notification' && options?.channel === undefined) {
|
701
|
+
throw new Error('Missing channel option for notification listener');
|
702
|
+
}
|
703
|
+
|
704
|
+
const client = await pool.connect();
|
705
|
+
client.on(name, async (msg) => {
|
706
|
+
try {
|
707
|
+
await fn(msg);
|
708
|
+
} catch (error) {
|
709
|
+
console.error('Error handling message:', error);
|
710
|
+
}
|
711
|
+
});
|
712
|
+
|
713
|
+
if (name === 'notification' && options?.channel !== undefined) {
|
714
|
+
await client.query(`LISTEN "${options.channel}";`);
|
715
|
+
}
|
716
|
+
},
|
692
717
|
transaction: createTransaction(async (stackTraceErr) => {
|
693
718
|
const client = await pool.connect();
|
694
719
|
const tx = new PostgresTx(client, false, stackTraceErr);
|
@@ -42,6 +42,7 @@ import { generateODataMetadata } from '../odata-metadata/odata-metadata-generato
|
|
42
42
|
|
43
43
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
44
44
|
const devModel = require('./dev.sbvr');
|
45
|
+
import * as tasks from '../tasks';
|
45
46
|
import * as permissions from './permissions';
|
46
47
|
import {
|
47
48
|
BadRequestError,
|
@@ -77,6 +78,7 @@ export {
|
|
77
78
|
addPureHook,
|
78
79
|
addSideEffectHook,
|
79
80
|
} from './hooks';
|
81
|
+
export { addTaskHandler } from '../tasks';
|
80
82
|
|
81
83
|
import memoizeWeak = require('memoizee/weak');
|
82
84
|
import * as controlFlow from './control-flow';
|
@@ -773,7 +775,7 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
|
|
773
775
|
// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
|
774
776
|
// Here the skipped migrations that haven't been set properly are covered
|
775
777
|
// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
|
776
|
-
// One
|
778
|
+
// One specific case are tests to run tests against migrated and unmigrated database states
|
777
779
|
|
778
780
|
for (const modelKey of Object.keys(models)) {
|
779
781
|
const pendingToSetExecutedMigrations =
|
@@ -783,6 +785,9 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
|
|
783
785
|
await setExecutedMigrations(tx, modelKey, pendingToSetExecutedMigrations);
|
784
786
|
}
|
785
787
|
}
|
788
|
+
|
789
|
+
// Initialize task worker and create required hooks
|
790
|
+
await tasks.setup(db);
|
786
791
|
};
|
787
792
|
|
788
793
|
const cleanupModel = (vocab: string) => {
|