@balena/pinejs 17.1.0-build-model-based-typings-437bb06f44567532aec78e550f3d545732466411-1 → 17.1.0-build-joshbwlng-tasks-61ce10e444abec6afea3fec43e9a5c37c7cedea6-1

package/out/tasks/worker.js

@@ -0,0 +1,228 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const promises_1 = require("node:timers/promises");
+const env_1 = require("../config-loader/env");
+const db_1 = require("../database-layer/db");
+const permissions = __importStar(require("../sbvr-api/permissions"));
+const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
+const module_1 = require("../server-glue/module");
+const common_1 = require("./common");
+const selectColumns = Object.entries({
+    id: 'id',
+    'is executed by-handler': 'is_executed_by__handler',
+    'is executed with-parameter set': 'is_executed_with__parameter_set',
+    'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
+    'attempt count': 'attempt_count',
+    'attempt limit': 'attempt_limit',
+    priority: 'priority',
+    'is created by-actor': 'is_created_by__actor',
+})
+    .map(([key, value]) => `t."${key}" AS "${value}"`)
+    .join(', ');
+class Worker {
+    client;
+    handlers = {};
+    concurrency;
+    interval;
+    executing = 0;
+    constructor(client) {
+        this.client = client;
+        this.concurrency = env_1.tasks.queueConcurrency;
+        this.interval = env_1.tasks.queueIntervalMS;
+    }
+    canExecute() {
+        return (this.executing < this.concurrency && Object.keys(this.handlers).length > 0);
+    }
+    async execute(task, tx) {
+        this.executing++;
+        try {
+            const handler = this.handlers[task.is_executed_by__handler];
+            const startedOnTime = new Date();
+            if (handler == null) {
+                await this.finalize(tx, task, startedOnTime, 'failed', 'Matching task handler not found');
+                return;
+            }
+            if (handler.validate != null &&
+                !handler.validate(task.is_executed_with__parameter_set)) {
+                await this.finalize(tx, task, startedOnTime, 'failed', `Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
+                return;
+            }
+            let status = 'queued';
+            let error;
+            try {
+                await module_1.sbvrUtils.db.transaction(async (handlerTx) => {
+                    const results = await handler.fn({
+                        api: new sbvr_utils_1.PinejsClient({
+                            passthrough: {
+                                tx: handlerTx,
+                            },
+                        }),
+                        params: task.is_executed_with__parameter_set ?? {},
+                        tx: handlerTx,
+                    });
+                    status = results.status;
+                    error = results.error;
+                    if (results.status !== 'succeeded' && !handlerTx.isClosed()) {
+                        await handlerTx.rollback();
+                    }
+                });
+            }
+            catch (err) {
+                if (!(err instanceof db_1.TransactionClosedError)) {
+                    throw err;
+                }
+            }
+            finally {
+                await this.finalize(tx, task, startedOnTime, status, error);
+            }
+        }
+        catch (err) {
+            console.error(`Failed to execute task ${task.id} with handler ${task.is_executed_by__handler}:`, err);
+            process.exit(1);
+        }
+        finally {
+            this.executing--;
+        }
+    }
+    async finalize(tx, task, startedOnTime, status, errorMessage) {
+        const attemptCount = task.attempt_count + 1;
+        const body = {
+            started_on__time: startedOnTime,
+            ended_on__time: new Date(),
+            status,
+            attempt_count: attemptCount,
+            ...(errorMessage != null && { error_message: errorMessage }),
+        };
+        if (status === 'failed' && attemptCount < task.attempt_limit) {
+            body.status = 'queued';
+            body.is_scheduled_to_execute_on__time =
+                this.getNextAttemptTime(attemptCount);
+        }
+        await this.client.patch({
+            resource: 'task',
+            passthrough: {
+                tx,
+                req: permissions.root,
+            },
+            id: task.id,
+            body,
+        });
+        if (['failed', 'succeeded'].includes(body.status) &&
+            task.is_scheduled_with__cron_expression != null) {
+            await this.client.post({
+                resource: 'task',
+                passthrough: {
+                    tx,
+                    req: permissions.root,
+                },
+                options: {
+                    returnResource: false,
+                },
+                body: {
+                    attempt_limit: task.attempt_limit,
+                    is_created_by__actor: task.is_created_by__actor,
+                    is_executed_by__handler: task.is_executed_by__handler,
+                    is_executed_with__parameter_set: task.is_executed_with__parameter_set,
+                    is_scheduled_with__cron_expression: task.is_scheduled_with__cron_expression,
+                    priority: task.priority,
+                },
+            });
+        }
+    }
+    getNextAttemptTime(attempt) {
+        const delay = Math.ceil(Math.exp(Math.min(10, attempt)));
+        return new Date(Date.now() + delay);
+    }
+    poll() {
+        let executed = false;
+        void (async () => {
+            try {
+                const handlerNames = Object.keys(this.handlers);
+                const binds = handlerNames
+                    .map((_, index) => `$${index + 1}`)
+                    .join(', ');
+                if (!this.canExecute()) {
+                    return;
+                }
+                await module_1.sbvrUtils.db.transaction(async (tx) => {
+                    const result = await tx.executeSql(`SELECT ${selectColumns}
+						FROM task AS t
+						WHERE
+							t."is executed by-handler" IN (${binds}) AND
+							t."status" = 'queued' AND
+							t."attempt count" <= t."attempt limit" AND
+							(
+								t."is scheduled to execute on-time" IS NULL OR
+								t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
+							)
+						ORDER BY
+							t."is scheduled to execute on-time" ASC,
+							t."priority" DESC,
+							t."id" ASC
+						LIMIT ${Math.max(this.concurrency - this.executing, 0)}
+						FOR UPDATE SKIP LOCKED`, handlerNames);
+                    if (result.rows.length === 0) {
+                        return;
+                    }
+                    await Promise.all(result.rows.map(async (row) => {
+                        await this.execute(row, tx);
+                    }));
+                    executed = true;
+                });
+            }
+            catch (err) {
+                console.error('Failed polling for tasks:', err);
+            }
+            finally {
+                if (!executed) {
+                    await (0, promises_1.setTimeout)(this.interval);
+                }
+                this.poll();
+            }
+        })();
+    }
+    start() {
+        if (module_1.sbvrUtils.db.engine !== 'postgres' || module_1.sbvrUtils.db.on == null) {
+            throw new Error('Database does not support tasks, giving up on starting worker');
+        }
+        module_1.sbvrUtils.db.on('notification', async (msg) => {
+            if (this.canExecute()) {
+                await module_1.sbvrUtils.db.transaction(async (tx) => {
+                    const result = await tx.executeSql(`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`, [msg.payload]);
+                    if (result.rows.length > 0) {
+                        await this.execute(result.rows[0], tx);
+                    }
+                });
+            }
+        }, {
+            channel: common_1.channel,
+        });
+        this.poll();
+    }
+}
+exports.Worker = Worker;
+//# sourceMappingURL=worker.js.map

package/out/tasks/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mDAAkD;AAElD,8CAAyD;AAEzD,6CAA8D;AAC9D,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IAMW;IALtB,QAAQ,GAAgC,EAAE,CAAC;IACjC,WAAW,CAAS;IACpB,QAAQ,CAAS;IAC1B,SAAS,GAAG,CAAC,CAAC;IAEtB,YAA6B,MAAoB;QAApB,WAAM,GAAN,MAAM,CAAc;QAChD,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,IAAiB,EAAE,EAAS;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,IAAI,MAAM,GAAe,QAAQ,CAAC;YAClC,IAAI,KAAyB,CAAC;YAC9B,IAAI,CAAC;gBACJ,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;oBAClD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC;wBAChC,GAAG,EAAE,IAAI,yBAAY,CAAC;4BACrB,WAAW,EAAE;gCACZ,EAAE,EAAE,SAAS;6BACb;yBACD,CAAC;wBACF,MAAM,EAAE,IAAI,CAAC,+BAA+B,IAAI,EAAE;wBAClD,EAAE,EAAE,SAAS;qBACb,CAAC,CAAC;oBACH,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBACxB,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;oBACtB,IAAI,OAAO,CAAC,MAAM,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,CAAC;wBAC7D,MAAM,SAAS,CAAC,QAAQ,EAAE,CAAC;oBAC5B,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBAEd,IAAI,CAAC,CAAC,GAAG,YAAY,2BAAsB,CAAC,EAAE,CAAC;oBAC9C,MAAM,GAAG,CAAC;gBACX,CAAC;YACF,CAAC;oBAAS,CAAC;gBAEV,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YAC7D,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAEd,OAAO,CAAC,KAAK,CACZ,0BAA0B,IAAI,CAAC,EAAE,iBAAiB,IAAI,CAAC,uBAAuB,GAAG,EACjF,GAAG,CACH,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,QAAQ,CACrB,EAAS,EACT,IAAiB,EACjB,aAAmB,EACnB,MAAkB,EAClB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAc;YACvB,gBAAgB,EAAE,aAAa;YAC/B,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;SAC5D,CAAC;QAIF,IAAI,MAAM,KAAK,QAAQ,IAAI,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;YAGvB,IAAI,CAAC,gCAAgC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAGD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE;gBACZ,EAAE;gBACF,GAAG,EAAE,WAAW,CAAC,IAAI;aACrB;YACD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;SACJ,CAAC,CAAC;QAIH,IACC,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7C,IAAI,CAAC,kCAAkC,IAAI,IAAI,EAC9C,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,MAAM;gBAChB,WAAW,EAAE;oBACZ,EAAE;oBACF,GAAG,EAAE,WAAW,CAAC,IAAI;iBACrB;gBACD,OAAO,EAAE;oBACR,cAAc,EAAE,KAAK;iBACrB;gBACD,IAAI,EAAE;oBACL,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,+BAA+B,EAAE,IAAI,CAAC,+BAA+B;oBACrE,kCAAkC,EACjC,IAAI,CAAC,kCAAkC;oBACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACvB;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAGO,kBAAkB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACrC,CAAC;IAGO,IAAI;QACX,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,KAAK,CAAC,KAAK,IAAI,EAAE;YAChB,IAAI,CAAC;gBACJ,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChD,MAAM,KAAK,GAAG,YAAY;qBACxB,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;qBAClC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;oBACxB,OAAO;gBACR,CAAC;gBACD,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa;;;wCAGW,KAAK;;;;;+EAKkC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;;;;;;cAMhG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;6BAC/B,EACvB,YAAY,CACZ,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;wBAC9B,OAAO;oBACR,CAAC;oBAGD,MAAM,OAAO,CAAC,GAAG,CAChB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;wBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC;oBAC5C,CAAC,CAAC,CACF,CAAC;oBACF,QAAQ,GAAG,IAAI,CAAC;gBACjB,CAAC,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;YACjD,CAAC;oBAAS,CAAC;gBACV,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACf,MAAM,IAAA,qBAAU,EAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,IAAI,EAAE,CAAC;YACb,CAAC;QACF,CAAC,CAAC,EAAE,CAAC;IACN,CAAC;IAGM,KAAK;QAEX,IAAI,kBAAS,CAAC,EAAE,CAAC,MAAM,KAAK,UAAU,IAAI,kBAAS,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,+DAA+D,CAC/D,CAAC;QACH,CAAC;QACD,kBAAS,CAAC,EAAE,CAAC,EAAE,CACd,cAAc,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACb,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa,sDAAsD,EAC7E,CAAC,GAAG,CAAC,OAAO,CAAC,CACb,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAgB,EAAE,EAAE,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,EACD;YACC,OAAO,EAAP,gBAAO;SACP,CACD,CAAC;QACF,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;CACD;AAzPD,wBAyPC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@balena/pinejs",
-  "version": "17.1.0-build-model-based-typings-437bb06f44567532aec78e550f3d545732466411-1",
+  "version": "17.1.0-build-joshbwlng-tasks-61ce10e444abec6afea3fec43e9a5c37c7cedea6-1",
   "main": "out/server-glue/module",
   "type": "commonjs",
   "repository": "git@github.com:balena-io/pinejs.git",
@@ -19,48 +19,48 @@
     "webpack-server": "grunt server",
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src build typings Gruntfile.ts && npx tsc --project tsconfig.dev.json --noEmit",
-    "test": "npm run lint && npm run build && npm run webpack-build && npm run test:compose && npm run test:generated-types",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 npm run mocha",
-    "test:generated-types": "npm run generate-types && git diff --exit-code ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts",
+    "test": "npm run lint && npm run build && npm run webpack-build && npm run test:compose",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' INT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 npm run mocha",
     "mocha": "TS_NODE_FILES=true mocha",
-    "prettify": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.ts",
-    "generate-types": "node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/user.sbvr ./src/sbvr-api/user.ts && node ./bin/sbvr-compiler.js generate-types ./src/migrator/migrations.sbvr ./src/migrator/migrations.ts && node ./bin/sbvr-compiler.js generate-types ./src/sbvr-api/dev.sbvr ./src/sbvr-api/dev.ts && balena-lint -t tsconfig.dev.json --fix ./src/sbvr-api/user.ts ./src/migrator/migrations.ts ./src/sbvr-api/dev.ts"
+    "prettify": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.ts"
   },
   "dependencies": {
-    "@balena/abstract-sql-compiler": "^9.2.0",
-    "@balena/abstract-sql-to-typescript": "^3.2.1",
+    "@balena/abstract-sql-compiler": "^9.1.4",
+    "@balena/abstract-sql-to-typescript": "^3.1.1",
     "@balena/env-parsing": "^1.1.12",
     "@balena/lf-to-abstract-sql": "^5.0.2",
     "@balena/odata-parser": "^3.0.8",
     "@balena/odata-to-abstract-sql": "^6.2.7",
     "@balena/sbvr-parser": "^1.4.4",
-    "@balena/sbvr-types": "^7.1.3",
+    "@balena/sbvr-types": "^7.1.1",
     "@types/body-parser": "^1.19.5",
     "@types/compression": "^1.7.5",
     "@types/cookie-parser": "^1.4.7",
     "@types/deep-freeze": "^0.1.5",
     "@types/express": "^4.17.21",
     "@types/express-session": "^1.18.0",
-    "@types/lodash": "^4.17.5",
+    "@types/lodash": "^4.17.4",
     "@types/memoizee": "^0.4.11",
     "@types/method-override": "^0.0.35",
     "@types/multer": "^1.4.11",
     "@types/mysql": "^2.15.26",
-    "@types/node": "^20.14.5",
+    "@types/node": "^20.14.2",
     "@types/passport": "^1.0.16",
     "@types/passport-local": "^1.0.38",
     "@types/passport-strategy": "^0.2.38",
     "@types/pg": "^8.11.6",
     "@types/randomstring": "^1.3.0",
     "@types/websql": "^0.0.30",
+    "ajv": "^8.12.0",
     "busboy": "^1.6.0",
     "commander": "^11.1.0",
+    "cron-parser": "^4.9.0",
     "deep-freeze": "^0.0.1",
     "eventemitter3": "^5.0.1",
     "express-session": "^1.18.0",
     "lodash": "^4.17.21",
     "memoizee": "^0.4.17",
-    "pinejs-client-core": "^6.15.1",
+    "pinejs-client-core": "^6.14.6",
     "randomstring": "^1.3.0",
     "typed-error": "^3.2.2"
   },
@@ -91,11 +91,12 @@
     "grunt-ts": "^6.0.0-beta.22",
     "grunt-webpack": "^6.0.0",
     "husky": "^9.0.11",
-    "lint-staged": "^15.2.7",
+    "json-schema-to-ts": "^3.1.0",
+    "lint-staged": "^15.2.5",
     "load-grunt-tasks": "^5.1.0",
     "mocha": "^10.4.0",
     "on-finished": "^2.4.1",
-    "pinejs-client-supertest": "^2.0.4",
+    "pinejs-client-supertest": "^2.0.3",
     "raw-loader": "^4.0.2",
     "request": "^2.88.2",
     "require-npm4-to-publish": "^1.0.0",
@@ -104,13 +105,13 @@
     "ts-loader": "^9.5.1",
     "ts-node": "^10.9.2",
     "typescript": "^5.4.5",
-    "webpack": "^5.92.0",
+    "webpack": "^5.91.0",
     "webpack-dev-server": "^4.15.2"
   },
   "optionalDependencies": {
-    "@aws-sdk/client-s3": "^3.598.0",
-    "@aws-sdk/lib-storage": "^3.598.0",
-    "@aws-sdk/s3-request-presigner": "^3.598.0",
+    "@aws-sdk/client-s3": "^3.590.0",
+    "@aws-sdk/lib-storage": "^3.590.0",
+    "@aws-sdk/s3-request-presigner": "^3.590.0",
     "bcrypt": "^5.1.1",
     "body-parser": "^1.20.2",
     "compression": "^1.7.4",
@@ -146,6 +147,6 @@
     "recursive": true
   },
   "versionist": {
-    "publishedAt": "2024-06-18T16:52:31.030Z"
+    "publishedAt": "2024-06-19T04:57:00.974Z"
   }
 }

package/src/config-loader/env.ts

@@ -49,7 +49,7 @@ export const cache = {
 	apiKeyActorId: false as CacheOpts,
 };
 
-import { boolVar } from '@balena/env-parsing';
+import { boolVar, intVar } from '@balena/env-parsing';
 import memoize from 'memoizee';
 import memoizeWeak = require('memoizee/weak');
 export const createCache = <T extends (...args: any[]) => any>(
@@ -146,3 +146,8 @@ export const migrator = {
 	 */
 	asyncMigrationIsEnabled: boolVar('PINEJS_ASYNC_MIGRATION_ENABLED', true),
 };
+
+export const tasks = {
+	queueConcurrency: intVar('PINEJS_QUEUE_CONCURRENCY', 0),
+	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
+};

package/src/data-server/sbvr-server.js

@@ -133,15 +133,14 @@ export async function setup(app, sbvrUtils, db) {
 						},
 					},
 				})
-				.then(async (result) => {
+				.then(async (/** @type { Array<{ [key: string]: any }> } */ result) => {
 					if (result.length === 0) {
 						throw new Error('No SE data model found');
 					}
 					const instance = result[0];
 					await sbvrUtils.executeModel(tx, {
 						apiRoot: instance.is_of__vocabulary,
-						// prettier-ignore
-						modelText: /** @type { string } */ (instance.model_value.value),
+						modelText: instance.model_value.value,
 					});
 				});
 			await isServerOnAir(true);

package/src/database-layer/db.ts

@@ -98,6 +98,13 @@ export interface Database extends BaseDatabase {
 	) => Promise<Result>;
 	transaction: TransactionFn;
 	readTransaction: TransactionFn;
+	on?: (
+		name: 'notification',
+		fn: (...args: any[]) => Promise<void>,
+		options?: {
+			channel?: string;
+		},
+	) => void;
 }
 
 interface EngineParams {
@@ -689,6 +696,24 @@ if (maybePg != null) {
 		return {
 			engine: Engines.postgres,
 			executeSql: atomicExecuteSql,
+			on: async (name, fn, options) => {
+				if (name === 'notification' && options?.channel === undefined) {
+					throw new Error('Missing channel option for notification listener');
+				}
+
+				const client = await pool.connect();
+				client.on(name, async (msg) => {
+					try {
+						await fn(msg);
+					} catch (error) {
+						console.error('Error handling message:', error);
+					}
+				});
+
+				if (name === 'notification' && options?.channel !== undefined) {
+					await client.query(`LISTEN "${options.channel}";`);
+				}
+			},
 			transaction: createTransaction(async (stackTraceErr) => {
 				const client = await pool.connect();
 				const tx = new PostgresTx(client, false, stackTraceErr);

package/src/migrator/sync.ts

@@ -1,4 +1,3 @@
-import type MigrationsModel from './migrations';
 import {
 	type MigrationTuple,
 	MigrationError,
@@ -17,7 +16,7 @@ import _ from 'lodash';
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const migrationsModel = require('./migrations.sbvr');
+const modelText = require('./migrations.sbvr');
 
 type ApiRootModel = Model & { apiRoot: string };
 
@@ -137,49 +136,45 @@ const executeMigration = async (
 	}
 };
 
-declare module '../sbvr-api/sbvr-utils' {
-	export interface API {
-		[migrationModelConfig.apiRoot]: PinejsClient<MigrationsModel>;
-	}
-}
-const migrationModelConfig = {
-	modelName: 'migrations',
-	apiRoot: 'migrations',
-	modelText: migrationsModel,
-	migrations: {
-		'11.0.0-modified-at': `
-			ALTER TABLE "migration"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-		`,
-		'11.0.1-modified-at': `
-			ALTER TABLE "migration lock"
-			ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
-		`,
-		'15.0.0-data-types': async (tx, { db }) => {
-			switch (db.engine) {
-				case 'mysql':
-					await tx.executeSql(`\
-						ALTER TABLE "migration"
-						MODIFY "executed migrations" JSON NOT NULL;`);
-					await tx.executeSql(`\
-						ALTER TABLE "migration status"
-						MODIFY "is backing off" BOOLEAN NOT NULL;`);
-					break;
-				case 'postgres':
-					await tx.executeSql(`\
-						ALTER TABLE "migration"
-						ALTER COLUMN "executed migrations" SET DATA TYPE JSONB USING "executed migrations"::JSONB;`);
-					await tx.executeSql(`\
-						ALTER TABLE "migration status"
-						ALTER COLUMN "is backing off" DROP DEFAULT,
-						ALTER COLUMN "is backing off" SET DATA TYPE BOOLEAN USING "is backing off"::BOOLEAN,
-						ALTER COLUMN "is backing off" SET DEFAULT FALSE;`);
-					break;
-				// No need to migrate for websql
-			}
-		},
-	},
-} as const satisfies sbvrUtils.ExecutableModel;
 export const config: Config = {
-	models: [migrationModelConfig],
+	models: [
+		{
+			modelName: 'migrations',
+			apiRoot: 'migrations',
+			modelText,
+			migrations: {
+				'11.0.0-modified-at': `
+					ALTER TABLE "migration"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'11.0.1-modified-at': `
+					ALTER TABLE "migration lock"
+					ADD COLUMN IF NOT EXISTS "modified at" TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL;
+				`,
+				'15.0.0-data-types': async (tx, { db }) => {
+					switch (db.engine) {
+						case 'mysql':
+							await tx.executeSql(`\
+								ALTER TABLE "migration"
+								MODIFY "executed migrations" JSON NOT NULL;`);
+							await tx.executeSql(`\
+								ALTER TABLE "migration status"
+								MODIFY "is backing off" BOOLEAN NOT NULL;`);
+							break;
+						case 'postgres':
+							await tx.executeSql(`\
+								ALTER TABLE "migration"
+								ALTER COLUMN "executed migrations" SET DATA TYPE JSONB USING "executed migrations"::JSONB;`);
+							await tx.executeSql(`\
+								ALTER TABLE "migration status"
+								ALTER COLUMN "is backing off" DROP DEFAULT,
+								ALTER COLUMN "is backing off" SET DATA TYPE BOOLEAN USING "is backing off"::BOOLEAN,
+								ALTER COLUMN "is backing off" SET DEFAULT FALSE;`);
+							break;
+						// No need to migrate for websql
+					}
+				},
+			},
+		},
+	],
 };

package/src/sbvr-api/hooks.ts

@@ -1,5 +1,5 @@
 import type { OptionalField, Resolvable } from './common-types';
-import type { Tx } from '../database-layer/db';
+import type { Result, Tx } from '../database-layer/db';
 import type { ODataRequest, ParsedODataRequest } from './uri-parser';
 import type { AnyObject } from 'pinejs-client-core';
 import type { TypedError } from 'typed-error';
@@ -9,6 +9,7 @@ import _ from 'lodash';
 import { settleMapSeries } from './control-flow';
 import memoize from 'memoizee';
 import {
+	type PinejsClient,
 	type User,
 	type ApiKey,
 	resolveSynonym,
@@ -30,36 +31,34 @@ export interface HookReq {
 	hooks?: InstantiatedHooks;
 	is?: (type: string | string[]) => string | false | null;
 }
-export interface HookArgs<Vocab extends string = string> {
+export interface HookArgs {
 	req: HookReq;
 	request: ODataRequest;
-	api: (typeof api)[Vocab];
+	api: PinejsClient;
 	tx?: Tx | undefined;
 }
 export type HookResponse = PromiseLike<any> | null | void;
 
-export interface Hooks<Vocab extends string = string> {
-	PREPARSE?: (
-		options: Omit<HookArgs<Vocab>, 'request' | 'api'>,
-	) => HookResponse;
-	POSTPARSE?: (options: HookArgs<Vocab>) => HookResponse;
-	PRERUN?: (options: HookArgs<Vocab> & { tx: Tx }) => HookResponse;
+export interface Hooks {
+	PREPARSE?: (options: Omit<HookArgs, 'request' | 'api'>) => HookResponse;
+	POSTPARSE?: (options: HookArgs) => HookResponse;
+	PRERUN?: (options: HookArgs & { tx: Tx }) => HookResponse;
 	/** These are run in reverse translation order from newest to oldest */
 	POSTRUN?: (
-		options: HookArgs<Vocab> & { tx: Tx; result: any },
+		options: HookArgs & { tx: Tx; result: Result | number | undefined },
 	) => HookResponse;
 	/** These are run in reverse translation order from newest to oldest */
 	PRERESPOND?: (
-		options: HookArgs<Vocab> & {
+		options: HookArgs & {
 			tx: Tx;
-			result: any;
+			result?: Result | number | AnyObject;
 			/** This can be mutated to modify the response sent to the client */
 			response: Response;
 		},
 	) => HookResponse;
 	/** These are run in reverse translation order from newest to oldest */
 	'POSTRUN-ERROR'?: (
-		options: HookArgs<Vocab> & { tx: Tx; error: TypedError | any },
+		options: HookArgs & { tx: Tx; error: TypedError | any },
 	) => HookResponse;
 }
 export type HookBlueprints = {
@@ -265,9 +264,9 @@ const apiHooks = {
 // Share hooks between merge and patch since they are the same operation,
 // just MERGE was the OData intermediary until the HTTP spec added PATCH.
 apiHooks.MERGE = apiHooks.PATCH;
-export const addHook = <Vocab extends string>(
+export const addHook = (
 	method: keyof typeof apiHooks,
-	vocabulary: Vocab,
+	vocabulary: string,
 	resourceName: string,
 	hooks:
 		| { [key in keyof Hooks]: HookBlueprint<NonNullable<Hooks[key]>> }
@@ -346,11 +345,11 @@ export const addHook = <Vocab extends string>(
 	getHooks.clear();
 };
 
-export const addSideEffectHook = <Vocab extends string>(
+export const addSideEffectHook = (
 	method: HookMethod,
-	apiRoot: Vocab,
+	apiRoot: string,
 	resourceName: string,
-	hooks: Hooks<NoInfer<Vocab>>,
+	hooks: Hooks,
 ): void => {
 	addHook(method, apiRoot, resourceName, {
 		...hooks,
@@ -359,11 +358,11 @@ export const addSideEffectHook = <Vocab extends string>(
 	});
 };
 
-export const addPureHook = <Vocab extends string>(
+export const addPureHook = (
 	method: HookMethod,
-	apiRoot: Vocab,
+	apiRoot: string,
 	resourceName: string,
-	hooks: Hooks<NoInfer<Vocab>>,
+	hooks: Hooks,
 ): void => {
 	addHook(method, apiRoot, resourceName, {
 		...hooks,

package/src/sbvr-api/odata-response.ts

@@ -76,11 +76,21 @@ const checkForExpansion = async (
 	}
 };
 
-export const resourceURI = (
+export function resourceURI(
 	vocab: string,
 	resourceName: string,
 	id: string | number,
-): string | undefined => {
+): string;
+export function resourceURI(
+	vocab: string,
+	resourceName: string,
+	id: string | number | null | undefined,
+): string | undefined;
+export function resourceURI(
+	vocab: string,
+	resourceName: string,
+	id: string | number | null | undefined,
+): string | undefined {
 	if (id == null) {
 		return;
 	}
@@ -88,7 +98,7 @@ export const resourceURI = (
 		id = "'" + encodeURIComponent(id) + "'";
 	}
 	return `/${vocab}/${resourceName}(@id)?@id=${id}`;
-};
+}
 
 const getLocalFields = (table: AbstractSqlTable) => {
 	if (table.localFields == null) {