@balena/pinejs 16.1.1-build-renovate-major--balenalint-02ba2563aa4ac82ad90accc30534f744c00a0c16-1 → 16.2.0-build-joshbwlng-tasks-046f828587ae6d1889a3ae3298b3e44e96c74f90-1

package/out/tasks/index.js

@@ -0,0 +1,139 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.addTaskHandler = exports.setup = exports.config = void 0;
+const cronParser = __importStar(require("cron-parser"));
+const env_1 = require("../config-loader/env");
+const errors_1 = require("../sbvr-api/errors");
+const hooks_1 = require("../sbvr-api/hooks");
+const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
+const common_1 = require("./common");
+const worker_1 = require("./worker");
+__exportStar(require("./types"), exports);
+const modelText = require('./model.sbvr');
+exports.config = {
+    models: [
+        {
+            modelName: common_1.apiRoot,
+            apiRoot: common_1.apiRoot,
+            modelText,
+            customServerCode: exports,
+        },
+    ],
+};
+async function createTrigger(tx) {
+    await tx.executeSql(`
+        CREATE OR REPLACE FUNCTION notify_task_insert()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            PERFORM pg_notify('${common_1.channel}', NEW.id::text);
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+    `);
+    await tx.executeSql(`
+        CREATE OR REPLACE TRIGGER task_insert_trigger
+        AFTER INSERT ON task
+        FOR EACH ROW WHEN (NEW.status = 'pending' AND NEW."is scheduled to execute on-time" IS NULL)
+        EXECUTE FUNCTION notify_task_insert();
+    `);
+}
+let worker = null;
+async function setup(db, tx) {
+    if (db.engine !== 'postgres') {
+        return;
+    }
+    await createTrigger(tx);
+    const client = new sbvr_utils_1.PinejsClient({
+        apiPrefix: `/${common_1.apiRoot}/`,
+    });
+    worker = new worker_1.Worker(client);
+    (0, hooks_1.addPureHook)('POST', common_1.apiRoot, 'task', {
+        POSTPARSE: async ({ req, request }) => {
+            request.values.is_created_by__actor =
+                req.user?.actor ?? req.apiKey?.actor;
+            if (request.values.is_created_by__actor == null) {
+                throw new errors_1.BadRequestError('Creating tasks with missing actor on req is not allowed');
+            }
+            request.values.status = 'pending';
+            request.values.attempt_count = 0;
+            request.values.priority ??= 1;
+            request.values.attempt_limit ??= 1;
+            if (request.values.is_scheduled_with__cron_expression != null &&
+                request.values.is_scheduled_to_execute_on__time == null) {
+                try {
+                    request.values.is_scheduled_to_execute_on__time = cronParser
+                        .parseExpression(request.values.is_scheduled_with__cron_expression)
+                        .next()
+                        .toDate()
+                        .toISOString();
+                }
+                catch (_) {
+                    throw new errors_1.BadRequestError(`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`);
+                }
+            }
+            if (request.values.is_scheduled_to_execute_on__time != null) {
+                const now = new Date(new Date().getTime() + env_1.tasks.queueIntervalMS);
+                const startTime = new Date(request.values.is_scheduled_to_execute_on__time);
+                if (startTime < now) {
+                    throw new errors_1.BadRequestError(`Task scheduled start time must be greater than ${env_1.tasks.queueIntervalMS} milliseconds in the future`);
+                }
+            }
+            const handlerName = request.values.is_executed_by__handler;
+            if (handlerName == null) {
+                throw new errors_1.BadRequestError(`Must specify a task handler to execute`);
+            }
+            const handler = worker?.handlers[handlerName];
+            if (handler == null) {
+                throw new errors_1.BadRequestError(`No task handler with name '${handlerName}' registered`);
+            }
+            if (handler.validate != null) {
+                if (!handler.validate(request.values.is_executed_with__parameter_set)) {
+                    throw new errors_1.BadRequestError(`Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
+                }
+            }
+        },
+    });
+    worker.start();
+}
+exports.setup = setup;
+function addTaskHandler(name, fn, schema) {
+    if (worker == null) {
+        return;
+    }
+    if (worker.handlers[name] != null) {
+        throw new Error(`Task handler with name '${name}' already registered`);
+    }
+    worker.handlers[name] = {
+        name,
+        fn,
+        validate: schema != null ? common_1.ajv.compile(schema) : undefined,
+    };
+}
+exports.addTaskHandler = addTaskHandler;
+//# sourceMappingURL=index.js.map

package/out/tasks/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tasks/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,wDAA0C;AAC1C,8CAAyD;AAEzD,+CAAqD;AACrD,6CAAgD;AAChD,uDAAsD;AAEtD,qCAAiD;AAEjD,qCAAkC;AAElC,0CAAwB;AAGxB,MAAM,SAAS,GAAW,OAAO,CAAC,cAAc,CAAC,CAAC;AAErC,QAAA,MAAM,GAAG;IACrB,MAAM,EAAE;QACP;YACC,SAAS,EAAE,gBAAO;YAClB,OAAO,EAAP,gBAAO;YACP,SAAS;YACT,gBAAgB,EAAE,OAAO;SACzB;KAC8B;CAChC,CAAC;AAIF,KAAK,UAAU,aAAa,CAAC,EAAS;IACrC,MAAM,EAAE,CAAC,UAAU,CAAC;;;;iCAIY,gBAAO;;;;KAInC,CAAC,CAAC;IAGN,MAAM,EAAE,CAAC,UAAU,CAAC;;;;;KAKhB,CAAC,CAAC;AACP,CAAC;AAED,IAAI,MAAM,GAAkB,IAAI,CAAC;AAC1B,KAAK,UAAU,KAAK,CAAC,EAAe,EAAE,EAAS;IAErD,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC9B,OAAO;IACR,CAAC;IAGD,MAAM,aAAa,CAAC,EAAE,CAAC,CAAC;IAExB,MAAM,MAAM,GAAG,IAAI,yBAAY,CAAC;QAC/B,SAAS,EAAE,IAAI,gBAAO,GAAG;KACzB,CAAC,CAAC;IACH,MAAM,GAAG,IAAI,eAAM,CAAC,MAAM,CAAC,CAAC;IAG5B,IAAA,mBAAW,EAAC,MAAM,EAAE,gBAAO,EAAE,MAAM,EAAE;QACpC,SAAS,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE;YAErC,OAAO,CAAC,MAAM,CAAC,oBAAoB;gBAClC,GAAG,CAAC,IAAI,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC;YACtC,IAAI,OAAO,CAAC,MAAM,CAAC,oBAAoB,IAAI,IAAI,EAAE,CAAC;gBACjD,MAAM,IAAI,wBAAe,CACxB,yDAAyD,CACzD,CAAC;YACH,CAAC;YAGD,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC;YACjC,OAAO,CAAC,MAAM,CAAC,QAAQ,KAAK,CAAC,CAAC;YAC9B,OAAO,CAAC,MAAM,CAAC,aAAa,KAAK,CAAC,CAAC;YAGnC,IACC,OAAO,CAAC,MAAM,CAAC,kCAAkC,IAAI,IAAI;gBACzD,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EACtD,CAAC;gBACF,IAAI,CAAC;oBACJ,OAAO,CAAC,MAAM,CAAC,gCAAgC,GAAG,UAAU;yBAC1D,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,kCAAkC,CAAC;yBAClE,IAAI,EAAE;yBACN,MAAM,EAAE;yBACR,WAAW,EAAE,CAAC;gBACjB,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACZ,MAAM,IAAI,wBAAe,CACxB,4BAA4B,OAAO,CAAC,MAAM,CAAC,kCAAkC,EAAE,CAC/E,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,IAAI,OAAO,CAAC,MAAM,CAAC,gCAAgC,IAAI,IAAI,EAAE,CAAC;gBAC7D,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,WAAQ,CAAC,eAAe,CAAC,CAAC;gBACtE,MAAM,SAAS,GAAG,IAAI,IAAI,CACzB,OAAO,CAAC,MAAM,CAAC,gCAAgC,CAC/C,CAAC;gBACF,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;oBACrB,MAAM,IAAI,wBAAe,CACxB,kDAAkD,WAAQ,CAAC,eAAe,6BAA6B,CACvG,CAAC;gBACH,CAAC;YACF,CAAC;YAGD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,uBAAuB,CAAC;YAC3D,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACzB,MAAM,IAAI,wBAAe,CAAC,wCAAwC,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC9C,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,wBAAe,CACxB,8BAA8B,WAAW,cAAc,CACvD,CAAC;YACH,CAAC;YAGD,IAAI,OAAO,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;gBAC9B,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,CAAC,EAAE,CAAC;oBACvE,MAAM,IAAI,wBAAe,CACxB,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACH,CAAC;YACF,CAAC;QACF,CAAC;KACD,CAAC,CAAC;IACH,MAAM,CAAC,KAAK,EAAE,CAAC;AAChB,CAAC;AAtFD,sBAsFC;AAGD,SAAgB,cAAc,CAC7B,IAAY,EACZ,EAAqB,EACrB,MAAe;IAEf,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACpB,OAAO;IACR,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,sBAAsB,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG;QACvB,IAAI;QACJ,EAAE;QACF,QAAQ,EAAE,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,YAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;KAC1D,CAAC;AACH,CAAC;AAjBD,wCAiBC"}

package/out/tasks/model.sbvr

@@ -0,0 +1,60 @@
+Vocabulary: tasks
+
+Term: id
+    Concept Type: Big Serial (Type)
+Term: actor
+	Concept Type: Integer (Type)
+Term: attempt count
+	Concept Type: Integer (Type)
+Term: attempt limit
+	Concept Type: Integer (Type)
+Term: cron expression
+	Concept Type: Short Text (Type)
+Term: error message
+	Concept Type: Short Text (Type)
+Term: handler
+	Concept Type: Short Text (Type)
+Term: key
+	Concept Type: Short Text (Type)
+Term: parameter set
+	Concept Type: JSON (Type)
+Term: priority
+	Concept Type: Integer (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: time
+	Concept Type: Date Time (Type)
+
+Term: task
+Fact type: task has id
+	Necessity: each task has exactly one id
+Fact type: task has key
+	Necessity: each task has at most one key
+Fact type: task is created by actor
+	Necessity: each task is created by exactly one actor
+Fact type: task is executed by handler
+	Necessity: each task is executed by exactly one handler
+Fact type: task is executed with parameter set
+	Necessity: each task is executed with at most one parameter set
+Fact type: task has priority
+	Necessity: each task has exactly one priority
+	Necessity: each task has a priority that is greater than or equal to 0
+Fact type: task is scheduled with cron expression
+	Necessity: each task is scheduled with at most one cron expression
+Fact type: task is scheduled to execute on time
+	Necessity: each task is scheduled to execute on at most one time
+Fact type: task has status
+	Necessity: each task has exactly one status
+	Definition: "pending" or "cancelled" or "success" or "failed"
+Fact type: task started on time
+	Necessity: each task started on at most one time
+Fact type: task ended on time
+	Necessity: each task ended on at most one time
+Fact type: task has error message
+	Necessity: each task has at most one error message
+Fact type: task has attempt count
+	Necessity: each task has exactly one attempt count
+Fact type: task has attempt limit
+	Necessity: each task has exactly one attempt limit
+	Necessity: each task has an attempt limit that is greater than or equal to 1
+

package/out/tasks/types.d.ts

@@ -0,0 +1,38 @@
+import type { ValidateFunction } from 'ajv';
+import type { AnyObject } from 'pinejs-client-core';
+import type * as Db from '../database-layer/db';
+import type { PinejsClient } from '../sbvr-api/sbvr-utils';
+export declare const taskStatuses: readonly ["pending", "cancelled", "success", "failed"];
+export type TaskStatus = (typeof taskStatuses)[number];
+export interface Task {
+    id: number;
+    created_at: Date;
+    modified_at: Date;
+    is_created_by__actor: number;
+    is_executed_by__handler: string;
+    is_executed_with__parameter_set: object | null;
+    is_scheduled_with__cron_expression: string | null;
+    is_scheduled_to_execute_on__time: Date | null;
+    priority: number;
+    status: TaskStatus;
+    started_on__time: Date | null;
+    ended_on__time: Date | null;
+    error_message: string | null;
+    attempt_count: number;
+    attempt_limit: number;
+}
+export type PartialTask = Pick<Task, 'id' | 'is_created_by__actor' | 'is_executed_by__handler' | 'is_executed_with__parameter_set' | 'is_scheduled_with__cron_expression' | 'priority' | 'attempt_count' | 'attempt_limit'>;
+export interface TaskArgs {
+    api: PinejsClient;
+    params: AnyObject;
+    tx: Db.Tx;
+}
+export type TaskResponse = Promise<{
+    status: TaskStatus;
+    error?: string;
+}>;
+export interface TaskHandler {
+    name: string;
+    fn: (options: TaskArgs) => TaskResponse;
+    validate?: ValidateFunction;
+}

package/out/tasks/types.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.taskStatuses = void 0;
+exports.taskStatuses = [
+    'pending',
+    'cancelled',
+    'success',
+    'failed',
+];
+//# sourceMappingURL=types.js.map

package/out/tasks/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tasks/types.ts"],"names":[],"mappings":";;;AAKa,QAAA,YAAY,GAAG;IAC3B,SAAS;IACT,WAAW;IACX,SAAS;IACT,QAAQ;CACC,CAAC"}

package/out/tasks/worker.d.ts

@@ -0,0 +1,16 @@
+import { PinejsClient } from '../sbvr-api/sbvr-utils';
+import type { TaskHandler } from './types';
+export declare class Worker {
+    handlers: Record<string, TaskHandler>;
+    private readonly concurrency;
+    private readonly interval;
+    private client;
+    private executing;
+    constructor(client: PinejsClient);
+    private canExecute;
+    private execute;
+    private finalize;
+    private getNextAttemptTime;
+    private poll;
+    start(): void;
+}

package/out/tasks/worker.js

@@ -0,0 +1,191 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const env_1 = require("../config-loader/env");
+const permissions = __importStar(require("../sbvr-api/permissions"));
+const sbvr_utils_1 = require("../sbvr-api/sbvr-utils");
+const module_1 = require("../server-glue/module");
+const common_1 = require("./common");
+const selectColumns = Object.entries({
+    id: 'id',
+    'is executed by-handler': 'is_executed_by__handler',
+    'is executed with-parameter set': 'is_executed_with__parameter_set',
+    'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
+    'attempt count': 'attempt_count',
+    'attempt limit': 'attempt_limit',
+    priority: 'priority',
+    'is created by-actor': 'is_created_by__actor',
+})
+    .map(([key, value]) => `t."${key}" AS "${value}"`)
+    .join(', ');
+class Worker {
+    constructor(client) {
+        this.handlers = {};
+        this.executing = 0;
+        this.client = client;
+        this.concurrency = env_1.tasks.queueConcurrency;
+        this.interval = env_1.tasks.queueIntervalMS;
+    }
+    canExecute() {
+        return (this.executing < this.concurrency && Object.keys(this.handlers).length > 0);
+    }
+    async execute(tx, task) {
+        this.executing++;
+        try {
+            const handler = this.handlers[task.is_executed_by__handler];
+            const startedOnTime = new Date();
+            if (handler == null) {
+                await this.finalize(tx, task, startedOnTime, 'failed', 'Matching task handler not found');
+                return;
+            }
+            if (handler.validate != null &&
+                !handler.validate(task.is_executed_with__parameter_set)) {
+                await this.finalize(tx, task, startedOnTime, 'failed', `Invalid parameter set: ${common_1.ajv.errorsText(handler.validate.errors)}`);
+                return;
+            }
+            const result = await handler.fn({
+                api: new sbvr_utils_1.PinejsClient({
+                    passthrough: {
+                        tx,
+                    },
+                }),
+                params: task.is_executed_with__parameter_set ?? {},
+                tx,
+            });
+            await this.finalize(tx, task, startedOnTime, result.status, result.error);
+        }
+        catch (err) {
+            console.error('Task execution failed:', err);
+            process.exit(1);
+        }
+        finally {
+            this.executing--;
+        }
+    }
+    async finalize(tx, task, startedOnTime, status, errorMessage) {
+        const attemptCount = task.attempt_count + 1;
+        const body = {
+            started_on__time: startedOnTime,
+            ended_on__time: new Date(),
+            status,
+            attempt_count: attemptCount,
+            ...(errorMessage != null && { error_message: errorMessage }),
+        };
+        if (status === 'failed' && attemptCount < task.attempt_limit) {
+            body.status = 'pending';
+            body.is_scheduled_to_execute_on__time =
+                this.getNextAttemptTime(attemptCount);
+        }
+        await this.client.patch({
+            resource: 'task',
+            passthrough: {
+                tx,
+                req: permissions.root,
+            },
+            id: task.id,
+            body,
+        });
+        if (['failed', 'success'].includes(body.status) &&
+            task.is_scheduled_with__cron_expression != null) {
+            await this.client.post({
+                resource: 'task',
+                passthrough: {
+                    tx,
+                    req: permissions.root,
+                },
+                body: {
+                    attempt_limit: task.attempt_limit,
+                    is_created_by__actor: task.is_created_by__actor,
+                    is_executed_by__handler: task.is_executed_by__handler,
+                    is_executed_with__parameter_set: task.is_executed_with__parameter_set,
+                    is_scheduled_with__cron_expression: task.is_scheduled_with__cron_expression,
+                    priority: task.priority,
+                },
+            });
+        }
+    }
+    getNextAttemptTime(attempt) {
+        const delay = Math.ceil(Math.exp(Math.min(10, attempt))) * 1000;
+        return new Date(Date.now() + delay);
+    }
+    poll() {
+        let executed = false;
+        const handlerNames = Object.keys(this.handlers);
+        const binds = handlerNames.map((_, index) => `$${index + 1}`).join(', ');
+        module_1.sbvrUtils.db
+            .transaction(async (tx) => {
+            if (!this.canExecute()) {
+                return;
+            }
+            const result = await tx.executeSql(`SELECT ${selectColumns}
+				FROM task AS t
+				WHERE
+					t."is executed by-handler" IN (${binds}) AND
+					t."status" = 'pending' AND
+					t."attempt count" <= t."attempt limit" AND
+					(
+						t."is scheduled to execute on-time" IS NULL OR
+						t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
+					)
+				ORDER BY
+					t."is scheduled to execute on-time" ASC,
+					t."priority" DESC,
+					t."id" ASC
+				LIMIT ${Math.max(this.concurrency - this.executing, 0)}
+				FOR UPDATE SKIP LOCKED`, handlerNames);
+            if (result.rows.length === 0) {
+                return;
+            }
+            await Promise.all(result.rows.map(async (row) => {
+                await this.execute(tx, row);
+            }));
+            executed = true;
+        })
+            .catch((err) => {
+            console.error('Failed polling for tasks:', err);
+        })
+            .finally(() => {
+            setTimeout(() => this.poll(), executed ? 0 : this.interval);
+        });
+    }
+    start() {
+        module_1.sbvrUtils.db.on?.('notification', async (msg) => {
+            if (this.canExecute()) {
+                await module_1.sbvrUtils.db.transaction(async (tx) => {
+                    const result = await tx.executeSql(`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`, [msg.payload]);
+                    if (result.rows.length > 0) {
+                        await this.execute(tx, result.rows[0]);
+                    }
+                });
+            }
+        }, {
+            channel: common_1.channel,
+        });
+        this.poll();
+    }
+}
+exports.Worker = Worker;
+//# sourceMappingURL=worker.js.map

package/out/tasks/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/tasks/worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,8CAAyD;AAEzD,qEAAuD;AACvD,uDAAsD;AACtD,kDAAkD;AAClD,qCAAwC;AAIxC,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC;IACpC,EAAE,EAAE,IAAI;IACR,wBAAwB,EAAE,yBAAyB;IACnD,gCAAgC,EAAE,iCAAiC;IACnE,mCAAmC,EAAE,oCAAoC;IACzE,eAAe,EAAE,eAAe;IAChC,eAAe,EAAE,eAAe;IAChC,QAAQ,EAAE,UAAU;IACpB,qBAAqB,EAAE,sBAAsB;CAC7C,CAAC;KACA,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,GAAG,SAAS,KAAK,GAAG,CAAC;KACjD,IAAI,CAAC,IAAI,CAAC,CAAC;AAKb,MAAa,MAAM;IAOlB,YAAY,MAAoB;QANzB,aAAQ,GAAgC,EAAE,CAAC;QAI1C,cAAS,GAAG,CAAC,CAAC;QAGrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAQ,CAAC,gBAAgB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,WAAQ,CAAC,eAAe,CAAC;IAC1C,CAAC;IAGO,UAAU;QACjB,OAAO,CACN,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAC1E,CAAC;IACH,CAAC;IAGO,KAAK,CAAC,OAAO,CAAC,EAAS,EAAE,IAAiB;QACjD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC;YAEJ,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAC5D,MAAM,aAAa,GAAG,IAAI,IAAI,EAAE,CAAC;YACjC,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,iCAAiC,CACjC,CAAC;gBACF,OAAO;YACR,CAAC;YAKD,IACC,OAAO,CAAC,QAAQ,IAAI,IAAI;gBACxB,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,+BAA+B,CAAC,EACtD,CAAC;gBACF,MAAM,IAAI,CAAC,QAAQ,CAClB,EAAE,EACF,IAAI,EACJ,aAAa,EACb,QAAQ,EACR,0BAA0B,YAAG,CAAC,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CACnE,CAAC;gBACF,OAAO;YACR,CAAC;YAGD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,EAAE,CAAC;gBAC/B,GAAG,EAAE,IAAI,yBAAY,CAAC;oBACrB,WAAW,EAAE;wBACZ,EAAE;qBACF;iBACD,CAAC;gBACF,MAAM,EAAE,IAAI,CAAC,+BAA+B,IAAI,EAAE;gBAClD,EAAE;aACF,CAAC,CAAC;YAGH,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YAEd,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,GAAG,CAAC,CAAC;YAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACV,IAAI,CAAC,SAAS,EAAE,CAAC;QAClB,CAAC;IACF,CAAC;IAGO,KAAK,CAAC,QAAQ,CACrB,EAAS,EACT,IAAiB,EACjB,aAAmB,EACnB,MAAkB,EAClB,YAAqB;QAErB,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAc;YACvB,gBAAgB,EAAE,aAAa;YAC/B,cAAc,EAAE,IAAI,IAAI,EAAE;YAC1B,MAAM;YACN,aAAa,EAAE,YAAY;YAC3B,GAAG,CAAC,YAAY,IAAI,IAAI,IAAI,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC;SAC5D,CAAC;QAIF,IAAI,MAAM,KAAK,QAAQ,IAAI,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAC9D,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;YAGxB,IAAI,CAAC,gCAAgC;gBACpC,IAAI,CAAC,kBAAkB,CAAC,YAAY,CAAC,CAAC;QACxC,CAAC;QAGD,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;YACvB,QAAQ,EAAE,MAAM;YAChB,WAAW,EAAE;gBACZ,EAAE;gBACF,GAAG,EAAE,WAAW,CAAC,IAAI;aACrB;YACD,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;SACJ,CAAC,CAAC;QAIH,IACC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;YAC3C,IAAI,CAAC,kCAAkC,IAAI,IAAI,EAC9C,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACtB,QAAQ,EAAE,MAAM;gBAChB,WAAW,EAAE;oBACZ,EAAE;oBACF,GAAG,EAAE,WAAW,CAAC,IAAI;iBACrB;gBACD,IAAI,EAAE;oBACL,aAAa,EAAE,IAAI,CAAC,aAAa;oBACjC,oBAAoB,EAAE,IAAI,CAAC,oBAAoB;oBAC/C,uBAAuB,EAAE,IAAI,CAAC,uBAAuB;oBACrD,+BAA+B,EAAE,IAAI,CAAC,+BAA+B;oBACrE,kCAAkC,EACjC,IAAI,CAAC,kCAAkC;oBACxC,QAAQ,EAAE,IAAI,CAAC,QAAQ;iBACvB;aACD,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAGO,kBAAkB,CAAC,OAAe;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAChE,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;IACrC,CAAC;IAGO,IAAI;QACX,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzE,kBAAS,CAAC,EAAE;aACV,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACxB,OAAO;YACR,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa;;;sCAGU,KAAK;;;;;6EAKkC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;;;;;;YAMhG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;2BAC/B,EACtB,YAAY,CACZ,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO;YACR,CAAC;YAGD,MAAM,OAAO,CAAC,GAAG,CAChB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,GAAkB,CAAC,CAAC;YAC5C,CAAC,CAAC,CACF,CAAC;YACF,QAAQ,GAAG,IAAI,CAAC;QACjB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACd,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,GAAG,CAAC,CAAC;QACjD,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACb,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;IACL,CAAC;IAGM,KAAK;QACX,kBAAS,CAAC,EAAE,CAAC,EAAE,EAAE,CAChB,cAAc,EACd,KAAK,EAAE,GAAG,EAAE,EAAE;YACb,IAAI,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC;gBACvB,MAAM,kBAAS,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;oBAC3C,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,UAAU,CACjC,UAAU,aAAa,sDAAsD,EAC7E,CAAC,GAAG,CAAC,OAAO,CAAC,CACb,CAAC;oBACF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAgB,CAAC,CAAC;oBACvD,CAAC;gBACF,CAAC,CAAC,CAAC;YACJ,CAAC;QACF,CAAC,EACD;YACC,OAAO,EAAP,gBAAO;SACP,CACD,CAAC;QACF,IAAI,CAAC,IAAI,EAAE,CAAC;IACb,CAAC;CACD;AA3ND,wBA2NC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@balena/pinejs",
-  "version": "16.1.1-build-renovate-major--balenalint-02ba2563aa4ac82ad90accc30534f744c00a0c16-1",
+  "version": "16.2.0-build-joshbwlng-tasks-046f828587ae6d1889a3ae3298b3e44e96c74f90-1",
   "main": "out/server-glue/module",
   "type": "commonjs",
   "repository": "git@github.com:balena-io/pinejs.git",
@@ -20,7 +20,7 @@
     "webpack-build": "npm run webpack-browser && npm run webpack-module && npm run webpack-server",
     "lint": "balena-lint -t tsconfig.dev.json -e js -e ts src build typings Gruntfile.ts && npx tsc --project tsconfig.dev.json --noEmit",
     "test": "npm run lint && npm run build && npm run webpack-build && npm run test:compose",
-    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' SIGINT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 npm run mocha",
+    "test:compose": "trap 'docker compose -f docker-compose.npm-test.yml down ; echo Stopped ; exit 0' SIGINT; docker compose -f docker-compose.npm-test.yml up -d && sleep 2 && DATABASE_URL=postgres://docker:docker@localhost:5431/postgres PINEJS_WEBRESOURCE_MAXFILESIZE=1000000000 S3_ENDPOINT=http://localhost:43680 S3_ACCESS_KEY=USERNAME S3_SECRET_KEY=PASSWORD S3_STORAGE_ADAPTER_BUCKET=balena-pine-web-resources S3_REGION=us-east-1 PINEJS_QUEUE_CONCURRENCY=1 npm run mocha",
     "mocha": "TS_NODE_FILES=true mocha",
     "prettify": "balena-lint -t tsconfig.dev.json -e js -e ts --fix src test build typings Gruntfile.ts"
   },
@@ -51,8 +51,10 @@
     "@types/pg": "^8.10.9",
     "@types/randomstring": "^1.1.11",
     "@types/websql": "^0.0.30",
+    "ajv": "^8.12.0",
     "busboy": "^1.6.0",
     "commander": "^11.1.0",
+    "cron-parser": "^4.9.0",
     "deep-freeze": "^0.0.1",
     "eventemitter3": "^5.0.1",
     "express-session": "^1.17.3",
@@ -63,7 +65,7 @@
     "typed-error": "^3.2.2"
   },
   "devDependencies": {
-    "@balena/lint": "^8.0.0",
+    "@balena/lint": "^7.2.4",
     "@faker-js/faker": "^8.3.1",
     "@types/busboy": "^1.5.3",
     "@types/chai": "^4.3.11",
@@ -89,6 +91,7 @@
     "grunt-ts": "^6.0.0-beta.22",
     "grunt-webpack": "^6.0.0",
     "husky": "^8.0.3",
+    "json-schema-to-ts": "^3.0.1",
     "lint-staged": "^15.2.0",
     "load-grunt-tasks": "^5.1.0",
     "mocha": "^10.2.0",
@@ -144,6 +147,6 @@
     "recursive": true
   },
   "versionist": {
-    "publishedAt": "2024-04-10T23:58:21.576Z"
+    "publishedAt": "2024-04-11T00:21:32.212Z"
   }
 }

package/src/config-loader/env.ts

@@ -49,7 +49,7 @@ export const cache = {
 	apiKeyActorId: false as CacheOpts,
 };
 
-import { boolVar } from '@balena/env-parsing';
+import { boolVar, intVar } from '@balena/env-parsing';
 import memoize from 'memoizee';
 import memoizeWeak = require('memoizee/weak');
 export const createCache = <T extends (...args: any[]) => any>(
@@ -146,3 +146,8 @@ export const migrator = {
 	 */
 	asyncMigrationIsEnabled: boolVar('PINEJS_ASYNC_MIGRATION_ENABLED', true),
 };
+
+export const tasks = {
+	queueConcurrency: intVar('PINEJS_QUEUE_CONCURRENCY', 0),
+	queueIntervalMS: intVar('PINEJS_QUEUE_INTERVAL_MS', 1000),
+};

package/src/database-layer/db.ts

@@ -98,6 +98,13 @@ export interface Database extends BaseDatabase {
 	) => Promise<Result>;
 	transaction: TransactionFn;
 	readTransaction: TransactionFn;
+	on?: (
+		name: 'notification',
+		fn: (...args: any[]) => Promise<void>,
+		options?: {
+			channel?: string;
+		},
+	) => void;
 }
 
 interface EngineParams {
@@ -689,6 +696,23 @@ if (maybePg != null) {
 		return {
 			engine: Engines.postgres,
 			executeSql: atomicExecuteSql,
+			on: async (name, fn, options) => {
+				if (name === 'notification' && options?.channel === undefined) {
+					throw new Error('Missing channel option for notification listener');
+				}
+
+				const client = await pool.connect();
+				client.on(name, (msg) => {
+					fn(msg).catch((error) => {
+						console.error('Error handling message:', error);
+					});
+				});
+
+				if (name === 'notification' && options?.channel !== undefined) {
+					await client.query(`LISTEN "${options.channel}";`);
+					// client.release();
+				}
+			},
 			transaction: createTransaction(async (stackTraceErr) => {
 				const client = await pool.connect();
 				const tx = new PostgresTx(client, false, stackTraceErr);

package/src/sbvr-api/sbvr-utils.ts

@@ -42,6 +42,7 @@ import { generateODataMetadata } from '../odata-metadata/odata-metadata-generato
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const devModel = require('./dev.sbvr');
+import * as tasks from '../tasks';
 import * as permissions from './permissions';
 import {
 	BadRequestError,
@@ -77,6 +78,7 @@ export {
 	addPureHook,
 	addSideEffectHook,
 } from './hooks';
+export { addTaskHandler } from '../tasks';
 
 import memoizeWeak = require('memoizee/weak');
 import * as controlFlow from './control-flow';
@@ -773,7 +775,7 @@ export const postExecuteModels = async (tx: Db.Tx): Promise<void> => {
 	// Hence, skipped migrations from earlier models are not set as executed as the `migration` table is missing
 	// Here the skipped migrations that haven't been set properly are covered
 	// This is mostly an edge case when running on an empty database schema and migrations model hasn't been executed, yet.
-	// One specifc case are tests to run tests against migrated and unmigrated database states
+	// One specific case are tests to run tests against migrated and unmigrated database states
 
 	for (const modelKey of Object.keys(models)) {
 		const pendingToSetExecutedMigrations =
@@ -1983,6 +1985,7 @@ export const executeStandardModels = async (tx: Db.Tx): Promise<void> => {
 			},
 		});
 		await executeModels(tx, permissions.config.models);
+		await executeModels(tx, tasks.config.models);
 		console.info('Successfully executed standard models.');
 	} catch (err: any) {
 		console.error('Failed to execute standard models.', err);
@@ -1999,6 +2002,7 @@ export const setup = async (
 		await db.transaction(async (tx) => {
 			await executeStandardModels(tx);
 			await permissions.setup();
+			await tasks.setup($db, tx);
 		});
 	} catch (err: any) {
 		console.error('Could not execute standard models', err);

package/src/server-glue/module.ts

@@ -19,6 +19,7 @@ export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
+export * as tasks from '../tasks';
 export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };

package/src/tasks/common.ts

@@ -0,0 +1,14 @@
+import Ajv from 'ajv';
+
+// Root path for the tasks API
+export const apiRoot = 'tasks';
+
+// Channel name for task insert notifications
+export const channel = 'task_insert';
+
+// Setting inlineRefs=false as without it we run into a
+// "Maximum call stack size exceeded" error apprarently caused
+// by String.prototype._uncountable_words being set in sbvr-parser?
+export const ajv = new Ajv({
+	inlineRefs: false,
+});