@balena/pinejs 17.2.0-build-add-large-file-uploads-interfaces-79c46b87ed3b7ae630ed6e3f9aad554c6647e8f7-1 → 17.2.0-build-joshbwlng-tasks-2f4d8f53b469a2eb62ffb71cb3f9e82258e9430c-1

package/src/server-glue/module.ts

@@ -6,6 +6,7 @@ import * as dbModule from '../database-layer/db';
 import * as configLoader from '../config-loader/config-loader';
 import * as migrator from '../migrator/sync';
 import type * as migratorUtils from '../migrator/utils';
+import * as tasks from '../tasks';
 
 import * as sbvrUtils from '../sbvr-api/sbvr-utils';
 import { PINEJS_ADVISORY_LOCK } from '../config-loader/env';
@@ -19,6 +20,7 @@ export * as errors from '../sbvr-api/errors';
 export * as env from '../config-loader/env';
 export * as types from '../sbvr-api/common-types';
 export * as hooks from '../sbvr-api/hooks';
+export * as tasks from '../tasks';
 export * as webResourceHandler from '../webresource-handler';
 export type { configLoader as ConfigLoader };
 export type { migratorUtils as Migrator };
@@ -63,6 +65,7 @@ export const init = async <T extends string>(
 		await sbvrUtils.setup(app, db);
 		const cfgLoader = await configLoader.setup(app);
 		await cfgLoader.loadConfig(migrator.config);
+		await cfgLoader.loadConfig(tasks.config);
 
 		const promises: Array<Promise<void>> = [];
 		if (process.env.SBVR_SERVER_ENABLED) {

package/src/tasks/common.ts

@@ -0,0 +1,9 @@
+import Ajv from 'ajv';
+
+// Root path for the tasks API
+export const apiRoot = 'tasks';
+
+// Channel name for task insert notifications
+export const channel = 'pinejs$task_insert';
+
+export const ajv = new Ajv();

package/src/tasks/index.ts

@@ -0,0 +1,176 @@
+import type { Schema } from 'ajv';
+import * as cronParser from 'cron-parser';
+import { tasks as tasksEnv } from '../config-loader/env';
+import { BadRequestError } from '../sbvr-api/errors';
+import { addPureHook } from '../sbvr-api/hooks';
+import * as sbvrUtils from '../sbvr-api/sbvr-utils';
+import type { ConfigLoader } from '../server-glue/module';
+import { ajv, apiRoot, channel } from './common';
+import type { TaskHandler } from './worker';
+import { Worker } from './worker';
+
+export * from './types';
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const modelText: string = require('./tasks.sbvr');
+
+// Create trigger for handling new tasks
+// Create index for polling tasks table
+const initSql = `
+CREATE OR REPLACE FUNCTION notify_task_insert()
+RETURNS TRIGGER AS $$
+BEGIN
+	PERFORM pg_notify('${channel}', NEW.id::text);
+	RETURN NEW;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE TRIGGER task_insert_trigger
+AFTER INSERT ON task
+FOR EACH ROW WHEN (NEW.status = 'queued' AND NEW."is scheduled to execute on-time" IS NULL)
+EXECUTE FUNCTION notify_task_insert();
+
+CREATE INDEX IF NOT EXISTS idx_task_poll ON task USING btree (
+	"is executed by-handler",
+	"is scheduled to execute on-time" ASC,
+	"id" ASC
+) WHERE status = 'queued';
+`;
+
+export const config: ConfigLoader.Config = {
+	models: [
+		{
+			modelName: apiRoot,
+			apiRoot,
+			modelText,
+			customServerCode: exports,
+			initSql,
+		},
+	],
+};
+
+let worker: Worker | null = null;
+export async function setup(): Promise<void> {
+	// Async task functionality is only supported on Postgres
+	if (sbvrUtils.db.engine !== 'postgres') {
+		console.warn('Skipping task setup as database not supported');
+		return;
+	}
+
+	const client = sbvrUtils.api[apiRoot];
+	worker = new Worker(client);
+
+	// Add resource hooks
+	addPureHook('POST', apiRoot, 'task', {
+		POSTPARSE: async ({ req, request }) => {
+			// Set the actor
+			request.values.is_created_by__actor =
+				req.user?.actor ?? req.apiKey?.actor;
+			if (request.values.is_created_by__actor == null) {
+				throw new BadRequestError(
+					'Creating tasks with missing actor on req is not allowed',
+				);
+			}
+
+			// Set defaults
+			request.values.status = 'queued';
+			request.values.attempt_count = 0;
+			request.values.attempt_limit ??= 1;
+
+			// Set scheduled start time using cron expression if provided
+			if (
+				request.values.is_scheduled_with__cron_expression != null &&
+				request.values.is_scheduled_to_execute_on__time == null
+			) {
+				try {
+					request.values.is_scheduled_to_execute_on__time = cronParser
+						.parseExpression(request.values.is_scheduled_with__cron_expression)
+						.next()
+						.toDate()
+						.toISOString();
+				} catch {
+					throw new BadRequestError(
+						`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`,
+					);
+				}
+			}
+
+			// Assert that the provided start time is far enough in the future
+			if (request.values.is_scheduled_to_execute_on__time != null) {
+				const now = new Date(Date.now() + tasksEnv.queueIntervalMS);
+				const startTime = new Date(
+					request.values.is_scheduled_to_execute_on__time,
+				);
+				if (startTime < now) {
+					throw new BadRequestError(
+						`Task scheduled start time must be greater than ${tasksEnv.queueIntervalMS} milliseconds in the future`,
+					);
+				}
+			}
+
+			// Assert that the requested handler exists
+			const handlerName = request.values.is_executed_by__handler;
+			if (handlerName == null) {
+				throw new BadRequestError(`Must specify a task handler to execute`);
+			}
+			const handler = worker?.handlers[handlerName];
+			if (handler == null) {
+				throw new BadRequestError(
+					`No task handler with name '${handlerName}' registered`,
+				);
+			}
+
+			// Assert that the provided parameter set is valid
+			if (handler.validate != null) {
+				if (!handler.validate(request.values.is_executed_with__parameter_set)) {
+					throw new BadRequestError(
+						`Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
+					);
+				}
+			}
+		},
+	});
+	addPureHook('all', apiRoot, 'task', {
+		// Convert bigints to strings in responses
+		PRERESPOND: async ({ response }) => {
+			if (typeof response.body === 'object') {
+				convertBigIntsToStrings(response.body);
+			}
+		},
+	});
+	worker.start();
+}
+
+// Recursively stringify bigints in an object
+function convertBigIntsToStrings(obj: any): void {
+	for (const [key, value] of Object.entries(obj)) {
+		if (value != null) {
+			const typeOfValue = typeof value;
+			if (typeOfValue === 'bigint') {
+				obj[key] = value.toString();
+			} else if (typeOfValue === 'object') {
+				convertBigIntsToStrings(value);
+			}
+		}
+	}
+}
+
+// Register a task handler
+export function addTaskHandler(
+	name: string,
+	fn: TaskHandler['fn'],
+	schema?: Schema,
+): void {
+	if (worker == null) {
+		throw new Error('Database does not support tasks');
+	}
+
+	if (worker.handlers[name] != null) {
+		throw new Error(`Task handler with name '${name}' already registered`);
+	}
+	worker.handlers[name] = {
+		name,
+		fn,
+		validate: schema != null ? ajv.compile(schema) : undefined,
+	};
+}

package/src/tasks/out.ts

@@ -0,0 +1,46 @@
+// These types were generated by @balena/abstract-sql-to-typescript v3.2.1
+
+import type { Types } from '@balena/abstract-sql-to-typescript';
+
+export interface Task {
+	Read: {
+		created_at: Types['Date Time']['Read'];
+		modified_at: Types['Date Time']['Read'];
+		id: Types['Big Serial']['Read'];
+		key: Types['Short Text']['Read'] | null;
+		is_created_by__actor: Types['Integer']['Read'];
+		is_executed_by__handler: Types['Short Text']['Read'];
+		is_executed_with__parameter_set: Types['JSON']['Read'] | null;
+		is_scheduled_with__cron_expression: Types['Short Text']['Read'] | null;
+		is_scheduled_to_execute_on__time: Types['Date Time']['Read'] | null;
+		status: 'queued' | 'cancelled' | 'succeeded' | 'failed';
+		started_on__time: Types['Date Time']['Read'] | null;
+		ended_on__time: Types['Date Time']['Read'] | null;
+		error_message: Types['Short Text']['Read'] | null;
+		attempt_count: Types['Integer']['Read'];
+		attempt_limit: Types['Integer']['Read'];
+	};
+	Write: {
+		created_at: Types['Date Time']['Write'];
+		modified_at: Types['Date Time']['Write'];
+		id: Types['Big Serial']['Write'];
+		key: Types['Short Text']['Write'] | null;
+		is_created_by__actor: Types['Integer']['Write'];
+		is_executed_by__handler: Types['Short Text']['Write'];
+		is_executed_with__parameter_set: Types['JSON']['Write'] | null;
+		is_scheduled_with__cron_expression: Types['Short Text']['Write'] | null;
+		is_scheduled_to_execute_on__time: Types['Date Time']['Write'] | null;
+		status: 'queued' | 'cancelled' | 'succeeded' | 'failed';
+		started_on__time: Types['Date Time']['Write'] | null;
+		ended_on__time: Types['Date Time']['Write'] | null;
+		error_message: Types['Short Text']['Write'] | null;
+		attempt_count: Types['Integer']['Write'];
+		attempt_limit: Types['Integer']['Write'];
+	};
+}
+
+export default interface $Model {
+	task: Task;
+
+
+}

package/src/tasks/tasks.sbvr

@@ -0,0 +1,55 @@
+Vocabulary: tasks
+
+Term: id
+    Concept Type: Big Serial (Type)
+Term: actor
+	Concept Type: Integer (Type)
+Term: attempt count
+	Concept Type: Integer (Type)
+Term: attempt limit
+	Concept Type: Integer (Type)
+Term: cron expression
+	Concept Type: Short Text (Type)
+Term: error message
+	Concept Type: Short Text (Type)
+Term: handler
+	Concept Type: Short Text (Type)
+Term: key
+	Concept Type: Short Text (Type)
+Term: parameter set
+	Concept Type: JSON (Type)
+Term: status
+	Concept Type: Short Text (Type)
+Term: time
+	Concept Type: Date Time (Type)
+
+Term: task
+Fact type: task has id
+	Necessity: each task has exactly one id
+Fact type: task has key
+	Necessity: each task has at most one key
+Fact type: task is created by actor
+	Necessity: each task is created by exactly one actor
+Fact type: task is executed by handler
+	Necessity: each task is executed by exactly one handler
+Fact type: task is executed with parameter set
+	Necessity: each task is executed with at most one parameter set
+Fact type: task is scheduled with cron expression
+	Necessity: each task is scheduled with at most one cron expression
+Fact type: task is scheduled to execute on time
+	Necessity: each task is scheduled to execute on at most one time
+Fact type: task has status
+	Necessity: each task has exactly one status
+	Definition: "queued" or "cancelled" or "succeeded" or "failed"
+Fact type: task started on time
+	Necessity: each task started on at most one time
+Fact type: task ended on time
+	Necessity: each task ended on at most one time
+Fact type: task has error message
+	Necessity: each task has at most one error message
+Fact type: task has attempt count
+	Necessity: each task has exactly one attempt count
+Fact type: task has attempt limit
+	Necessity: each task has exactly one attempt limit
+	Necessity: each task has an attempt limit that is greater than or equal to 1
+

package/src/tasks/types.ts

@@ -0,0 +1,44 @@
+// These types were generated by @balena/abstract-sql-to-typescript v3.2.1
+
+import type { Types } from '@balena/abstract-sql-to-typescript';
+
+export interface Task {
+	Read: {
+		created_at: Types['Date Time']['Read'];
+		modified_at: Types['Date Time']['Read'];
+		id: Types['Big Serial']['Read'];
+		key: Types['Short Text']['Read'] | null;
+		is_created_by__actor: Types['Integer']['Read'];
+		is_executed_by__handler: Types['Short Text']['Read'];
+		is_executed_with__parameter_set: Types['JSON']['Read'] | null;
+		is_scheduled_with__cron_expression: Types['Short Text']['Read'] | null;
+		is_scheduled_to_execute_on__time: Types['Date Time']['Read'] | null;
+		status: 'queued' | 'cancelled' | 'succeeded' | 'failed';
+		started_on__time: Types['Date Time']['Read'] | null;
+		ended_on__time: Types['Date Time']['Read'] | null;
+		error_message: Types['Short Text']['Read'] | null;
+		attempt_count: Types['Integer']['Read'];
+		attempt_limit: Types['Integer']['Read'];
+	};
+	Write: {
+		created_at: Types['Date Time']['Write'];
+		modified_at: Types['Date Time']['Write'];
+		id: Types['Big Serial']['Write'];
+		key: Types['Short Text']['Write'] | null;
+		is_created_by__actor: Types['Integer']['Write'];
+		is_executed_by__handler: Types['Short Text']['Write'];
+		is_executed_with__parameter_set: Types['JSON']['Write'] | null;
+		is_scheduled_with__cron_expression: Types['Short Text']['Write'] | null;
+		is_scheduled_to_execute_on__time: Types['Date Time']['Write'] | null;
+		status: 'queued' | 'cancelled' | 'succeeded' | 'failed';
+		started_on__time: Types['Date Time']['Write'] | null;
+		ended_on__time: Types['Date Time']['Write'] | null;
+		error_message: Types['Short Text']['Write'] | null;
+		attempt_count: Types['Integer']['Write'];
+		attempt_limit: Types['Integer']['Write'];
+	};
+}
+
+export default interface $Model {
+	task: Task;
+}

package/src/tasks/worker.ts

@@ -0,0 +1,282 @@
+import type { ValidateFunction } from 'ajv';
+import { setTimeout } from 'node:timers/promises';
+import type { AnyObject } from 'pinejs-client-core';
+import { tasks as tasksEnv } from '../config-loader/env';
+import type * as Db from '../database-layer/db';
+import * as permissions from '../sbvr-api/permissions';
+import { PinejsClient } from '../sbvr-api/sbvr-utils';
+import { sbvrUtils } from '../server-glue/module';
+import { ajv, channel } from './common';
+import type { Task } from './types';
+
+interface TaskArgs {
+	api: PinejsClient;
+	params: AnyObject;
+}
+
+type TaskResponse = Promise<{
+	status: Task['Read']['status'];
+	error?: string;
+}>;
+
+export interface TaskHandler {
+	name: string;
+	fn: (options: TaskArgs) => TaskResponse;
+	validate?: ValidateFunction;
+}
+
+type PartialTask = Pick<
+	Task['Read'],
+	| 'id'
+	| 'is_created_by__actor'
+	| 'is_executed_by__handler'
+	| 'is_executed_with__parameter_set'
+	| 'is_scheduled_with__cron_expression'
+	| 'attempt_count'
+	| 'attempt_limit'
+>;
+
+// Map of column names with SBVR names used in SELECT queries
+const selectColumns = Object.entries({
+	id: 'id',
+	'is executed by-handler': 'is_executed_by__handler',
+	'is executed with-parameter set': 'is_executed_with__parameter_set',
+	'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
+	'attempt count': 'attempt_count',
+	'attempt limit': 'attempt_limit',
+	'is created by-actor': 'is_created_by__actor',
+})
+	.map(([key, value]) => `t."${key}" AS "${value}"`)
+	.join(', ');
+
+// The worker is responsible for executing tasks in the queue. It listens for
+// notifications and polls the database for tasks to execute. It will execute
+// tasks in parallel up to a certain concurrency limit.
+export class Worker {
+	public handlers: Record<string, TaskHandler> = {};
+	private readonly concurrency: number;
+	private readonly interval: number;
+	private executing = 0;
+
+	constructor(private readonly client: PinejsClient) {
+		this.concurrency = tasksEnv.queueConcurrency;
+		this.interval = tasksEnv.queueIntervalMS;
+	}
+
+	// Check if instance can execute more tasks
+	private canExecute(): boolean {
+		return (
+			this.executing < this.concurrency && Object.keys(this.handlers).length > 0
+		);
+	}
+
+	private async execute(task: PartialTask, tx: Db.Tx): Promise<void> {
+		this.executing++;
+		try {
+			// Get specified handler
+			const handler = this.handlers[task.is_executed_by__handler];
+			const startedOnTime = new Date();
+			if (handler == null) {
+				await this.update(
+					tx,
+					task,
+					startedOnTime,
+					'failed',
+					'Matching task handler not found',
+				);
+				return;
+			}
+
+			// Validate parameters before execution so we can fail early if
+			// the parameter set is invalid. This can happen if the handler
+			// definition changes after a task is added to the queue.
+			if (
+				handler.validate != null &&
+				!handler.validate(task.is_executed_with__parameter_set)
+			) {
+				await this.update(
+					tx,
+					task,
+					startedOnTime,
+					'failed',
+					`Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
+				);
+				return;
+			}
+
+			// Execute handler and update task with results
+			let status: Task['Read']['status'] = 'queued';
+			let error: string | undefined;
+			try {
+				const results = await handler.fn({
+					api: new PinejsClient({}),
+					params: task.is_executed_with__parameter_set ?? {},
+				});
+				status = results.status;
+				error = results.error;
+			} finally {
+				await this.update(tx, task, startedOnTime, status, error);
+			}
+		} catch (err) {
+			// This shouldn't happen, but if it does we want to log and kill the process
+			console.error(
+				`Failed to execute task ${task.id} with handler ${task.is_executed_by__handler}:`,
+				err,
+			);
+			process.exit(1);
+		} finally {
+			this.executing--;
+		}
+	}
+
+	// Update task and schedule next attempt if needed
+	private async update(
+		tx: Db.Tx,
+		task: PartialTask,
+		startedOnTime: Date,
+		status: Task['Read']['status'],
+		errorMessage?: string,
+	): Promise<void> {
+		const attemptCount = task.attempt_count + 1;
+		const body: AnyObject = {
+			started_on__time: startedOnTime,
+			ended_on__time: new Date(),
+			status,
+			attempt_count: attemptCount,
+			...(errorMessage != null && { error_message: errorMessage }),
+		};
+
+		// Re-enqueue if the task failed but has retries left, remember that
+		// attemptCount includes the initial attempt while attempt_limit does not
+		if (status === 'failed' && attemptCount < task.attempt_limit) {
+			body.status = 'queued';
+
+			// Schedule next attempt using exponential backoff
+			body.is_scheduled_to_execute_on__time =
+				this.getNextAttemptTime(attemptCount);
+		}
+
+		// Patch current task
+		await this.client.patch({
+			resource: 'task',
+			passthrough: {
+				tx,
+				req: permissions.root,
+			},
+			id: task.id,
+			body,
+		});
+
+		// Create new task with same configuration if previous
+		// iteration completed and has a cron expression
+		if (
+			['failed', 'succeeded'].includes(body.status) &&
+			task.is_scheduled_with__cron_expression != null
+		) {
+			await this.client.post({
+				resource: 'task',
+				passthrough: {
+					tx,
+					req: permissions.root,
+				},
+				options: {
+					returnResource: false,
+				},
+				body: {
+					attempt_limit: task.attempt_limit,
+					is_created_by__actor: task.is_created_by__actor,
+					is_executed_by__handler: task.is_executed_by__handler,
+					is_executed_with__parameter_set: task.is_executed_with__parameter_set,
+					is_scheduled_with__cron_expression:
+						task.is_scheduled_with__cron_expression,
+				},
+			});
+		}
+	}
+
+	// Calculate next attempt time using exponential backoff
+	private getNextAttemptTime(attempt: number): Date | null {
+		const delay = Math.ceil(Math.exp(Math.min(10, attempt)));
+		return new Date(Date.now() + delay);
+	}
+
+	// Poll for tasks and execute them
+	// This is recursive and is spawned once per concurrency limit
+	private poll(): void {
+		let executed = false;
+		void (async () => {
+			try {
+				if (!this.canExecute()) {
+					return;
+				}
+				const handlerNames = Object.keys(this.handlers);
+				await sbvrUtils.db.transaction(async (tx) => {
+					const result = await sbvrUtils.db.executeSql(
+						`SELECT ${selectColumns}
+						FROM task AS t
+						WHERE
+							t."is executed by-handler" IN (${handlerNames.map((_, index) => `$${index + 1}`).join(', ')}) AND
+							t."status" = 'queued' AND
+							t."attempt count" <= t."attempt limit" AND
+							(
+								t."is scheduled to execute on-time" IS NULL OR
+								t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + $${handlerNames.length + 1} * INTERVAL '1 SECOND'
+							)
+						ORDER BY
+							t."is scheduled to execute on-time" ASC,
+							t."id" ASC
+						LIMIT 1 FOR UPDATE SKIP LOCKED`,
+						[...handlerNames, Math.ceil(this.interval / 1000)],
+					);
+
+					// Execute task if one was found
+					if (result.rows.length > 0) {
+						await this.execute(result.rows[0] as PartialTask, tx);
+						executed = true;
+					}
+				});
+			} catch (err) {
+				console.error('Failed polling for tasks:', err);
+			} finally {
+				if (!executed) {
+					await setTimeout(this.interval);
+				}
+				this.poll();
+			}
+		})();
+	}
+
+	// Start listening and polling for tasks
+	public start(): void {
+		// Tasks only support postgres for now
+		if (sbvrUtils.db.engine !== 'postgres' || sbvrUtils.db.on == null) {
+			throw new Error(
+				'Database does not support tasks, giving up on starting worker',
+			);
+		}
+		sbvrUtils.db.on(
+			'notification',
+			async (msg) => {
+				if (this.canExecute()) {
+					await sbvrUtils.db.transaction(async (tx) => {
+						const result = await sbvrUtils.db.executeSql(
+							`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`,
+							[msg.payload],
+						);
+						if (result.rows.length > 0) {
+							await this.execute(result.rows[0] as PartialTask, tx);
+						}
+					});
+				}
+			},
+			{
+				channel,
+			},
+		);
+
+		// Spawn children to poll for and execute tasks
+		for (let i = 0; i < this.concurrency; i++) {
+			this.poll();
+		}
+	}
+}