@balena/pinejs 17.1.0-build-joshbwlng-tasks-e8a81b1d884e0039e67661a25a5cb81d46966640-1 → 17.1.0-build-model-based-typings-86a981c89f828217571c93fa010174529a5345f9-1

package/src/tasks/index.ts

@@ -1,158 +0,0 @@
-import type { Schema } from 'ajv';
-import * as cronParser from 'cron-parser';
-import { tasks as tasksEnv } from '../config-loader/env';
-import type * as Db from '../database-layer/db';
-import { BadRequestError } from '../sbvr-api/errors';
-import { addPureHook } from '../sbvr-api/hooks';
-import * as sbvrUtils from '../sbvr-api/sbvr-utils';
-import type { ConfigLoader } from '../server-glue/module';
-import { ajv, apiRoot, channel } from './common';
-import type { TaskHandler } from './types';
-import { Worker } from './worker';
-
-export * from './types';
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const modelText: string = require('./tasks.sbvr');
-
-// Create trigger for handling new tasks
-// Create index for polling tasks table
-const initSql = `
-CREATE OR REPLACE FUNCTION notify_task_insert()
-RETURNS TRIGGER AS $$
-BEGIN
-	PERFORM pg_notify('${channel}', NEW.id::text);
-	RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE TRIGGER task_insert_trigger
-AFTER INSERT ON task
-FOR EACH ROW WHEN (NEW.status = 'queued' AND NEW."is scheduled to execute on-time" IS NULL)
-EXECUTE FUNCTION notify_task_insert();
-
-CREATE INDEX IF NOT EXISTS idx_task_poll ON task USING btree (
-	"is executed by-handler",
-	"is scheduled to execute on-time" ASC,
-	"priority" DESC,
-	"id" ASC
-) WHERE status = 'queued';
-`;
-
-export const config: ConfigLoader.Config = {
-	models: [
-		{
-			modelName: apiRoot,
-			apiRoot,
-			modelText,
-			customServerCode: exports,
-			initSql,
-		},
-	],
-};
-
-let worker: Worker | null = null;
-export async function setup(db: Db.Database): Promise<void> {
-	// Async task functionality is only supported on Postgres
-	if (db.engine !== 'postgres') {
-		console.warn('Skipping task setup as database not supported');
-		return;
-	}
-
-	const client = sbvrUtils.api[apiRoot];
-	worker = new Worker(client);
-
-	// Add resource hooks
-	addPureHook('POST', apiRoot, 'task', {
-		POSTPARSE: async ({ req, request }) => {
-			// Set the actor
-			request.values.is_created_by__actor =
-				req.user?.actor ?? req.apiKey?.actor;
-			if (request.values.is_created_by__actor == null) {
-				throw new BadRequestError(
-					'Creating tasks with missing actor on req is not allowed',
-				);
-			}
-
-			// Set defaults
-			request.values.status = 'queued';
-			request.values.attempt_count = 0;
-			request.values.attempt_limit ??= 1;
-			// TODO: Implement a balancer to better enqueue tasks based on actor usage
-			request.values.priority ??= 1;
-
-			// Set scheduled start time using cron expression if provided
-			if (
-				request.values.is_scheduled_with__cron_expression != null &&
-				request.values.is_scheduled_to_execute_on__time == null
-			) {
-				try {
-					request.values.is_scheduled_to_execute_on__time = cronParser
-						.parseExpression(request.values.is_scheduled_with__cron_expression)
-						.next()
-						.toDate()
-						.toISOString();
-				} catch {
-					throw new BadRequestError(
-						`Invalid cron expression: ${request.values.is_scheduled_with__cron_expression}`,
-					);
-				}
-			}
-
-			// Assert that the provided start time is far enough in the future
-			if (request.values.is_scheduled_to_execute_on__time != null) {
-				const now = new Date(Date.now() + tasksEnv.queueIntervalMS);
-				const startTime = new Date(
-					request.values.is_scheduled_to_execute_on__time,
-				);
-				if (startTime < now) {
-					throw new BadRequestError(
-						`Task scheduled start time must be greater than ${tasksEnv.queueIntervalMS} milliseconds in the future`,
-					);
-				}
-			}
-
-			// Assert that the requested handler exists
-			const handlerName = request.values.is_executed_by__handler;
-			if (handlerName == null) {
-				throw new BadRequestError(`Must specify a task handler to execute`);
-			}
-			const handler = worker?.handlers[handlerName];
-			if (handler == null) {
-				throw new BadRequestError(
-					`No task handler with name '${handlerName}' registered`,
-				);
-			}
-
-			// Assert that the provided parameter set is valid
-			if (handler.validate != null) {
-				if (!handler.validate(request.values.is_executed_with__parameter_set)) {
-					throw new BadRequestError(
-						`Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
-					);
-				}
-			}
-		},
-	});
-	worker.start();
-}
-
-// Register a task handler
-export function addTaskHandler(
-	name: string,
-	fn: TaskHandler['fn'],
-	schema?: Schema,
-): void {
-	if (worker == null) {
-		throw new Error('Database does not support tasks');
-	}
-
-	if (worker.handlers[name] != null) {
-		throw new Error(`Task handler with name '${name}' already registered`);
-	}
-	worker.handlers[name] = {
-		name,
-		fn,
-		validate: schema != null ? ajv.compile(schema) : undefined,
-	};
-}

package/src/tasks/tasks.sbvr

@@ -1,60 +0,0 @@
-Vocabulary: tasks
-
-Term: id
-    Concept Type: Big Serial (Type)
-Term: actor
-	Concept Type: Integer (Type)
-Term: attempt count
-	Concept Type: Integer (Type)
-Term: attempt limit
-	Concept Type: Integer (Type)
-Term: cron expression
-	Concept Type: Short Text (Type)
-Term: error message
-	Concept Type: Short Text (Type)
-Term: handler
-	Concept Type: Short Text (Type)
-Term: key
-	Concept Type: Short Text (Type)
-Term: parameter set
-	Concept Type: JSON (Type)
-Term: priority
-	Concept Type: Integer (Type)
-Term: status
-	Concept Type: Short Text (Type)
-Term: time
-	Concept Type: Date Time (Type)
-
-Term: task
-Fact type: task has id
-	Necessity: each task has exactly one id
-Fact type: task has key
-	Necessity: each task has at most one key
-Fact type: task is created by actor
-	Necessity: each task is created by exactly one actor
-Fact type: task is executed by handler
-	Necessity: each task is executed by exactly one handler
-Fact type: task is executed with parameter set
-	Necessity: each task is executed with at most one parameter set
-Fact type: task has priority
-	Necessity: each task has exactly one priority
-	Necessity: each task has a priority that is greater than or equal to 0
-Fact type: task is scheduled with cron expression
-	Necessity: each task is scheduled with at most one cron expression
-Fact type: task is scheduled to execute on time
-	Necessity: each task is scheduled to execute on at most one time
-Fact type: task has status
-	Necessity: each task has exactly one status
-	Definition: "queued" or "cancelled" or "succeeded" or "failed"
-Fact type: task started on time
-	Necessity: each task started on at most one time
-Fact type: task ended on time
-	Necessity: each task ended on at most one time
-Fact type: task has error message
-	Necessity: each task has at most one error message
-Fact type: task has attempt count
-	Necessity: each task has exactly one attempt count
-Fact type: task has attempt limit
-	Necessity: each task has exactly one attempt limit
-	Necessity: each task has an attempt limit that is greater than or equal to 1
-

package/src/tasks/types.ts

@@ -1,58 +0,0 @@
-import type { ValidateFunction } from 'ajv';
-import type { AnyObject } from 'pinejs-client-core';
-import type * as Db from '../database-layer/db';
-import type { PinejsClient } from '../sbvr-api/sbvr-utils';
-
-export const taskStatuses = [
-	'queued',
-	'cancelled',
-	'succeeded',
-	'failed',
-] as const;
-export type TaskStatus = (typeof taskStatuses)[number];
-export interface Task {
-	id: number;
-	created_at: Date;
-	modified_at: Date;
-	is_created_by__actor: number;
-	is_executed_by__handler: string;
-	is_executed_with__parameter_set: object | null;
-	is_scheduled_with__cron_expression: string | null;
-	is_scheduled_to_execute_on__time: Date | null;
-	priority: number;
-	status: TaskStatus;
-	started_on__time: Date | null;
-	ended_on__time: Date | null;
-	error_message: string | null;
-	attempt_count: number;
-	attempt_limit: number;
-}
-
-export type PartialTask = Pick<
-	Task,
-	| 'id'
-	| 'is_created_by__actor'
-	| 'is_executed_by__handler'
-	| 'is_executed_with__parameter_set'
-	| 'is_scheduled_with__cron_expression'
-	| 'priority'
-	| 'attempt_count'
-	| 'attempt_limit'
->;
-
-export interface TaskArgs {
-	api: PinejsClient;
-	params: AnyObject;
-	tx: Db.Tx;
-}
-
-export type TaskResponse = Promise<{
-	status: TaskStatus;
-	error?: string;
-}>;
-
-export interface TaskHandler {
-	name: string;
-	fn: (options: TaskArgs) => TaskResponse;
-	validate?: ValidateFunction;
-}

package/src/tasks/worker.ts

@@ -1,281 +0,0 @@
-import { setTimeout } from 'node:timers/promises';
-import type { AnyObject } from 'pinejs-client-core';
-import { tasks as tasksEnv } from '../config-loader/env';
-import type * as Db from '../database-layer/db';
-import { TransactionClosedError } from '../database-layer/db';
-import * as permissions from '../sbvr-api/permissions';
-import { PinejsClient } from '../sbvr-api/sbvr-utils';
-import { sbvrUtils } from '../server-glue/module';
-import { ajv, channel } from './common';
-import type { PartialTask, TaskHandler, TaskStatus } from './types';
-
-// Map of column names with SBVR names used in SELECT queries
-const selectColumns = Object.entries({
-	id: 'id',
-	'is executed by-handler': 'is_executed_by__handler',
-	'is executed with-parameter set': 'is_executed_with__parameter_set',
-	'is scheduled with-cron expression': 'is_scheduled_with__cron_expression',
-	'attempt count': 'attempt_count',
-	'attempt limit': 'attempt_limit',
-	priority: 'priority',
-	'is created by-actor': 'is_created_by__actor',
-})
-	.map(([key, value]) => `t."${key}" AS "${value}"`)
-	.join(', ');
-
-// The worker is responsible for executing tasks in the queue. It listens for
-// notifications and polls the database for tasks to execute. It will execute
-// tasks in parallel up to a certain concurrency limit.
-export class Worker {
-	public handlers: Record<string, TaskHandler> = {};
-	private readonly concurrency: number;
-	private readonly interval: number;
-	private client: PinejsClient;
-	private executing = 0;
-
-	constructor(client: PinejsClient) {
-		this.client = client;
-		this.concurrency = tasksEnv.queueConcurrency;
-		this.interval = tasksEnv.queueIntervalMS;
-	}
-
-	// Check if instance can execute more tasks
-	private canExecute(): boolean {
-		return (
-			this.executing < this.concurrency && Object.keys(this.handlers).length > 0
-		);
-	}
-
-	// Execute a task
-	private async execute(tx: Db.Tx, task: PartialTask): Promise<void> {
-		this.executing++;
-		try {
-			// Get specified handler
-			const handler = this.handlers[task.is_executed_by__handler];
-			const startedOnTime = new Date();
-			if (handler == null) {
-				await this.finalize(
-					tx,
-					task,
-					startedOnTime,
-					'failed',
-					'Matching task handler not found',
-				);
-				return;
-			}
-
-			// Validate parameters before execution so we can fail early if
-			// the parameter set is invalid. This can happen if the handler
-			// definition changes after a task is added to the queue.
-			if (
-				handler.validate != null &&
-				!handler.validate(task.is_executed_with__parameter_set)
-			) {
-				await this.finalize(
-					tx,
-					task,
-					startedOnTime,
-					'failed',
-					`Invalid parameter set: ${ajv.errorsText(handler.validate.errors)}`,
-				);
-				return;
-			}
-
-			// Execute handler
-			let status: TaskStatus = 'queued';
-			let error: string | undefined;
-			try {
-				await sbvrUtils.db.transaction(async (handlerTx) => {
-					const results = await handler.fn({
-						api: new PinejsClient({
-							passthrough: {
-								tx: handlerTx,
-							},
-						}),
-						params: task.is_executed_with__parameter_set ?? {},
-						tx: handlerTx,
-					});
-					status = results.status;
-					error = results.error;
-					if (results.status !== 'succeeded' && !handlerTx.isClosed()) {
-						await handlerTx.rollback();
-					}
-				});
-			} catch (err) {
-				// Ignore closed/rollback errors
-				if (!(err instanceof TransactionClosedError)) {
-					throw err;
-				}
-			} finally {
-				// Update task with results
-				await this.finalize(tx, task, startedOnTime, status, error);
-			}
-		} catch (err) {
-			// This shouldn't happen, but if it does we want to log and kill the process
-			console.error(
-				`Failed to execute task ${task.id} with handler ${task.is_executed_by__handler}:`,
-				err,
-			);
-			process.exit(1);
-		} finally {
-			this.executing--;
-		}
-	}
-
-	// Update task and schedule next attempt if needed
-	private async finalize(
-		tx: Db.Tx,
-		task: PartialTask,
-		startedOnTime: Date,
-		status: TaskStatus,
-		errorMessage?: string,
-	): Promise<void> {
-		const attemptCount = task.attempt_count + 1;
-		const body: AnyObject = {
-			started_on__time: startedOnTime,
-			ended_on__time: new Date(),
-			status,
-			attempt_count: attemptCount,
-			...(errorMessage != null && { error_message: errorMessage }),
-		};
-
-		// Re-enqueue if the task failed but has retries left, remember that
-		// attemptCount includes the initial attempt while attempt_limit does not
-		if (status === 'failed' && attemptCount < task.attempt_limit) {
-			body.status = 'queued';
-
-			// Schedule next attempt using exponential backoff
-			body.is_scheduled_to_execute_on__time =
-				this.getNextAttemptTime(attemptCount);
-		}
-
-		// Patch current task
-		await this.client.patch({
-			resource: 'task',
-			passthrough: {
-				tx,
-				req: permissions.root,
-			},
-			id: task.id,
-			body,
-		});
-
-		// Create new task with same configuration if previous
-		// iteration completed and has a cron expression
-		if (
-			['failed', 'succeeded'].includes(body.status) &&
-			task.is_scheduled_with__cron_expression != null
-		) {
-			await this.client.post({
-				resource: 'task',
-				passthrough: {
-					tx,
-					req: permissions.root,
-				},
-				options: {
-					returnResource: false,
-				},
-				body: {
-					attempt_limit: task.attempt_limit,
-					is_created_by__actor: task.is_created_by__actor,
-					is_executed_by__handler: task.is_executed_by__handler,
-					is_executed_with__parameter_set: task.is_executed_with__parameter_set,
-					is_scheduled_with__cron_expression:
-						task.is_scheduled_with__cron_expression,
-					priority: task.priority,
-				},
-			});
-		}
-	}
-
-	// Calculate next attempt time using exponential backoff
-	private getNextAttemptTime(attempt: number): Date | null {
-		const delay = Math.ceil(Math.exp(Math.min(10, attempt)));
-		return new Date(Date.now() + delay);
-	}
-
-	// Poll for tasks to execute
-	private poll(): void {
-		let executed = false;
-		void (async () => {
-			try {
-				const handlerNames = Object.keys(this.handlers);
-				const binds = handlerNames
-					.map((_, index) => `$${index + 1}`)
-					.join(', ');
-				if (!this.canExecute()) {
-					return;
-				}
-				await sbvrUtils.db.transaction(async (tx) => {
-					const result = await tx.executeSql(
-						`SELECT ${selectColumns}
-						FROM task AS t
-						WHERE
-							t."is executed by-handler" IN (${binds}) AND
-							t."status" = 'queued' AND
-							t."attempt count" <= t."attempt limit" AND
-							(
-								t."is scheduled to execute on-time" IS NULL OR
-								t."is scheduled to execute on-time" <= CURRENT_TIMESTAMP + INTERVAL '${Math.ceil(this.interval / 1000)} second'
-							)
-						ORDER BY
-							t."is scheduled to execute on-time" ASC,
-							t."priority" DESC,
-							t."id" ASC
-						LIMIT ${Math.max(this.concurrency - this.executing, 0)}
-						FOR UPDATE SKIP LOCKED`,
-						handlerNames,
-					);
-					if (result.rows.length === 0) {
-						return;
-					}
-
-					// Tasks found, execute them in parallel
-					await Promise.all(
-						result.rows.map(async (row) => {
-							await this.execute(tx, row as PartialTask);
-						}),
-					);
-					executed = true;
-				});
-			} catch (err) {
-				console.error('Failed polling for tasks:', err);
-			} finally {
-				if (!executed) {
-					await setTimeout(this.interval);
-				}
-				this.poll();
-			}
-		})();
-	}
-
-	// Start listening and polling for tasks
-	public start(): void {
-		// Tasks only support postgres for now
-		if (sbvrUtils.db.engine !== 'postgres' || sbvrUtils.db.on == null) {
-			throw new Error(
-				'Database does not support tasks, giving up on starting worker',
-			);
-		}
-		sbvrUtils.db.on(
-			'notification',
-			async (msg) => {
-				if (this.canExecute()) {
-					await sbvrUtils.db.transaction(async (tx) => {
-						const result = await tx.executeSql(
-							`SELECT ${selectColumns} FROM task AS t WHERE id = $1 FOR UPDATE SKIP LOCKED`,
-							[msg.payload],
-						);
-						if (result.rows.length > 0) {
-							await this.execute(tx, result.rows[0] as PartialTask);
-						}
-					});
-				}
-			},
-			{
-				channel,
-			},
-		);
-		this.poll();
-	}
-}