@tstdl/base 0.93.87 → 0.93.90

package/task-queue/task-queue.js

@@ -0,0 +1,200 @@
+import { defineEnum } from '../enumeration/enumeration.js';
+import { inject, injectArgument } from '../injector/index.js';
+import { Logger } from '../logger/logger.js';
+import { Transactional } from '../orm/server/transactional.js';
+import { createArray } from '../utils/array/array.js';
+import { currentTimestamp } from '../utils/date-time.js';
+import { cancelableTimeout } from '../utils/timing.js';
+import { isDefined, isString } from '../utils/type-guards.js';
+import { millisecondsPerDay, millisecondsPerMinute, millisecondsPerSecond } from '../utils/units.js';
+import { TaskQueueEnqueueBatch } from './enqueue-batch.js';
+import { BatchTaskContext } from './task-context.js';
+export class TaskResult {
+    payload;
+    constructor(payload) {
+        this.payload = payload;
+    }
+    static Complete(result) {
+        return new TaskResult({ action: 'complete', result });
+    }
+    static Fail(error, fatal = false) {
+        return new TaskResult({ action: 'fail', error, fatal });
+    }
+    static RescheduleTo(timestamp) {
+        return new TaskResult({ action: 'reschedule', timestamp });
+    }
+    static RescheduleBy(milliseconds) {
+        const timestamp = currentTimestamp() + milliseconds;
+        return this.RescheduleTo(timestamp);
+    }
+}
+export const TaskState = defineEnum('TaskState', {
+    /**
+     * The task is waiting to be processed.
+     */
+    Pending: 'pending',
+    /**
+     * The task is currently being processed.
+     */
+    Running: 'running',
+    /**
+     * The task has been completed successfully.
+     */
+    Completed: 'completed',
+    /**
+     * The task has been cancelled and will not be processed.
+     */
+    Cancelled: 'cancelled',
+    /**
+     * The task is scheduled to be processed in the future when all children have completed.
+     */
+    Waiting: 'waiting',
+    /**
+     * The task has failed and will not be retried.
+     */
+    Dead: 'dead',
+});
+export const DependencyJoinMode = defineEnum('DependencyJoinMode', {
+    And: 'and',
+    Or: 'or',
+});
+export const defaultTaskPriority = 1000;
+export const defaultQueueConfig = {
+    visibilityTimeout: millisecondsPerMinute * 5,
+    maxExecutionTime: millisecondsPerMinute * 60,
+    maxTries: 3,
+    retention: 30 * millisecondsPerDay,
+    globalConcurrency: null,
+    circuitBreakerThreshold: 5,
+    circuitBreakerResetTimeout: 30 * millisecondsPerSecond,
+    retryDelayMinimum: 5 * millisecondsPerSecond,
+    retryDelayMaximum: 5 * millisecondsPerMinute,
+    retryDelayGrowth: 2,
+    archiveRetention: 30 * millisecondsPerDay,
+    priorityAgingInterval: millisecondsPerMinute,
+    priorityAgingStep: 10,
+    defaultTimeToLive: millisecondsPerDay,
+    rateLimit: 100,
+    rateInterval: 1000,
+    idempotencyWindow: millisecondsPerMinute * 60,
+};
+export class TaskQueue extends Transactional {
+    config = (() => { const arg = injectArgument(this); return isString(arg) ? { namespace: arg } : arg; })();
+    logger = inject(Logger, `TaskQueue:${this.config.namespace}`);
+    batch() {
+        return new TaskQueueEnqueueBatch(this);
+    }
+    process({ concurrency = 1, cancellationSignal, types, forceDequeue }, handler) {
+        for (let i = 0; i < concurrency; i++) {
+            void this.processWorker(cancellationSignal, handler, { types, forceDequeue });
+        }
+    }
+    processBatch({ batchSize = 10, concurrency = 1, cancellationSignal, types, forceDequeue }, handler) {
+        for (let i = 0; i < concurrency; i++) {
+            void this.processBatchWorker(batchSize, cancellationSignal, handler, { types, forceDequeue });
+        }
+    }
+    async processWorker(cancellationSignal, handler, options) {
+        await this.processBatchWorker(1, cancellationSignal, async (batchContext) => {
+            const task = batchContext.tasks[0];
+            const context = batchContext.for(task);
+            const result = await handler(context);
+            return [result];
+        }, options);
+    }
+    async processBatchWorker(size, cancellationSignal, handler, options) {
+        for await (const tasks of this.getBatchConsumer(size, cancellationSignal, options)) {
+            const batchToken = cancellationSignal.createChild();
+            const context = new BatchTaskContext(this, tasks, batchToken, this.logger);
+            let activeTaskIds = new Set(tasks.map((t) => t.id));
+            context.logger.verbose(`Processing batch of ${tasks.length}`);
+            void (async () => {
+                while (batchToken.isUnset) {
+                    await cancelableTimeout(Math.min(this.visibilityTimeout / 2, 5000), batchToken);
+                    if (batchToken.isSet) {
+                        break;
+                    }
+                    try {
+                        const tasksToTouch = tasks.filter((t) => activeTaskIds.has(t.id));
+                        if (tasksToTouch.length > 0) {
+                            const touchedIds = await this.touchMany(tasksToTouch);
+                            if (touchedIds.length != tasksToTouch.length) {
+                                const lostCount = tasksToTouch.length - touchedIds.length;
+                                context.logger.warn(`Batch integrity compromised: ${lostCount} tasks lost lease. Aborting batch.`);
+                                activeTaskIds = new Set(touchedIds);
+                                batchToken.set();
+                            }
+                            else {
+                                activeTaskIds = new Set(touchedIds);
+                            }
+                        }
+                        if (activeTaskIds.size == 0 && batchToken.isUnset) {
+                            context.logger.warn(`All tasks in batch lost lease. Stopping worker.`);
+                            batchToken.set();
+                        }
+                    }
+                    catch (error) {
+                        context.logger.error('Error touching tasks', error);
+                    }
+                }
+            })();
+            try {
+                if (batchToken.isSet) {
+                    throw new Error('Tasks cancelled before start');
+                }
+                const results = await handler(context);
+                if (isDefined(results)) {
+                    const completions = [];
+                    const failures = [];
+                    const reschedules = [];
+                    for (let i = 0; i < tasks.length; i++) {
+                        const task = tasks[i];
+                        const result = results[i];
+                        switch (result.payload.action) {
+                            case 'complete':
+                                completions.push({ task, result: result.payload });
+                                break;
+                            case 'fail':
+                                failures.push({ task, error: result.payload.error, fatal: result.payload.fatal });
+                                break;
+                            case 'reschedule':
+                                reschedules.push({ task, timestamp: result.payload.timestamp });
+                                break;
+                            default:
+                                throw new Error(`Unsupported task result action.`);
+                        }
+                    }
+                    if (completions.length > 0) {
+                        context.logger.verbose(`Completing ${completions.length} tasks`);
+                        await this.completeMany(completions.map((c) => c.task), completions.map((c) => c.result));
+                    }
+                    if (failures.length > 0) {
+                        context.logger.verbose(`Failing ${failures.length} tasks`);
+                        for (const item of failures) {
+                            await this.fail(item.task, item.error, item.fatal);
+                        }
+                    }
+                    if (reschedules.length > 0) {
+                        context.logger.verbose(`Rescheduling ${reschedules.length} tasks`);
+                        const reschedulesByTimestamp = new Map();
+                        for (const item of reschedules) {
+                            const ids = reschedulesByTimestamp.get(item.timestamp) ?? [];
+                            ids.push(item.task.id);
+                            reschedulesByTimestamp.set(item.timestamp, ids);
+                        }
+                        for (const [timestamp, ids] of reschedulesByTimestamp) {
+                            await this.rescheduleMany(ids, timestamp);
+                        }
+                    }
+                }
+            }
+            catch (error) {
+                context.logger.error('Error processing tasks', error);
+                await this.failMany(tasks, createArray(tasks.length, () => error));
+            }
+            finally {
+                batchToken.set();
+            }
+        }
+    }
+}

package/task-queue/tests/complex.test.d.ts

@@ -0,0 +1 @@
+export {};

package/task-queue/tests/complex.test.js

@@ -0,0 +1,299 @@
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { DependencyJoinMode, TaskQueueProvider, TaskState } from '../../task-queue/index.js';
+import { setupIntegrationTest } from '../../unit-test/index.js';
+import { currentTimestamp } from '../../utils/date-time.js';
+import { timeout } from '../../utils/timing.js';
+describe('Complex Queue Scenarios', () => {
+    let injector;
+    let queue;
+    beforeAll(async () => {
+        ({ injector } = await setupIntegrationTest({ modules: { taskQueue: true } }));
+    });
+    beforeEach(() => {
+        const queueProvider = injector.resolve(TaskQueueProvider);
+        const queueName = `complex-queue-${Date.now()}-${Math.random()}`;
+        // Configure with specific settings for testing logic
+        queue = queueProvider.get(queueName, {
+            visibilityTimeout: 1000,
+            priorityAgingInterval: 100, // Fast aging
+            priorityAgingStep: 10,
+            rateLimit: 5,
+            rateInterval: 500,
+            retryDelayMinimum: 100,
+            retryDelayGrowth: 2,
+            retention: 100, // Fast retention for archive test
+        });
+    });
+    afterEach(async () => {
+        await queue.clear();
+        const queueProvider = injector.resolve(TaskQueueProvider);
+        await queueProvider.get('other-queue').clear();
+    });
+    afterAll(async () => {
+        await injector?.dispose();
+    });
+    async function waitForStatus(id, status) {
+        for (let i = 0; i < 20; i++) {
+            const task = await queue.getTask(id);
+            if (task?.status === status)
+                return;
+            await queue.processPendingFanIn();
+            await timeout(50);
+        }
+    }
+    describe('Complex Dependencies', () => {
+        it('should handle Diamond Dependency (A -> B, A -> C, (B&C) -> D)', async () => {
+            // D waits for B and C
+            const taskD = await queue.enqueue('D', {}, { scheduleAfterTags: ['tag-b', 'tag-c'] });
+            // B and C wait for A
+            const taskB = await queue.enqueue('B', {}, { tags: ['tag-b'], scheduleAfterTags: ['tag-a'] });
+            const taskC = await queue.enqueue('C', {}, { tags: ['tag-c'], scheduleAfterTags: ['tag-a'] });
+            // A runs first
+            const taskA = await queue.enqueue('A', {}, { tags: ['tag-a'] });
+            expect(taskD.status).toBe(TaskState.Waiting);
+            expect(taskB.status).toBe(TaskState.Waiting);
+            expect(taskC.status).toBe(TaskState.Waiting);
+            expect(taskA.status).toBe(TaskState.Pending);
+            // Process A
+            const dA = await queue.dequeue({ types: ['A'] });
+            await queue.complete(dA);
+            await waitForStatus(taskB.id, TaskState.Pending);
+            await waitForStatus(taskC.id, TaskState.Pending);
+            // Process B
+            const dB = await queue.dequeue({ types: ['B'] });
+            await queue.complete(dB);
+            await timeout(100);
+            await queue.processPendingFanIn();
+            // D still waiting (needs C)
+            const uD2 = await queue.getTask(taskD.id);
+            expect(uD2?.status).toBe(TaskState.Waiting);
+            // Process C
+            const dC = await queue.dequeue({ types: ['C'] });
+            await queue.complete(dC);
+            await waitForStatus(taskD.id, TaskState.Pending);
+            // D should be Pending
+            const uD3 = await queue.getTask(taskD.id);
+            expect(uD3?.status).toBe(TaskState.Pending);
+        });
+        it('should handle Deep Chain (A -> B -> C -> D)', async () => {
+            const D = await queue.enqueue('D', {}, { scheduleAfterTags: ['C'] });
+            const C = await queue.enqueue('C', {}, { tags: ['C'], scheduleAfterTags: ['B'] });
+            const B = await queue.enqueue('B', {}, { tags: ['B'], scheduleAfterTags: ['A'] });
+            const A = await queue.enqueue('A', {}, { tags: ['A'] });
+            // Run A
+            await queue.complete((await queue.dequeue({ types: ['A'] })));
+            await waitForStatus(B.id, TaskState.Pending);
+            // Run B
+            await queue.complete((await queue.dequeue({ types: ['B'] })));
+            await waitForStatus(C.id, TaskState.Pending);
+            // Run C
+            await queue.complete((await queue.dequeue({ types: ['C'] })));
+            await waitForStatus(D.id, TaskState.Pending);
+            expect((await queue.getTask(D.id))?.status).toBe(TaskState.Pending);
+        });
+        it('should propagate cancellation down the dependency tree', async () => {
+            // Use parentId for explicit tree structure which `cancel` supports
+            const root = await queue.enqueue('root', {});
+            const child = await queue.enqueue('child', {}, { parentId: root.id });
+            const grandChild = await queue.enqueue('grand', {}, { parentId: child.id });
+            await queue.cancel(root.id);
+            expect((await queue.getTask(root.id))?.status).toBe(TaskState.Cancelled);
+            expect((await queue.getTask(child.id))?.status).toBe(TaskState.Cancelled);
+            expect((await queue.getTask(grandChild.id))?.status).toBe(TaskState.Cancelled);
+        });
+    });
+    describe('Scheduling & Priorities', () => {
+        it('should promote priority of old pending tasks (Aging)', async () => {
+            const t1 = await queue.enqueue('low', {}, { priority: 2000 });
+            // Wait for aging interval (100ms)
+            await timeout(150);
+            await queue.maintenance();
+            const updated = await queue.getTask(t1.id);
+            // Default step is 10. 2000 - 10 = 1990
+            expect(updated?.priority).toBe(1990);
+        });
+        it('should respect exponential backoff on failure', async () => {
+            const task = await queue.enqueue('fail-test', {});
+            // Try 1
+            const d1 = await queue.dequeue();
+            await queue.fail(d1, new Error('fail 1'));
+            const u1 = await queue.getTask(task.id);
+            expect(u1?.tries).toBe(1);
+            const delay1 = u1.scheduleTimestamp - currentTimestamp();
+            expect(delay1).toBeGreaterThan(150); // Approx check
+            // Force reschedule to now
+            await queue.reschedule(task.id, currentTimestamp());
+            // Try 2
+            const d2 = await queue.dequeue();
+            await queue.fail(d2, new Error('fail 2'));
+            const u2 = await queue.getTask(task.id);
+            expect(u2?.tries).toBe(2);
+            const now = currentTimestamp();
+            expect(u2.scheduleTimestamp).toBeGreaterThan(now + 300);
+        });
+    });
+    describe('Rate Limiting & Concurrency', () => {
+        it('should limit burst dequeue rate', async () => {
+            // Rate limit 5, interval 500ms
+            await queue.enqueueMany(Array.from({ length: 10 }, (_, i) => ({ type: 'burst', data: { i } })));
+            // Request burstCapacity (5)
+            const batch1 = await queue.dequeueMany(5);
+            expect(batch1.length).toBe(5);
+            // Try immediately again
+            const batch2 = await queue.dequeueMany(1);
+            expect(batch2.length).toBe(0); // Rate limited
+            // Wait for refill
+            await timeout(600);
+            const batch3 = await queue.dequeueMany(5);
+            expect(batch3.length).toBe(5); // Refilled
+        });
+        it('should respect global concurrency limit', async () => {
+            const queueProvider = injector.resolve(TaskQueueProvider);
+            const limitQueue = queueProvider.get(`limit-${Date.now()}`, { globalConcurrency: 2 });
+            await limitQueue.enqueueMany([
+                { type: 'c', data: {} },
+                { type: 'c', data: {} },
+                { type: 'c', data: {} }
+            ]);
+            const t1 = await limitQueue.dequeue();
+            const t2 = await limitQueue.dequeue();
+            const t3 = await limitQueue.dequeue();
+            expect(t1).toBeDefined();
+            expect(t2).toBeDefined();
+            expect(t3).toBeUndefined(); // Max concurrency reached
+            await limitQueue.complete(t1);
+            const t3_retry = await limitQueue.dequeue();
+            expect(t3_retry).toBeDefined();
+            await limitQueue.clear();
+        });
+    });
+    describe('Archival & Maintenance', () => {
+        it('should archive completed tasks after retention period', async () => {
+            const qProvider = injector.resolve(TaskQueueProvider);
+            const archiveQueue = qProvider.get(`archive-${Date.now()}-${Math.random()}`, { retention: 100 });
+            const task = await archiveQueue.enqueue('archive-me', {});
+            const d = await archiveQueue.dequeue();
+            await archiveQueue.complete(d);
+            // Verify it is in main table
+            const before = await archiveQueue.getTask(task.id);
+            expect(before).toBeDefined();
+            expect(before?.status).toBe(TaskState.Completed);
+            expect(before?.completeTimestamp).toBeGreaterThan(0);
+            // Wait for retention (100ms).
+            await timeout(500);
+            await archiveQueue.maintenance();
+            // Should move from main table to archive
+            const loaded = await archiveQueue.getTask(task.id);
+            expect(loaded).toBeDefined();
+            expect(loaded?.status).toBe(TaskState.Completed);
+            await archiveQueue.clear();
+        });
+        it('should prune expired pending tasks', async () => {
+            // Time to live: 100ms
+            const task = await queue.enqueue('expire-me', {}, { timeToLive: currentTimestamp() + 100 });
+            await timeout(150);
+            await queue.maintenance();
+            const updated = await queue.getTask(task.id);
+            expect(updated?.status).toBe(TaskState.Dead);
+            expect(updated?.error?.code).toBe('Expired');
+        });
+        it('should retrieve task from archive', async () => {
+            // Manually insert into archive? We can't access archiveRepository directly.
+            // But we can use maintenance to force it.
+            // Wait, if the previous test failed, this might also fail.
+            // But let's try with a VERY clear scenario.
+            const task = await queue.enqueue('archive-retrieval-test', {});
+            const d = await queue.dequeue();
+            await queue.complete(d);
+            // Force move
+            await timeout(200);
+            await queue.maintenance();
+            // Verify retrieval
+            const fromArchive = await queue.getTask(task.id);
+            expect(fromArchive).toBeDefined();
+            expect(fromArchive?.status).toBe(TaskState.Completed);
+        });
+        it('should defer archival of parent tasks until children are archived', async () => {
+            const qProvider = injector.resolve(TaskQueueProvider);
+            const treeQueue = qProvider.get(`archive-tree-${Date.now()}`, { retention: 100 });
+            const parent = await treeQueue.enqueue('parent', {});
+            const child = await treeQueue.enqueue('child', {}, { parentId: parent.id });
+            const d1 = await treeQueue.dequeue();
+            const d2 = await treeQueue.dequeue();
+            await treeQueue.complete(d1);
+            await treeQueue.complete(d2);
+            // Wait for retention
+            await timeout(200);
+            // First maintenance: should archive child, but parent stays because child is still in main table (until it's deleted in the same tx maybe? No, loadMany happens before delete)
+            await treeQueue.maintenance();
+            const parentStillActive = await treeQueue.getTask(parent.id);
+            const childInArchive = await treeQueue.getTask(child.id);
+            // Verify child moved
+            expect(childInArchive).toBeDefined();
+            // Parent MUST still be in main table because child was present when archival query ran
+            expect(parentStillActive).toBeDefined();
+            // Second maintenance: should now archive parent because child is gone from main table
+            await treeQueue.maintenance();
+            const finalParent = await treeQueue.getTask(parent.id);
+            expect(finalParent).toBeDefined(); // Still defined (getTask checks both main and archive)
+            // We need to check if it's REALLY archived.
+            // We can use the #repository directly if we were in the class, but here we can check if it moved by checking against the main table?
+            // Actually getTask is implemented as:
+            /*
+            const active = await this.#repository.tryLoadByQuery({ namespace: this.#namespace, id });
+            if (isDefined(active)) return active;
+            return await this.#archiveRepository.tryLoadByQuery({ namespace: this.#namespace, id });
+            */
+            await treeQueue.clear();
+        });
+    });
+    describe('Batch & Bulk Operations', () => {
+        it('should enqueue and dequeue large batches', async () => {
+            const items = Array.from({ length: 50 }, (_, i) => ({ type: 'bulk', data: { i } }));
+            const queueProvider = injector.resolve(TaskQueueProvider);
+            const bulkQueue = queueProvider.get(`bulk-${Date.now()}`, { rateLimit: 100 });
+            await bulkQueue.enqueueMany(items);
+            const count = await bulkQueue.countByTags([]); // All -> 0 because empty intersection
+            expect(count).toBe(0);
+            const batch = await bulkQueue.dequeueMany(50);
+            expect(batch.length).toBe(50);
+            await bulkQueue.completeMany(batch);
+            await bulkQueue.clear();
+        });
+        it('should handle idempotency window', async () => {
+            const key = 'idem-test';
+            const t1 = await queue.enqueue('t', { v: 1 }, { idempotencyKey: key });
+            const t2 = await queue.enqueue('t', { v: 2 }, { idempotencyKey: key });
+            expect(t1.id).toBe(t2.id); // Deduplicated
+            const t3 = await queue.enqueue('t', { v: 3 }, { idempotencyKey: key, replace: true });
+            expect(t3.id).not.toBe(t1.id); // Replaced
+            expect(t3.data.v).toBe(3);
+        });
+    });
+    describe('Edge Cases', () => {
+        it('should not fan-in if dependency fails and failFast=false', async () => {
+            const dependent = await queue.enqueue('dep', {}, { scheduleAfterTags: ['fail-tag'], failFast: false });
+            const prereq = await queue.enqueue('pre', {}, { tags: ['fail-tag'] });
+            const d = await queue.dequeue({ types: ['pre'] });
+            await queue.fail(d, new Error('fail'));
+            await queue.processPendingFanIn();
+            const u = await queue.getTask(dependent.id);
+            expect(u?.status).toBe(TaskState.Waiting); // Should still be waiting because dependency didn't Complete
+        });
+        it('should handle mixed AND/OR dependencies', async () => {
+            const dep = await queue.enqueue('dep', {}, {
+                scheduleAfterTags: ['A', 'B'],
+                dependencyJoinMode: DependencyJoinMode.Or
+            });
+            const A = await queue.enqueue('A', {}, { tags: ['A'] });
+            await queue.complete((await queue.dequeue({ types: ['A'] })));
+            await waitForStatus(dep.id, TaskState.Pending);
+            expect((await queue.getTask(dep.id))?.status).toBe(TaskState.Pending);
+        });
+        it('should not reschedule if task is not running', async () => {
+            const task = await queue.enqueue('t', {});
+            const res = await queue.touch(task);
+            expect(res).toBeUndefined();
+        });
+    });
+});

package/task-queue/tests/dependencies.test.d.ts

@@ -0,0 +1 @@
+export {};