@pikku/queue-pg-boss 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/pg-boss-queue-worker.d.ts +9 -5
- package/dist/pg-boss-queue-worker.js +47 -33
- package/dist/pg-boss-scheduler-service.d.ts +20 -2
- package/dist/pg-boss-scheduler-service.js +47 -1
- package/dist/pg-boss-service-factory.d.ts +7 -2
- package/dist/pg-boss-service-factory.js +16 -1
- package/package.json +3 -3
- package/src/pg-boss-queue-worker.test.ts +15 -0
- package/src/pg-boss-queue-worker.ts +67 -45
- package/src/pg-boss-scheduler-service.ts +57 -1
- package/src/pg-boss-service-factory.ts +28 -15
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
|
+
## 0.12.0
|
|
2
|
+
|
|
3
|
+
### New Features
|
|
4
|
+
|
|
5
|
+
- `setServices()`, `start()`, `stop()` lifecycle methods on scheduler and queue workers
|
|
6
|
+
|
|
1
7
|
## 0.11.0
|
|
2
8
|
|
|
9
|
+
## 0.11.2
|
|
10
|
+
|
|
11
|
+
### Patch Changes
|
|
12
|
+
|
|
13
|
+
- db9c7bf: Simplify CreateWireServices type signature for custom Config compatibility
|
|
14
|
+
- Updated dependencies [db9c7bf]
|
|
15
|
+
- @pikku/core@0.11.2
|
|
16
|
+
|
|
17
|
+
### Fixes
|
|
18
|
+
|
|
19
|
+
- ddd87eaf: Simplify CreateWireServices type signature for custom Config compatibility
|
|
20
|
+
|
|
3
21
|
## 0.11.1
|
|
4
22
|
|
|
5
23
|
### Patch Changes
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { PgBoss, type WorkOptions } from 'pg-boss';
|
|
2
|
-
import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult } from '@pikku/core/queue';
|
|
3
|
-
import {
|
|
2
|
+
import type { QueueWorkers, PikkuWorkerConfig, QueueConfigMapping, ConfigValidationResult, QueueJob } from '@pikku/core/queue';
|
|
3
|
+
import type { Logger } from '@pikku/core/services';
|
|
4
4
|
export declare const mapPikkuWorkerToPgBoss: (workerConfig?: PikkuWorkerConfig) => WorkOptions;
|
|
5
5
|
/**
|
|
6
6
|
* pg-boss/PostgreSQL queue worker implementation
|
|
7
7
|
* Supports job results, retries, and PostgreSQL-based queuing
|
|
8
8
|
*/
|
|
9
9
|
export declare class PgBossQueueWorkers implements QueueWorkers {
|
|
10
|
-
private singletonServices;
|
|
11
|
-
private createWireServices?;
|
|
12
10
|
readonly name = "pg-boss";
|
|
13
11
|
readonly supportsResults = true;
|
|
14
12
|
/**
|
|
@@ -18,7 +16,13 @@ export declare class PgBossQueueWorkers implements QueueWorkers {
|
|
|
18
16
|
readonly configMappings: QueueConfigMapping;
|
|
19
17
|
private pgBoss;
|
|
20
18
|
private activeWorkers;
|
|
21
|
-
|
|
19
|
+
private runJob?;
|
|
20
|
+
private logger?;
|
|
21
|
+
constructor(pgBoss: PgBoss);
|
|
22
|
+
setJobRunner(runJob: (params: {
|
|
23
|
+
job: QueueJob;
|
|
24
|
+
updateProgress?: (progress: number | string | object) => Promise<void>;
|
|
25
|
+
}) => Promise<void>, logger: Logger): void;
|
|
22
26
|
/**
|
|
23
27
|
* Scan state and register all compatible processors
|
|
24
28
|
*/
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { registerQueueWorkers, QueueJobFailedError, QueueJobDiscardedError, } from '@pikku/core/queue';
|
|
2
2
|
import { mapPgBossJobToQueueJob } from './utils.js';
|
|
3
3
|
export const mapPikkuWorkerToPgBoss = (workerConfig) => {
|
|
4
|
-
const workerOptions = {
|
|
4
|
+
const workerOptions = {
|
|
5
|
+
batchSize: 10,
|
|
6
|
+
};
|
|
5
7
|
if (workerConfig?.batchSize !== undefined) {
|
|
6
8
|
workerOptions.batchSize = workerConfig.batchSize;
|
|
7
9
|
}
|
|
@@ -15,8 +17,6 @@ export const mapPikkuWorkerToPgBoss = (workerConfig) => {
|
|
|
15
17
|
* Supports job results, retries, and PostgreSQL-based queuing
|
|
16
18
|
*/
|
|
17
19
|
export class PgBossQueueWorkers {
|
|
18
|
-
singletonServices;
|
|
19
|
-
createWireServices;
|
|
20
20
|
name = 'pg-boss';
|
|
21
21
|
supportsResults = true;
|
|
22
22
|
/**
|
|
@@ -28,7 +28,7 @@ export class PgBossQueueWorkers {
|
|
|
28
28
|
supported: {
|
|
29
29
|
batchSize: {
|
|
30
30
|
queueProperty: 'batchSize',
|
|
31
|
-
description: 'Number of jobs to process in
|
|
31
|
+
description: 'Number of jobs to fetch and process concurrently in parallel',
|
|
32
32
|
},
|
|
33
33
|
pollInterval: {
|
|
34
34
|
queueProperty: 'pollingIntervalSeconds',
|
|
@@ -83,47 +83,61 @@ export class PgBossQueueWorkers {
|
|
|
83
83
|
};
|
|
84
84
|
pgBoss;
|
|
85
85
|
activeWorkers = new Map();
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
86
|
+
runJob;
|
|
87
|
+
logger;
|
|
88
|
+
constructor(pgBoss) {
|
|
89
89
|
this.pgBoss = pgBoss;
|
|
90
90
|
}
|
|
91
|
+
setJobRunner(runJob, logger) {
|
|
92
|
+
this.runJob = runJob;
|
|
93
|
+
this.logger = logger;
|
|
94
|
+
}
|
|
91
95
|
/**
|
|
92
96
|
* Scan state and register all compatible processors
|
|
93
97
|
*/
|
|
94
98
|
async registerQueues() {
|
|
95
|
-
|
|
99
|
+
if (!this.runJob || !this.logger) {
|
|
100
|
+
throw new Error('PgBossQueueWorkers requires setJobRunner() before registerQueues()');
|
|
101
|
+
}
|
|
102
|
+
return await registerQueueWorkers(this.configMappings, this.logger, async (queueName, processor) => {
|
|
96
103
|
await this.pgBoss.createQueue(queueName);
|
|
97
|
-
const workerId = await this.pgBoss.work(processor.
|
|
104
|
+
const workerId = await this.pgBoss.work(processor.name, {
|
|
98
105
|
...mapPikkuWorkerToPgBoss(processor.config),
|
|
99
106
|
includeMetadata: true,
|
|
100
|
-
}, async (
|
|
101
|
-
if (!
|
|
102
|
-
|
|
103
|
-
}
|
|
104
|
-
if (!job) {
|
|
105
|
-
this.singletonServices.logger.warn(`No job received for queue ${queueName}`);
|
|
107
|
+
}, async (jobs) => {
|
|
108
|
+
if (!jobs || jobs.length === 0) {
|
|
109
|
+
this.logger.warn(`No jobs received for queue ${queueName}`);
|
|
106
110
|
return;
|
|
107
111
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
catch (error) {
|
|
116
|
-
if (error instanceof QueueJobFailedError) {
|
|
117
|
-
// Let pg-boss handle this as a failed job
|
|
118
|
-
throw new Error(error.message);
|
|
112
|
+
// Process all jobs in parallel
|
|
113
|
+
await Promise.all(jobs.map(async (job) => {
|
|
114
|
+
try {
|
|
115
|
+
await this.runJob({
|
|
116
|
+
job: mapPgBossJobToQueueJob(job, this.pgBoss),
|
|
117
|
+
});
|
|
119
118
|
}
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
119
|
+
catch (error) {
|
|
120
|
+
if (error instanceof QueueJobFailedError) {
|
|
121
|
+
// Mark this specific job as failed
|
|
122
|
+
await this.pgBoss.fail(queueName, job.id, {
|
|
123
|
+
message: error.message,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
else if (error instanceof QueueJobDiscardedError) {
|
|
127
|
+
// For pg-boss, complete the job successfully to discard it
|
|
128
|
+
this.logger.info(`PgBoss job ${job.id} discarded: ${error.message}`);
|
|
129
|
+
await this.pgBoss.complete(queueName, job.id);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
// For other errors, fail the job
|
|
133
|
+
await this.pgBoss.fail(queueName, job.id, {
|
|
134
|
+
message: error instanceof Error
|
|
135
|
+
? error.message
|
|
136
|
+
: 'Unknown error',
|
|
137
|
+
});
|
|
138
|
+
}
|
|
124
139
|
}
|
|
125
|
-
|
|
126
|
-
}
|
|
140
|
+
}));
|
|
127
141
|
});
|
|
128
142
|
this.activeWorkers.set(queueName, workerId);
|
|
129
143
|
});
|
|
@@ -1,12 +1,21 @@
|
|
|
1
1
|
import { PgBoss } from 'pg-boss';
|
|
2
|
-
import { SchedulerService, ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
|
|
2
|
+
import { SchedulerService, SchedulerRuntimeHandlers, ScheduledTaskInfo, ScheduledTaskSummary, CoreUserSession } from '@pikku/core';
|
|
3
3
|
/**
|
|
4
4
|
* pg-boss scheduler service implementation
|
|
5
|
-
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL
|
|
5
|
+
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
|
|
6
|
+
* Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
|
|
6
7
|
*/
|
|
7
8
|
export declare class PgBossSchedulerService extends SchedulerService {
|
|
8
9
|
private pgBoss;
|
|
10
|
+
private scheduledCronNames;
|
|
11
|
+
private handlers?;
|
|
9
12
|
constructor(pgBoss: PgBoss);
|
|
13
|
+
/**
|
|
14
|
+
* Set services needed for processing recurring tasks.
|
|
15
|
+
* Must be called before start() since the scheduler is typically
|
|
16
|
+
* created before singletonServices are available.
|
|
17
|
+
*/
|
|
18
|
+
setServices(handlers: SchedulerRuntimeHandlers): void;
|
|
10
19
|
/**
|
|
11
20
|
* Initialize - no-op as pg-boss is already initialized by factory
|
|
12
21
|
*/
|
|
@@ -32,4 +41,13 @@ export declare class PgBossSchedulerService extends SchedulerService {
|
|
|
32
41
|
* Close - no-op as pg-boss lifecycle is managed by factory
|
|
33
42
|
*/
|
|
34
43
|
close(): Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Start recurring scheduled tasks.
|
|
46
|
+
* Registers a pg-boss worker to process recurring jobs via runScheduledTask.
|
|
47
|
+
*/
|
|
48
|
+
start(): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Stop recurring scheduled tasks by unscheduling them from pg-boss.
|
|
51
|
+
*/
|
|
52
|
+
stop(): Promise<void>;
|
|
35
53
|
}
|
|
@@ -1,14 +1,26 @@
|
|
|
1
1
|
import { SchedulerService, parseDurationString, } from '@pikku/core';
|
|
2
|
+
import { getScheduledTasks } from '@pikku/core/scheduler';
|
|
2
3
|
/**
|
|
3
4
|
* pg-boss scheduler service implementation
|
|
4
|
-
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL
|
|
5
|
+
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
|
|
6
|
+
* Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
|
|
5
7
|
*/
|
|
6
8
|
export class PgBossSchedulerService extends SchedulerService {
|
|
7
9
|
pgBoss;
|
|
10
|
+
scheduledCronNames = [];
|
|
11
|
+
handlers;
|
|
8
12
|
constructor(pgBoss) {
|
|
9
13
|
super();
|
|
10
14
|
this.pgBoss = pgBoss;
|
|
11
15
|
}
|
|
16
|
+
/**
|
|
17
|
+
* Set services needed for processing recurring tasks.
|
|
18
|
+
* Must be called before start() since the scheduler is typically
|
|
19
|
+
* created before singletonServices are available.
|
|
20
|
+
*/
|
|
21
|
+
setServices(handlers) {
|
|
22
|
+
this.handlers = handlers;
|
|
23
|
+
}
|
|
12
24
|
/**
|
|
13
25
|
* Initialize - no-op as pg-boss is already initialized by factory
|
|
14
26
|
*/
|
|
@@ -77,4 +89,38 @@ export class PgBossSchedulerService extends SchedulerService {
|
|
|
77
89
|
async close() {
|
|
78
90
|
// No-op - pg-boss lifecycle is managed by factory
|
|
79
91
|
}
|
|
92
|
+
/**
|
|
93
|
+
* Start recurring scheduled tasks.
|
|
94
|
+
* Registers a pg-boss worker to process recurring jobs via runScheduledTask.
|
|
95
|
+
*/
|
|
96
|
+
async start() {
|
|
97
|
+
if (!this.handlers) {
|
|
98
|
+
throw new Error('PgBossSchedulerService requires setServices() before start()');
|
|
99
|
+
}
|
|
100
|
+
const scheduledTasks = getScheduledTasks();
|
|
101
|
+
for (const [name, task] of scheduledTasks) {
|
|
102
|
+
const cronName = `pikku-recurring-scheduled-task:${name}`;
|
|
103
|
+
await this.pgBoss.schedule(cronName, task.schedule, {
|
|
104
|
+
rpcName: name,
|
|
105
|
+
});
|
|
106
|
+
this.scheduledCronNames.push(cronName);
|
|
107
|
+
// Register a worker to process the scheduled jobs
|
|
108
|
+
await this.pgBoss.work(cronName, async (jobs) => {
|
|
109
|
+
for (const job of jobs) {
|
|
110
|
+
const { rpcName } = job.data;
|
|
111
|
+
this.handlers.logger.info(`Running scheduled task: ${rpcName}`);
|
|
112
|
+
await this.handlers.runScheduledTask(rpcName);
|
|
113
|
+
}
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Stop recurring scheduled tasks by unscheduling them from pg-boss.
|
|
119
|
+
*/
|
|
120
|
+
async stop() {
|
|
121
|
+
for (const cronName of this.scheduledCronNames) {
|
|
122
|
+
await this.pgBoss.unschedule(cronName);
|
|
123
|
+
}
|
|
124
|
+
this.scheduledCronNames = [];
|
|
125
|
+
}
|
|
80
126
|
}
|
|
@@ -2,7 +2,7 @@ import { PgBoss, type ConstructorOptions } from 'pg-boss';
|
|
|
2
2
|
import { PgBossQueueService } from './pg-boss-queue-service.js';
|
|
3
3
|
import { PgBossQueueWorkers } from './pg-boss-queue-worker.js';
|
|
4
4
|
import { PgBossSchedulerService } from './pg-boss-scheduler-service.js';
|
|
5
|
-
import type {
|
|
5
|
+
import type { CoreSingletonServices, CreateWireServices } from '@pikku/core';
|
|
6
6
|
/**
|
|
7
7
|
* Factory class for pg-boss services
|
|
8
8
|
* Manages a single PgBoss instance shared across queue, worker, and scheduler services
|
|
@@ -25,13 +25,18 @@ export declare class PgBossServiceFactory {
|
|
|
25
25
|
/**
|
|
26
26
|
* Get the queue workers for processing jobs
|
|
27
27
|
*/
|
|
28
|
-
getQueueWorkers(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices
|
|
28
|
+
getQueueWorkers(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices): PgBossQueueWorkers;
|
|
29
29
|
/**
|
|
30
30
|
* Get the scheduler service for managing recurring tasks
|
|
31
31
|
*/
|
|
32
32
|
getSchedulerService(): PgBossSchedulerService;
|
|
33
|
+
setSchedulerRuntime(singletonServices: CoreSingletonServices, createWireServices?: CreateWireServices): void;
|
|
33
34
|
/**
|
|
34
35
|
* Close all services and the pg-boss connection
|
|
35
36
|
*/
|
|
36
37
|
close(): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Alias for close() - used by stopSingletonServices
|
|
40
|
+
*/
|
|
41
|
+
stop(): Promise<void>;
|
|
37
42
|
}
|
|
@@ -2,6 +2,8 @@ import { PgBoss } from 'pg-boss';
|
|
|
2
2
|
import { PgBossQueueService } from './pg-boss-queue-service.js';
|
|
3
3
|
import { PgBossQueueWorkers } from './pg-boss-queue-worker.js';
|
|
4
4
|
import { PgBossSchedulerService } from './pg-boss-scheduler-service.js';
|
|
5
|
+
import { createQueueJobRunner } from '@pikku/core/queue';
|
|
6
|
+
import { createSchedulerRuntimeHandlers } from '@pikku/core/scheduler';
|
|
5
7
|
/**
|
|
6
8
|
* Factory class for pg-boss services
|
|
7
9
|
* Manages a single PgBoss instance shared across queue, worker, and scheduler services
|
|
@@ -46,8 +48,9 @@ export class PgBossServiceFactory {
|
|
|
46
48
|
*/
|
|
47
49
|
getQueueWorkers(singletonServices, createWireServices) {
|
|
48
50
|
if (!this.queueWorkers) {
|
|
49
|
-
this.queueWorkers = new PgBossQueueWorkers(this.pgBoss
|
|
51
|
+
this.queueWorkers = new PgBossQueueWorkers(this.pgBoss);
|
|
50
52
|
}
|
|
53
|
+
this.queueWorkers.setJobRunner(createQueueJobRunner({ singletonServices, createWireServices }), singletonServices.logger);
|
|
51
54
|
return this.queueWorkers;
|
|
52
55
|
}
|
|
53
56
|
/**
|
|
@@ -59,6 +62,12 @@ export class PgBossServiceFactory {
|
|
|
59
62
|
}
|
|
60
63
|
return this.schedulerService;
|
|
61
64
|
}
|
|
65
|
+
setSchedulerRuntime(singletonServices, createWireServices) {
|
|
66
|
+
this.getSchedulerService().setServices(createSchedulerRuntimeHandlers({
|
|
67
|
+
singletonServices,
|
|
68
|
+
createWireServices,
|
|
69
|
+
}));
|
|
70
|
+
}
|
|
62
71
|
/**
|
|
63
72
|
* Close all services and the pg-boss connection
|
|
64
73
|
*/
|
|
@@ -68,4 +77,10 @@ export class PgBossServiceFactory {
|
|
|
68
77
|
this.initialized = false;
|
|
69
78
|
}
|
|
70
79
|
}
|
|
80
|
+
/**
|
|
81
|
+
* Alias for close() - used by stopSingletonServices
|
|
82
|
+
*/
|
|
83
|
+
async stop() {
|
|
84
|
+
await this.close();
|
|
85
|
+
}
|
|
71
86
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/queue-pg-boss",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.12.0",
|
|
4
4
|
"author": "yasser.fadl@gmail.com",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "dist/index.js",
|
|
@@ -13,10 +13,10 @@
|
|
|
13
13
|
"release": "npm run build && npm test"
|
|
14
14
|
},
|
|
15
15
|
"peerDependencies": {
|
|
16
|
-
"@pikku/core": "^0.
|
|
16
|
+
"@pikku/core": "^0.12.0"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"pg-boss": "^12.
|
|
19
|
+
"pg-boss": "^12.9.0"
|
|
20
20
|
},
|
|
21
21
|
"devDependencies": {
|
|
22
22
|
"typescript": "^5.9"
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { describe, test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
|
|
4
|
+
import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
|
|
5
|
+
|
|
6
|
+
describe('PgBossQueueWorkers', () => {
|
|
7
|
+
test('requires setJobRunner before registerQueues', async () => {
|
|
8
|
+
const workers = new PgBossQueueWorkers({} as any)
|
|
9
|
+
|
|
10
|
+
await assert.rejects(() => workers.registerQueues(), {
|
|
11
|
+
message:
|
|
12
|
+
'PgBossQueueWorkers requires setJobRunner() before registerQueues()',
|
|
13
|
+
})
|
|
14
|
+
})
|
|
15
|
+
})
|
|
@@ -4,24 +4,22 @@ import type {
|
|
|
4
4
|
PikkuWorkerConfig,
|
|
5
5
|
QueueConfigMapping,
|
|
6
6
|
ConfigValidationResult,
|
|
7
|
+
QueueJob,
|
|
7
8
|
} from '@pikku/core/queue'
|
|
8
9
|
import {
|
|
9
|
-
runQueueJob,
|
|
10
10
|
registerQueueWorkers,
|
|
11
11
|
QueueJobFailedError,
|
|
12
12
|
QueueJobDiscardedError,
|
|
13
13
|
} from '@pikku/core/queue'
|
|
14
|
-
import {
|
|
15
|
-
CoreServices,
|
|
16
|
-
CoreSingletonServices,
|
|
17
|
-
CreateWireServices,
|
|
18
|
-
} from '@pikku/core'
|
|
14
|
+
import type { Logger } from '@pikku/core/services'
|
|
19
15
|
import { mapPgBossJobToQueueJob } from './utils.js'
|
|
20
16
|
|
|
21
17
|
export const mapPikkuWorkerToPgBoss = (
|
|
22
18
|
workerConfig?: PikkuWorkerConfig
|
|
23
19
|
): WorkOptions => {
|
|
24
|
-
const workerOptions: WorkOptions = {
|
|
20
|
+
const workerOptions: WorkOptions = {
|
|
21
|
+
batchSize: 10,
|
|
22
|
+
}
|
|
25
23
|
|
|
26
24
|
if (workerConfig?.batchSize !== undefined) {
|
|
27
25
|
workerOptions.batchSize = workerConfig.batchSize
|
|
@@ -53,7 +51,8 @@ export class PgBossQueueWorkers implements QueueWorkers {
|
|
|
53
51
|
supported: {
|
|
54
52
|
batchSize: {
|
|
55
53
|
queueProperty: 'batchSize',
|
|
56
|
-
description:
|
|
54
|
+
description:
|
|
55
|
+
'Number of jobs to fetch and process concurrently in parallel',
|
|
57
56
|
},
|
|
58
57
|
pollInterval: {
|
|
59
58
|
queueProperty: 'pollingIntervalSeconds',
|
|
@@ -121,63 +120,86 @@ export class PgBossQueueWorkers implements QueueWorkers {
|
|
|
121
120
|
|
|
122
121
|
private pgBoss: PgBoss
|
|
123
122
|
private activeWorkers = new Map<string, string>()
|
|
123
|
+
private runJob?:
|
|
124
|
+
| ((params: {
|
|
125
|
+
job: QueueJob
|
|
126
|
+
updateProgress?: (progress: number | string | object) => Promise<void>
|
|
127
|
+
}) => Promise<void>)
|
|
128
|
+
| undefined
|
|
129
|
+
private logger?: Logger
|
|
124
130
|
|
|
125
|
-
constructor(
|
|
126
|
-
pgBoss: PgBoss,
|
|
127
|
-
private singletonServices: CoreSingletonServices,
|
|
128
|
-
private createWireServices?: CreateWireServices<
|
|
129
|
-
CoreSingletonServices,
|
|
130
|
-
CoreServices,
|
|
131
|
-
any
|
|
132
|
-
>
|
|
133
|
-
) {
|
|
131
|
+
constructor(pgBoss: PgBoss) {
|
|
134
132
|
this.pgBoss = pgBoss
|
|
135
133
|
}
|
|
136
134
|
|
|
135
|
+
setJobRunner(
|
|
136
|
+
runJob: (params: {
|
|
137
|
+
job: QueueJob
|
|
138
|
+
updateProgress?: (progress: number | string | object) => Promise<void>
|
|
139
|
+
}) => Promise<void>,
|
|
140
|
+
logger: Logger
|
|
141
|
+
): void {
|
|
142
|
+
this.runJob = runJob
|
|
143
|
+
this.logger = logger
|
|
144
|
+
}
|
|
145
|
+
|
|
137
146
|
/**
|
|
138
147
|
* Scan state and register all compatible processors
|
|
139
148
|
*/
|
|
140
149
|
async registerQueues(): Promise<Record<string, ConfigValidationResult[]>> {
|
|
150
|
+
if (!this.runJob || !this.logger) {
|
|
151
|
+
throw new Error(
|
|
152
|
+
'PgBossQueueWorkers requires setJobRunner() before registerQueues()'
|
|
153
|
+
)
|
|
154
|
+
}
|
|
155
|
+
|
|
141
156
|
return await registerQueueWorkers(
|
|
142
157
|
this.configMappings,
|
|
143
|
-
this.
|
|
158
|
+
this.logger,
|
|
144
159
|
async (queueName, processor) => {
|
|
145
160
|
await this.pgBoss.createQueue(queueName)
|
|
146
161
|
const workerId = await this.pgBoss.work<any>(
|
|
147
|
-
processor.
|
|
162
|
+
processor.name,
|
|
148
163
|
{
|
|
149
164
|
...mapPikkuWorkerToPgBoss(processor.config),
|
|
150
165
|
includeMetadata: true,
|
|
151
166
|
},
|
|
152
|
-
async (
|
|
153
|
-
if (!
|
|
154
|
-
|
|
155
|
-
}
|
|
156
|
-
if (!job) {
|
|
157
|
-
this.singletonServices.logger.warn(
|
|
158
|
-
`No job received for queue ${queueName}`
|
|
159
|
-
)
|
|
167
|
+
async (jobs) => {
|
|
168
|
+
if (!jobs || jobs.length === 0) {
|
|
169
|
+
this.logger!.warn(`No jobs received for queue ${queueName}`)
|
|
160
170
|
return
|
|
161
171
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
172
|
+
// Process all jobs in parallel
|
|
173
|
+
await Promise.all(
|
|
174
|
+
jobs.map(async (job) => {
|
|
175
|
+
try {
|
|
176
|
+
await this.runJob!({
|
|
177
|
+
job: mapPgBossJobToQueueJob(job, this.pgBoss),
|
|
178
|
+
})
|
|
179
|
+
} catch (error: unknown) {
|
|
180
|
+
if (error instanceof QueueJobFailedError) {
|
|
181
|
+
// Mark this specific job as failed
|
|
182
|
+
await this.pgBoss.fail(queueName, job.id, {
|
|
183
|
+
message: error.message,
|
|
184
|
+
})
|
|
185
|
+
} else if (error instanceof QueueJobDiscardedError) {
|
|
186
|
+
// For pg-boss, complete the job successfully to discard it
|
|
187
|
+
this.logger!.info(
|
|
188
|
+
`PgBoss job ${job.id} discarded: ${error.message}`
|
|
189
|
+
)
|
|
190
|
+
await this.pgBoss.complete(queueName, job.id)
|
|
191
|
+
} else {
|
|
192
|
+
// For other errors, fail the job
|
|
193
|
+
await this.pgBoss.fail(queueName, job.id, {
|
|
194
|
+
message:
|
|
195
|
+
error instanceof Error
|
|
196
|
+
? error.message
|
|
197
|
+
: 'Unknown error',
|
|
198
|
+
})
|
|
199
|
+
}
|
|
200
|
+
}
|
|
167
201
|
})
|
|
168
|
-
|
|
169
|
-
if (error instanceof QueueJobFailedError) {
|
|
170
|
-
// Let pg-boss handle this as a failed job
|
|
171
|
-
throw new Error(error.message)
|
|
172
|
-
} else if (error instanceof QueueJobDiscardedError) {
|
|
173
|
-
// For pg-boss, complete the job successfully to discard it
|
|
174
|
-
this.singletonServices.logger.info(
|
|
175
|
-
`PgBoss job ${job.id} discarded: ${error.message}`
|
|
176
|
-
)
|
|
177
|
-
return // Successfully "completed" by discarding
|
|
178
|
-
}
|
|
179
|
-
throw error
|
|
180
|
-
}
|
|
202
|
+
)
|
|
181
203
|
}
|
|
182
204
|
)
|
|
183
205
|
this.activeWorkers.set(queueName, workerId)
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import { PgBoss } from 'pg-boss'
|
|
2
2
|
import {
|
|
3
3
|
SchedulerService,
|
|
4
|
+
SchedulerRuntimeHandlers,
|
|
4
5
|
ScheduledTaskInfo,
|
|
5
6
|
ScheduledTaskSummary,
|
|
6
7
|
CoreUserSession,
|
|
7
8
|
parseDurationString,
|
|
8
9
|
} from '@pikku/core'
|
|
10
|
+
import { getScheduledTasks } from '@pikku/core/scheduler'
|
|
9
11
|
|
|
10
12
|
/**
|
|
11
13
|
* Data stored in scheduled job
|
|
@@ -18,13 +20,26 @@ interface ScheduledJobData {
|
|
|
18
20
|
|
|
19
21
|
/**
|
|
20
22
|
* pg-boss scheduler service implementation
|
|
21
|
-
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL
|
|
23
|
+
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
|
|
24
|
+
* Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
|
|
22
25
|
*/
|
|
23
26
|
export class PgBossSchedulerService extends SchedulerService {
|
|
27
|
+
private scheduledCronNames: string[] = []
|
|
28
|
+
private handlers?: SchedulerRuntimeHandlers
|
|
29
|
+
|
|
24
30
|
constructor(private pgBoss: PgBoss) {
|
|
25
31
|
super()
|
|
26
32
|
}
|
|
27
33
|
|
|
34
|
+
/**
|
|
35
|
+
* Set services needed for processing recurring tasks.
|
|
36
|
+
* Must be called before start() since the scheduler is typically
|
|
37
|
+
* created before singletonServices are available.
|
|
38
|
+
*/
|
|
39
|
+
setServices(handlers: SchedulerRuntimeHandlers): void {
|
|
40
|
+
this.handlers = handlers
|
|
41
|
+
}
|
|
42
|
+
|
|
28
43
|
/**
|
|
29
44
|
* Initialize - no-op as pg-boss is already initialized by factory
|
|
30
45
|
*/
|
|
@@ -118,4 +133,45 @@ export class PgBossSchedulerService extends SchedulerService {
|
|
|
118
133
|
async close(): Promise<void> {
|
|
119
134
|
// No-op - pg-boss lifecycle is managed by factory
|
|
120
135
|
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Start recurring scheduled tasks.
|
|
139
|
+
* Registers a pg-boss worker to process recurring jobs via runScheduledTask.
|
|
140
|
+
*/
|
|
141
|
+
async start(): Promise<void> {
|
|
142
|
+
if (!this.handlers) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
'PgBossSchedulerService requires setServices() before start()'
|
|
145
|
+
)
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const scheduledTasks = getScheduledTasks()
|
|
149
|
+
|
|
150
|
+
for (const [name, task] of scheduledTasks) {
|
|
151
|
+
const cronName = `pikku-recurring-scheduled-task:${name}`
|
|
152
|
+
await this.pgBoss.schedule(cronName, task.schedule, {
|
|
153
|
+
rpcName: name,
|
|
154
|
+
} as ScheduledJobData)
|
|
155
|
+
this.scheduledCronNames.push(cronName)
|
|
156
|
+
|
|
157
|
+
// Register a worker to process the scheduled jobs
|
|
158
|
+
await this.pgBoss.work<ScheduledJobData>(cronName, async (jobs) => {
|
|
159
|
+
for (const job of jobs) {
|
|
160
|
+
const { rpcName } = job.data
|
|
161
|
+
this.handlers!.logger.info(`Running scheduled task: ${rpcName}`)
|
|
162
|
+
await this.handlers!.runScheduledTask(rpcName)
|
|
163
|
+
}
|
|
164
|
+
})
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Stop recurring scheduled tasks by unscheduling them from pg-boss.
|
|
170
|
+
*/
|
|
171
|
+
async stop(): Promise<void> {
|
|
172
|
+
for (const cronName of this.scheduledCronNames) {
|
|
173
|
+
await this.pgBoss.unschedule(cronName)
|
|
174
|
+
}
|
|
175
|
+
this.scheduledCronNames = []
|
|
176
|
+
}
|
|
121
177
|
}
|
|
@@ -2,11 +2,9 @@ import { PgBoss, type ConstructorOptions } from 'pg-boss'
|
|
|
2
2
|
import { PgBossQueueService } from './pg-boss-queue-service.js'
|
|
3
3
|
import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
|
|
4
4
|
import { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
|
|
5
|
-
import type {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
CreateWireServices,
|
|
9
|
-
} from '@pikku/core'
|
|
5
|
+
import type { CoreSingletonServices, CreateWireServices } from '@pikku/core'
|
|
6
|
+
import { createQueueJobRunner } from '@pikku/core/queue'
|
|
7
|
+
import { createSchedulerRuntimeHandlers } from '@pikku/core/scheduler'
|
|
10
8
|
|
|
11
9
|
/**
|
|
12
10
|
* Factory class for pg-boss services
|
|
@@ -56,19 +54,15 @@ export class PgBossServiceFactory {
|
|
|
56
54
|
*/
|
|
57
55
|
getQueueWorkers(
|
|
58
56
|
singletonServices: CoreSingletonServices,
|
|
59
|
-
createWireServices?: CreateWireServices
|
|
60
|
-
CoreSingletonServices,
|
|
61
|
-
CoreServices,
|
|
62
|
-
any
|
|
63
|
-
>
|
|
57
|
+
createWireServices?: CreateWireServices
|
|
64
58
|
): PgBossQueueWorkers {
|
|
65
59
|
if (!this.queueWorkers) {
|
|
66
|
-
this.queueWorkers = new PgBossQueueWorkers(
|
|
67
|
-
this.pgBoss,
|
|
68
|
-
singletonServices,
|
|
69
|
-
createWireServices
|
|
70
|
-
)
|
|
60
|
+
this.queueWorkers = new PgBossQueueWorkers(this.pgBoss)
|
|
71
61
|
}
|
|
62
|
+
this.queueWorkers.setJobRunner(
|
|
63
|
+
createQueueJobRunner({ singletonServices, createWireServices }),
|
|
64
|
+
singletonServices.logger
|
|
65
|
+
)
|
|
72
66
|
return this.queueWorkers
|
|
73
67
|
}
|
|
74
68
|
|
|
@@ -82,6 +76,18 @@ export class PgBossServiceFactory {
|
|
|
82
76
|
return this.schedulerService
|
|
83
77
|
}
|
|
84
78
|
|
|
79
|
+
setSchedulerRuntime(
|
|
80
|
+
singletonServices: CoreSingletonServices,
|
|
81
|
+
createWireServices?: CreateWireServices
|
|
82
|
+
): void {
|
|
83
|
+
this.getSchedulerService().setServices(
|
|
84
|
+
createSchedulerRuntimeHandlers({
|
|
85
|
+
singletonServices,
|
|
86
|
+
createWireServices,
|
|
87
|
+
})
|
|
88
|
+
)
|
|
89
|
+
}
|
|
90
|
+
|
|
85
91
|
/**
|
|
86
92
|
* Close all services and the pg-boss connection
|
|
87
93
|
*/
|
|
@@ -91,4 +97,11 @@ export class PgBossServiceFactory {
|
|
|
91
97
|
this.initialized = false
|
|
92
98
|
}
|
|
93
99
|
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Alias for close() - used by stopSingletonServices
|
|
103
|
+
*/
|
|
104
|
+
async stop(): Promise<void> {
|
|
105
|
+
await this.close()
|
|
106
|
+
}
|
|
94
107
|
}
|