@pikku/queue-pg-boss 0.12.10 → 0.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/dist/pg-boss-queue-service.d.ts +1 -1
- package/package.json +9 -7
- package/run-tests.sh +21 -9
- package/src/index.ts +0 -19
- package/src/pg-boss-queue-service.test.ts +0 -83
- package/src/pg-boss-queue-service.ts +0 -113
- package/src/pg-boss-queue-worker.test.ts +0 -93
- package/src/pg-boss-queue-worker.ts +0 -231
- package/src/pg-boss-scheduler-service.ts +0 -192
- package/src/pg-boss-service-factory.ts +0 -85
- package/src/utils.ts +0 -87
- package/tsconfig.json +0 -17
- package/tsconfig.tsbuildinfo +0 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,31 @@
|
|
|
1
|
+
## 0.12.11
|
|
2
|
+
|
|
3
|
+
### Patch Changes
|
|
4
|
+
|
|
5
|
+
- 4c7a1b5: Run the monorepo's own scripts through bun instead of yarn. What moves is the
|
|
6
|
+
package manager each package's `prepublishOnly` and build scripts invoke, plus
|
|
7
|
+
the two manifest fixes bun needs to resolve the tree: `uWebSockets.js` is
|
|
8
|
+
declared with an explicit `github:` specifier, and `@pikku/uws-handler` marks
|
|
9
|
+
its `uWebSockets.js` peer optional so a bun install of a consumer that brings
|
|
10
|
+
its own uWS app does not try to fetch it from the registry.
|
|
11
|
+
|
|
12
|
+
Three published behaviours change, all of them cases where an isolated
|
|
13
|
+
`node_modules` or bun as the runtime had been papered over by yarn's hoisting:
|
|
14
|
+
|
|
15
|
+
- `@pikku/migrator-sql` turns foreign keys on when it opens a sqlite database
|
|
16
|
+
through bun. `node:sqlite` enforces them by default and `bun:sqlite` does not,
|
|
17
|
+
which silently turned every `ON DELETE CASCADE` into a no-op under
|
|
18
|
+
`bunx --bun pikku`.
|
|
19
|
+
- `@pikku/cli` resolves a deploy provider against the project being deployed
|
|
20
|
+
rather than against wherever the CLI itself is installed, which is what its
|
|
21
|
+
own "is not installed" error asks the user to arrange.
|
|
22
|
+
- `@pikku/cli` treats a specifier a runtime hands straight back — bun does this
|
|
23
|
+
for the modules it implements itself — as not resolved from the project, so
|
|
24
|
+
it falls back rather than loading the runtime's own copy.
|
|
25
|
+
|
|
26
|
+
- Updated dependencies [4c7a1b5]
|
|
27
|
+
- @pikku/core@0.12.108
|
|
28
|
+
|
|
1
29
|
## 0.12.10
|
|
2
30
|
|
|
3
31
|
### Patch Changes
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { PgBoss, Request as PgBossRequest } from 'pg-boss';
|
|
2
2
|
import type { QueueJob, QueueService } from '@pikku/core/queue';
|
|
3
3
|
import type { JobOptions } from '@pikku/core/queue';
|
|
4
|
-
export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBossRequest[
|
|
4
|
+
export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBossRequest['options'];
|
|
5
5
|
/**
|
|
6
6
|
* pg-boss/PostgreSQL queue service implementation
|
|
7
7
|
* Supports job results, retries, and PostgreSQL-based queuing
|
package/package.json
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pikku/queue-pg-boss",
|
|
3
|
-
"version": "0.12.
|
|
3
|
+
"version": "0.12.11",
|
|
4
|
+
"repository": {
|
|
5
|
+
"type": "git",
|
|
6
|
+
"url": "git+https://github.com/pikkujs/pikku.git",
|
|
7
|
+
"directory": "packages/services/queue-pg-boss"
|
|
8
|
+
},
|
|
4
9
|
"description": "pg-boss queue, worker and scheduler services for Pikku, backed by Postgres",
|
|
5
10
|
"author": "yasser.fadl@gmail.com",
|
|
6
11
|
"license": "MIT",
|
|
@@ -13,18 +18,15 @@
|
|
|
13
18
|
"ncu": "npx npm-check-updates",
|
|
14
19
|
"build": "tsc -b",
|
|
15
20
|
"release": "npm run build && npm test",
|
|
16
|
-
"prepublishOnly": "
|
|
21
|
+
"prepublishOnly": "bun run build"
|
|
17
22
|
},
|
|
18
23
|
"peerDependencies": {
|
|
19
|
-
"@pikku/core": "^0.12.
|
|
24
|
+
"@pikku/core": "^0.12.108"
|
|
20
25
|
},
|
|
21
26
|
"dependencies": {
|
|
22
27
|
"pg-boss": "^12.27.0"
|
|
23
28
|
},
|
|
24
|
-
"devDependencies": {
|
|
25
|
-
"typescript": "^6.0.3"
|
|
26
|
-
},
|
|
27
29
|
"exports": {
|
|
28
30
|
".": "./dist/index.js"
|
|
29
31
|
}
|
|
30
|
-
}
|
|
32
|
+
}
|
package/run-tests.sh
CHANGED
|
@@ -1,8 +1,5 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
2
|
|
|
3
|
-
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
4
|
-
cd "$script_dir"
|
|
5
|
-
|
|
6
3
|
# Enable nullglob to handle cases where no files match the pattern
|
|
7
4
|
shopt -s nullglob
|
|
8
5
|
|
|
@@ -37,20 +34,35 @@ files=($(find src -type f -name "*.test.ts"))
|
|
|
37
34
|
# Check if any files matched the pattern
|
|
38
35
|
if [ ${#files[@]} -eq 0 ]; then
|
|
39
36
|
echo "No test files found matching pattern: $pattern"
|
|
37
|
+
if [ "${STRICT_TEST_DISCOVERY}" = "1" ]; then
|
|
38
|
+
exit 1
|
|
39
|
+
fi
|
|
40
40
|
exit 0
|
|
41
41
|
fi
|
|
42
42
|
|
|
43
|
-
# Construct the
|
|
44
|
-
|
|
43
|
+
# Construct the bun command
|
|
44
|
+
#
|
|
45
|
+
# --parallel gives every test file its own worker process, which is what node's
|
|
46
|
+
# `--test` did. Several suites keep module-level caches (a TypeScript program, a
|
|
47
|
+
# temp fixture directory) that a shared process would leak between files.
|
|
48
|
+
#
|
|
49
|
+
# --timeout because bun caps a test at 5s by default where node had no cap, and
|
|
50
|
+
# the suites that build real TypeScript programs run well past that.
|
|
51
|
+
bun_cmd=(bun test --parallel --timeout 120000)
|
|
45
52
|
|
|
46
53
|
# Append options based on flags
|
|
47
54
|
if [ "$watch_mode" = true ]; then
|
|
48
|
-
|
|
55
|
+
# --watch reruns in one process; parallel workers have nothing to watch.
|
|
56
|
+
bun_cmd=(bun test --watch --timeout 120000)
|
|
49
57
|
fi
|
|
50
58
|
|
|
51
59
|
if [ "$coverage_mode" = true ]; then
|
|
52
|
-
|
|
60
|
+
# --coverage-dir=. keeps lcov.info at the package root: CI merges every
|
|
61
|
+
# package's report by prefixing its SF paths with the file's own directory,
|
|
62
|
+
# so a nested coverage/ directory would re-root them one level too deep.
|
|
63
|
+
bun_cmd+=(--coverage --coverage-reporter=lcov --coverage-reporter=text --coverage-dir=.)
|
|
64
|
+
export PIKKU_TEST_COVERAGE=1
|
|
53
65
|
fi
|
|
54
66
|
|
|
55
|
-
# Execute the
|
|
56
|
-
"${
|
|
67
|
+
# Execute the bun command with the expanded list of files
|
|
68
|
+
"${bun_cmd[@]}" "${files[@]}"
|
package/src/index.ts
DELETED
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module @pikku/queue-pg-boss
|
|
3
|
-
*
|
|
4
|
-
* PostgreSQL queue service for Pikku using pg-boss
|
|
5
|
-
* Provides job results, retries, priorities, and PostgreSQL-based queuing
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export { PgBossServiceFactory } from './pg-boss-service-factory.js'
|
|
9
|
-
export { PgBossQueueService } from './pg-boss-queue-service.js'
|
|
10
|
-
export { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
|
|
11
|
-
export { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
|
|
12
|
-
|
|
13
|
-
// Re-export core queue types for convenience
|
|
14
|
-
export type { QueueJob, QueueService } from '@pikku/core/queue'
|
|
15
|
-
export type {
|
|
16
|
-
PikkuWorkerConfig,
|
|
17
|
-
PikkuJobConfig,
|
|
18
|
-
JobOptions,
|
|
19
|
-
} from '@pikku/core/queue'
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
import { describe, test } from 'node:test'
|
|
2
|
-
import assert from 'node:assert/strict'
|
|
3
|
-
|
|
4
|
-
import { mapPikkuJobToPgBoss } from './pg-boss-queue-service.js'
|
|
5
|
-
|
|
6
|
-
// These guard the contract the workflow engine relies on: a step's resolved
|
|
7
|
-
// retry policy (attempts + backoff) must survive translation into pg-boss
|
|
8
|
-
// terms, so "workflow owns retries" actually reaches the queue.
|
|
9
|
-
describe('mapPikkuJobToPgBoss — retry mapping', () => {
|
|
10
|
-
test('attempts → retryLimit (attempts - 1)', () => {
|
|
11
|
-
// Default policy: 5 retries → 6 attempts → retryLimit 5.
|
|
12
|
-
assert.equal(mapPikkuJobToPgBoss({ attempts: 6 }).retryLimit, 5)
|
|
13
|
-
})
|
|
14
|
-
|
|
15
|
-
test('attempts: 1 (retries: 0) → retryLimit 0 (pg-boss never re-runs it)', () => {
|
|
16
|
-
assert.equal(mapPikkuJobToPgBoss({ attempts: 1 }).retryLimit, 0)
|
|
17
|
-
})
|
|
18
|
-
|
|
19
|
-
test("backoff 'exponential' → retryBackoff true + 1s base delay", () => {
|
|
20
|
-
const opts = mapPikkuJobToPgBoss({ attempts: 6, backoff: 'exponential' })
|
|
21
|
-
assert.equal(opts.retryBackoff, true)
|
|
22
|
-
// pg-boss multiplies retry_delay by 2^n and the queue default is 0 —
|
|
23
|
-
// without a base, "exponential" retries all fire immediately.
|
|
24
|
-
assert.equal(opts.retryDelay, 1)
|
|
25
|
-
})
|
|
26
|
-
|
|
27
|
-
test('sub-second fixed delay rounds up to 1s, not down to immediate', () => {
|
|
28
|
-
const opts = mapPikkuJobToPgBoss({
|
|
29
|
-
attempts: 4,
|
|
30
|
-
backoff: { type: 'fixed', delay: 300 },
|
|
31
|
-
})
|
|
32
|
-
assert.equal(opts.retryDelay, 1)
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
test('fixed backoff → retryBackoff false + retryDelay in seconds', () => {
|
|
36
|
-
const opts = mapPikkuJobToPgBoss({
|
|
37
|
-
attempts: 4,
|
|
38
|
-
backoff: { type: 'fixed', delay: 5000 },
|
|
39
|
-
})
|
|
40
|
-
assert.equal(opts.retryBackoff, false)
|
|
41
|
-
// pg-boss takes seconds; our delay is milliseconds.
|
|
42
|
-
assert.equal(opts.retryDelay, 5)
|
|
43
|
-
})
|
|
44
|
-
|
|
45
|
-
test('exponential object backoff → retryBackoff true + retryDelay in seconds', () => {
|
|
46
|
-
const opts = mapPikkuJobToPgBoss({
|
|
47
|
-
attempts: 4,
|
|
48
|
-
backoff: { type: 'exponential', delay: 2000 },
|
|
49
|
-
})
|
|
50
|
-
assert.equal(opts.retryBackoff, true)
|
|
51
|
-
assert.equal(opts.retryDelay, 2)
|
|
52
|
-
})
|
|
53
|
-
|
|
54
|
-
test('no options → no retry fields set', () => {
|
|
55
|
-
const opts = mapPikkuJobToPgBoss(undefined)
|
|
56
|
-
assert.equal(opts.retryLimit, undefined)
|
|
57
|
-
assert.equal(opts.retryBackoff, undefined)
|
|
58
|
-
})
|
|
59
|
-
})
|
|
60
|
-
|
|
61
|
-
describe('mapPikkuJobToPgBoss — group mapping', () => {
|
|
62
|
-
test('group id is forwarded so the worker can enforce a per-group limit', () => {
|
|
63
|
-
assert.deepEqual(
|
|
64
|
-
mapPikkuJobToPgBoss({ group: { id: 'deployWorkflow' } }).group,
|
|
65
|
-
{ id: 'deployWorkflow' }
|
|
66
|
-
)
|
|
67
|
-
})
|
|
68
|
-
|
|
69
|
-
test('tier is only set when provided', () => {
|
|
70
|
-
const opts = mapPikkuJobToPgBoss({
|
|
71
|
-
group: { id: 'deployWorkflow', tier: 'slow' },
|
|
72
|
-
})
|
|
73
|
-
assert.deepEqual(opts.group, { id: 'deployWorkflow', tier: 'slow' })
|
|
74
|
-
assert.equal(
|
|
75
|
-
'tier' in mapPikkuJobToPgBoss({ group: { id: 'a' } }).group!,
|
|
76
|
-
false
|
|
77
|
-
)
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
test('no group → no group field', () => {
|
|
81
|
-
assert.equal(mapPikkuJobToPgBoss({ attempts: 2 }).group, undefined)
|
|
82
|
-
})
|
|
83
|
-
})
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import type { PgBoss, Request as PgBossRequest } from 'pg-boss'
|
|
2
|
-
import type { QueueJob, QueueService } from '@pikku/core/queue'
|
|
3
|
-
import type { JobOptions } from '@pikku/core/queue'
|
|
4
|
-
import { mapPgBossJobToQueueJob } from './utils.js'
|
|
5
|
-
|
|
6
|
-
export const mapPikkuJobToPgBoss = (
|
|
7
|
-
options?: JobOptions
|
|
8
|
-
): PgBossRequest['options'] => {
|
|
9
|
-
const pgBossOptions: any = {}
|
|
10
|
-
|
|
11
|
-
if (options?.priority !== undefined) {
|
|
12
|
-
pgBossOptions.priority = options.priority
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
if (options?.delay !== undefined) {
|
|
16
|
-
pgBossOptions.startAfter = new Date(Date.now() + options.delay)
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
if (options?.jobId !== undefined) {
|
|
20
|
-
pgBossOptions.singletonKey = options.jobId
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
if (options?.group !== undefined) {
|
|
24
|
-
pgBossOptions.group = {
|
|
25
|
-
id: options.group.id,
|
|
26
|
-
...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// Map retry options
|
|
31
|
-
if (options?.attempts !== undefined) {
|
|
32
|
-
pgBossOptions.retryLimit = options.attempts - 1
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if (options?.backoff !== undefined) {
|
|
36
|
-
const type =
|
|
37
|
-
typeof options.backoff === 'string'
|
|
38
|
-
? options.backoff
|
|
39
|
-
: options.backoff.type
|
|
40
|
-
const delay =
|
|
41
|
-
typeof options.backoff === 'object' ? options.backoff.delay : undefined
|
|
42
|
-
pgBossOptions.retryBackoff = type === 'exponential'
|
|
43
|
-
if (delay !== undefined) {
|
|
44
|
-
// pg-boss uses seconds, we use milliseconds — never round a sub-second
|
|
45
|
-
// delay down to 0, which pg-boss treats as retry-immediately
|
|
46
|
-
pgBossOptions.retryDelay = Math.max(1, Math.round(delay / 1000))
|
|
47
|
-
} else if (type === 'exponential') {
|
|
48
|
-
// pg-boss computes exponential backoff as retry_delay * 2^n, and the
|
|
49
|
-
// queue-level retry_delay defaults to 0 — without an explicit base every
|
|
50
|
-
// "exponential" retry fires immediately. 1s base → ~1s, 2s, 4s, 8s…
|
|
51
|
-
pgBossOptions.retryDelay = 1
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
if (options?.removeOnComplete !== undefined) {
|
|
56
|
-
pgBossOptions.onComplete = options.removeOnComplete === 0
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
if (options?.removeOnFail !== undefined) {
|
|
60
|
-
// pg-boss doesn't have a direct equivalent, but we can track this for consistency
|
|
61
|
-
// Note: pg-boss keeps failed jobs by default
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
return pgBossOptions as PgBossRequest['options']
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
/**
|
|
68
|
-
* pg-boss/PostgreSQL queue service implementation
|
|
69
|
-
* Supports job results, retries, and PostgreSQL-based queuing
|
|
70
|
-
*/
|
|
71
|
-
export class PgBossQueueService implements QueueService {
|
|
72
|
-
readonly name = 'pg-boss'
|
|
73
|
-
readonly supportsResults = true
|
|
74
|
-
protected pgBoss: PgBoss
|
|
75
|
-
|
|
76
|
-
constructor(pgBoss: PgBoss) {
|
|
77
|
-
this.pgBoss = pgBoss
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Add a job to the queue
|
|
82
|
-
*/
|
|
83
|
-
public async add<In>(
|
|
84
|
-
queueName: string,
|
|
85
|
-
data: In,
|
|
86
|
-
options?: JobOptions
|
|
87
|
-
): Promise<string> {
|
|
88
|
-
await this.pgBoss.createQueue(queueName)
|
|
89
|
-
const jobId = await this.pgBoss.send(
|
|
90
|
-
queueName,
|
|
91
|
-
data as any,
|
|
92
|
-
mapPikkuJobToPgBoss(options)
|
|
93
|
-
)
|
|
94
|
-
if (!jobId) {
|
|
95
|
-
throw new Error('Failed to create job')
|
|
96
|
-
}
|
|
97
|
-
return jobId
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Get job by ID
|
|
102
|
-
*/
|
|
103
|
-
async getJob<T, R>(
|
|
104
|
-
queueName: string,
|
|
105
|
-
jobId: string
|
|
106
|
-
): Promise<QueueJob<T, R> | null> {
|
|
107
|
-
const job = await this.pgBoss.getJobById<T>(queueName, jobId)
|
|
108
|
-
if (!job) {
|
|
109
|
-
return null
|
|
110
|
-
}
|
|
111
|
-
return mapPgBossJobToQueueJob<T, R>(job, this.pgBoss)
|
|
112
|
-
}
|
|
113
|
-
}
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
import { describe, test } from 'node:test'
|
|
2
|
-
import assert from 'node:assert/strict'
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
PgBossQueueWorkers,
|
|
6
|
-
mapPikkuWorkerToPgBoss,
|
|
7
|
-
} from './pg-boss-queue-worker.js'
|
|
8
|
-
|
|
9
|
-
describe('PgBossQueueWorkers', () => {
|
|
10
|
-
test('requires a logger (explicit or from singleton services)', async () => {
|
|
11
|
-
const workers = new PgBossQueueWorkers({} as any)
|
|
12
|
-
|
|
13
|
-
await assert.rejects(() => workers.registerQueues(), {
|
|
14
|
-
message:
|
|
15
|
-
'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first',
|
|
16
|
-
})
|
|
17
|
-
})
|
|
18
|
-
})
|
|
19
|
-
|
|
20
|
-
describe('mapPikkuWorkerToPgBoss', () => {
|
|
21
|
-
test('parallelism maps to independent workers (localConcurrency), never batch fetching', () => {
|
|
22
|
-
// batchSize: 1 keeps each worker single-job so a slow job can never hold its
|
|
23
|
-
// batch siblings — head-of-line blocking is impossible.
|
|
24
|
-
assert.deepEqual(mapPikkuWorkerToPgBoss(), {
|
|
25
|
-
localConcurrency: 10,
|
|
26
|
-
batchSize: 1,
|
|
27
|
-
})
|
|
28
|
-
|
|
29
|
-
assert.deepEqual(mapPikkuWorkerToPgBoss({ batchSize: 25 }), {
|
|
30
|
-
localConcurrency: 25,
|
|
31
|
-
batchSize: 1,
|
|
32
|
-
})
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
test('pollInterval is converted from ms to seconds', () => {
|
|
36
|
-
assert.deepEqual(mapPikkuWorkerToPgBoss({ pollInterval: 3000 }), {
|
|
37
|
-
localConcurrency: 10,
|
|
38
|
-
batchSize: 1,
|
|
39
|
-
pollingIntervalSeconds: 3,
|
|
40
|
-
})
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
test('groupConcurrency caps how much of the worker one group can take', () => {
|
|
44
|
-
assert.deepEqual(mapPikkuWorkerToPgBoss({ groupConcurrency: 2 }), {
|
|
45
|
-
localConcurrency: 10,
|
|
46
|
-
batchSize: 1,
|
|
47
|
-
localGroupConcurrency: 2,
|
|
48
|
-
})
|
|
49
|
-
})
|
|
50
|
-
|
|
51
|
-
test('tiered groupConcurrency passes tiers through', () => {
|
|
52
|
-
assert.deepEqual(
|
|
53
|
-
mapPikkuWorkerToPgBoss({
|
|
54
|
-
batchSize: 20,
|
|
55
|
-
groupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
|
|
56
|
-
}),
|
|
57
|
-
{
|
|
58
|
-
localConcurrency: 20,
|
|
59
|
-
batchSize: 1,
|
|
60
|
-
localGroupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
|
|
61
|
-
}
|
|
62
|
-
)
|
|
63
|
-
})
|
|
64
|
-
|
|
65
|
-
test('group limits are clamped to the worker concurrency', () => {
|
|
66
|
-
// pg-boss asserts every group limit is <= localConcurrency and throws at
|
|
67
|
-
// worker start otherwise — clamp so an over-eager config degrades to "this
|
|
68
|
-
// group may use the whole worker" instead of failing to start.
|
|
69
|
-
assert.deepEqual(
|
|
70
|
-
mapPikkuWorkerToPgBoss({
|
|
71
|
-
batchSize: 4,
|
|
72
|
-
groupConcurrency: { default: 10, tiers: { greedy: 99 } },
|
|
73
|
-
}),
|
|
74
|
-
{
|
|
75
|
-
localConcurrency: 4,
|
|
76
|
-
batchSize: 1,
|
|
77
|
-
localGroupConcurrency: { default: 4, tiers: { greedy: 4 } },
|
|
78
|
-
}
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
assert.deepEqual(
|
|
82
|
-
mapPikkuWorkerToPgBoss({ batchSize: 3, groupConcurrency: 8 }),
|
|
83
|
-
{ localConcurrency: 3, batchSize: 1, localGroupConcurrency: 3 }
|
|
84
|
-
)
|
|
85
|
-
})
|
|
86
|
-
|
|
87
|
-
test('no group config means no group limits', () => {
|
|
88
|
-
assert.equal(
|
|
89
|
-
'localGroupConcurrency' in mapPikkuWorkerToPgBoss({ batchSize: 5 }),
|
|
90
|
-
false
|
|
91
|
-
)
|
|
92
|
-
})
|
|
93
|
-
})
|
|
@@ -1,231 +0,0 @@
|
|
|
1
|
-
import type { PgBoss, WorkOptions } from 'pg-boss'
|
|
2
|
-
import type {
|
|
3
|
-
QueueWorkers,
|
|
4
|
-
PikkuWorkerConfig,
|
|
5
|
-
QueueConfigMapping,
|
|
6
|
-
ConfigValidationResult,
|
|
7
|
-
} from '@pikku/core/queue'
|
|
8
|
-
import { runQueueJob } from '@pikku/core/queue'
|
|
9
|
-
import type { Logger } from '@pikku/core/services'
|
|
10
|
-
import {
|
|
11
|
-
registerQueueWorkers,
|
|
12
|
-
QueueJobFailedError,
|
|
13
|
-
QueueJobDiscardedError,
|
|
14
|
-
} from '@pikku/core/queue'
|
|
15
|
-
import { pikkuState } from '@pikku/core/state'
|
|
16
|
-
import { mapPgBossJobToQueueJob } from './utils.js'
|
|
17
|
-
|
|
18
|
-
export const mapPikkuWorkerToPgBoss = (
|
|
19
|
-
workerConfig?: PikkuWorkerConfig
|
|
20
|
-
): WorkOptions => {
|
|
21
|
-
// Map the requested parallelism to pg-boss `localConcurrency` (independent
|
|
22
|
-
// per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
|
|
23
|
-
// batch to one handler and only fetches again once that handler resolves — so
|
|
24
|
-
// a single slow job blocks every sibling in its batch AND stalls the next
|
|
25
|
-
// fetch. On a shared queue that is head-of-line blocking: one long job can
|
|
26
|
-
// freeze the entire worker and starve everything queued behind it.
|
|
27
|
-
// `localConcurrency` spawns N pollers that each fetch+process a single job and
|
|
28
|
-
// refill the instant they finish, so a slow job never holds up the others.
|
|
29
|
-
const localConcurrency = workerConfig?.batchSize ?? 10
|
|
30
|
-
const workerOptions: WorkOptions = {
|
|
31
|
-
localConcurrency,
|
|
32
|
-
batchSize: 1,
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
if (workerConfig?.pollInterval !== undefined) {
|
|
36
|
-
workerOptions.pollingIntervalSeconds = Math.round(
|
|
37
|
-
workerConfig.pollInterval / 1000
|
|
38
|
-
)
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
|
|
42
|
-
// already at capacity are excluded from the fetch itself, so an at-capacity
|
|
43
|
-
// group costs nothing rather than being fetched and restored. pg-boss asserts
|
|
44
|
-
// every limit is <= localConcurrency, so clamp instead of letting a config
|
|
45
|
-
// that reads as "let this tier use more" throw at worker start.
|
|
46
|
-
if (workerConfig?.groupConcurrency !== undefined) {
|
|
47
|
-
const clamp = (limit: number) => Math.min(limit, localConcurrency)
|
|
48
|
-
workerOptions.localGroupConcurrency =
|
|
49
|
-
typeof workerConfig.groupConcurrency === 'number'
|
|
50
|
-
? clamp(workerConfig.groupConcurrency)
|
|
51
|
-
: {
|
|
52
|
-
default: clamp(workerConfig.groupConcurrency.default),
|
|
53
|
-
...(workerConfig.groupConcurrency.tiers
|
|
54
|
-
? {
|
|
55
|
-
tiers: Object.fromEntries(
|
|
56
|
-
Object.entries(workerConfig.groupConcurrency.tiers).map(
|
|
57
|
-
([tier, limit]) => [tier, clamp(limit)]
|
|
58
|
-
)
|
|
59
|
-
),
|
|
60
|
-
}
|
|
61
|
-
: {}),
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
return workerOptions
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
/**
|
|
69
|
-
* pg-boss/PostgreSQL queue worker implementation
|
|
70
|
-
* Supports job results, retries, and PostgreSQL-based queuing
|
|
71
|
-
*/
|
|
72
|
-
export class PgBossQueueWorkers implements QueueWorkers {
|
|
73
|
-
readonly name = 'pg-boss'
|
|
74
|
-
readonly supportsResults = true
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* Configuration mapping rules for pg-boss
|
|
78
|
-
* This defines how Pikku worker configs map to pg-boss configurations
|
|
79
|
-
*/
|
|
80
|
-
readonly configMappings: QueueConfigMapping = {
|
|
81
|
-
// Configurations that are directly supported
|
|
82
|
-
supported: {
|
|
83
|
-
batchSize: {
|
|
84
|
-
queueProperty: 'localConcurrency',
|
|
85
|
-
description: 'Number of jobs to process concurrently',
|
|
86
|
-
},
|
|
87
|
-
groupConcurrency: {
|
|
88
|
-
queueProperty: 'localGroupConcurrency',
|
|
89
|
-
description:
|
|
90
|
-
'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
|
|
91
|
-
},
|
|
92
|
-
pollInterval: {
|
|
93
|
-
queueProperty: 'pollingIntervalSeconds',
|
|
94
|
-
transform: (value: number) => Math.round(value / 1000),
|
|
95
|
-
description:
|
|
96
|
-
'How often to poll for new jobs (converted from ms to seconds)',
|
|
97
|
-
},
|
|
98
|
-
},
|
|
99
|
-
|
|
100
|
-
// Configurations that are not supported but have reasonable explanations
|
|
101
|
-
unsupported: {
|
|
102
|
-
name: {
|
|
103
|
-
reason: 'Worker names are not supported in pg-boss',
|
|
104
|
-
explanation:
|
|
105
|
-
'pg-boss identifies workers by their queue name and process, not custom names',
|
|
106
|
-
},
|
|
107
|
-
autorun: {
|
|
108
|
-
reason: 'Autorun is not configurable in pg-boss (always enabled)',
|
|
109
|
-
explanation:
|
|
110
|
-
'pg-boss workers automatically start processing when created',
|
|
111
|
-
},
|
|
112
|
-
lockDuration: {
|
|
113
|
-
reason: 'Lock duration is managed differently in pg-boss',
|
|
114
|
-
explanation:
|
|
115
|
-
'pg-boss uses job-level expiration instead of worker-level locks',
|
|
116
|
-
},
|
|
117
|
-
drainDelay: {
|
|
118
|
-
reason: 'Drain delay is not configurable in pg-boss',
|
|
119
|
-
explanation: 'pg-boss handles graceful shutdown internally',
|
|
120
|
-
},
|
|
121
|
-
maxStalledCount: {
|
|
122
|
-
reason: 'Max stalled count is not configurable in pg-boss',
|
|
123
|
-
explanation:
|
|
124
|
-
'pg-boss handles stalled jobs through its built-in retry mechanism',
|
|
125
|
-
},
|
|
126
|
-
prefetch: {
|
|
127
|
-
reason: 'Prefetch is managed internally by pg-boss',
|
|
128
|
-
explanation:
|
|
129
|
-
'pg-boss optimizes job fetching automatically based on batch size',
|
|
130
|
-
},
|
|
131
|
-
visibilityTimeout: {
|
|
132
|
-
reason: "Visibility timeout concept doesn't apply to pg-boss",
|
|
133
|
-
explanation:
|
|
134
|
-
'pg-boss uses PostgreSQL locks and job expiration instead of visibility timeout',
|
|
135
|
-
},
|
|
136
|
-
},
|
|
137
|
-
|
|
138
|
-
// Configurations that have partial support or workarounds
|
|
139
|
-
fallbacks: {
|
|
140
|
-
removeOnComplete: {
|
|
141
|
-
reason: 'Job retention is managed through pg-boss archival system',
|
|
142
|
-
explanation:
|
|
143
|
-
'pg-boss automatically archives completed jobs based on database settings',
|
|
144
|
-
fallbackValue: 'Managed by pg-boss database archival',
|
|
145
|
-
},
|
|
146
|
-
removeOnFail: {
|
|
147
|
-
reason:
|
|
148
|
-
'Failed job retention is managed through pg-boss archival system',
|
|
149
|
-
explanation:
|
|
150
|
-
'pg-boss automatically archives failed jobs based on database settings',
|
|
151
|
-
fallbackValue: 'Managed by pg-boss database archival',
|
|
152
|
-
},
|
|
153
|
-
},
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
private pgBoss: PgBoss
|
|
157
|
-
private activeWorkers = new Map<string, string>()
|
|
158
|
-
|
|
159
|
-
constructor(pgBoss: PgBoss) {
|
|
160
|
-
this.pgBoss = pgBoss
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Scan state and register all compatible processors
|
|
165
|
-
*/
|
|
166
|
-
async registerQueues(
|
|
167
|
-
logger?: Logger
|
|
168
|
-
): Promise<Record<string, ConfigValidationResult[]>> {
|
|
169
|
-
if (!logger) {
|
|
170
|
-
logger = pikkuState(null, 'package', 'singletonServices')?.logger
|
|
171
|
-
}
|
|
172
|
-
if (!logger) {
|
|
173
|
-
throw new Error(
|
|
174
|
-
'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first'
|
|
175
|
-
)
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
return await registerQueueWorkers(
|
|
179
|
-
this.configMappings,
|
|
180
|
-
logger,
|
|
181
|
-
async (queueName, processor) => {
|
|
182
|
-
await this.pgBoss.createQueue(queueName)
|
|
183
|
-
const workerId = await this.pgBoss.work(
|
|
184
|
-
processor.name,
|
|
185
|
-
{
|
|
186
|
-
...mapPikkuWorkerToPgBoss(processor.config),
|
|
187
|
-
includeMetadata: true,
|
|
188
|
-
},
|
|
189
|
-
async (jobs) => {
|
|
190
|
-
if (!jobs || jobs.length === 0) {
|
|
191
|
-
logger.warn(`No jobs received for queue ${queueName}`)
|
|
192
|
-
return
|
|
193
|
-
}
|
|
194
|
-
// Process all jobs in parallel
|
|
195
|
-
await Promise.all(
|
|
196
|
-
jobs.map(async (job) => {
|
|
197
|
-
try {
|
|
198
|
-
await runQueueJob({
|
|
199
|
-
job: mapPgBossJobToQueueJob(job, this.pgBoss),
|
|
200
|
-
})
|
|
201
|
-
} catch (error: unknown) {
|
|
202
|
-
if (error instanceof QueueJobFailedError) {
|
|
203
|
-
// Mark this specific job as failed
|
|
204
|
-
await this.pgBoss.fail(queueName, job.id, {
|
|
205
|
-
message: error.message,
|
|
206
|
-
})
|
|
207
|
-
} else if (error instanceof QueueJobDiscardedError) {
|
|
208
|
-
// For pg-boss, complete the job successfully to discard it
|
|
209
|
-
logger.info(
|
|
210
|
-
`PgBoss job ${job.id} discarded: ${error.message}`
|
|
211
|
-
)
|
|
212
|
-
await this.pgBoss.complete(queueName, job.id)
|
|
213
|
-
} else {
|
|
214
|
-
// For other errors, fail the job
|
|
215
|
-
await this.pgBoss.fail(queueName, job.id, {
|
|
216
|
-
message:
|
|
217
|
-
error instanceof Error
|
|
218
|
-
? error.message
|
|
219
|
-
: 'Unknown error',
|
|
220
|
-
})
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
})
|
|
224
|
-
)
|
|
225
|
-
}
|
|
226
|
-
)
|
|
227
|
-
this.activeWorkers.set(queueName, workerId)
|
|
228
|
-
}
|
|
229
|
-
)
|
|
230
|
-
}
|
|
231
|
-
}
|
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
import type { PgBoss } from 'pg-boss'
|
|
2
|
-
import { runScheduledTask } from '@pikku/core/scheduler'
|
|
3
|
-
import { SchedulerService } from '@pikku/core/services'
|
|
4
|
-
import type { CoreUserSession } from '@pikku/core/types'
|
|
5
|
-
import type {
|
|
6
|
-
ScheduledTaskInfo,
|
|
7
|
-
ScheduledTaskSummary,
|
|
8
|
-
} from '@pikku/core/services'
|
|
9
|
-
import { parseDurationString } from '@pikku/core/utils'
|
|
10
|
-
import { pikkuState } from '@pikku/core/state'
|
|
11
|
-
import { getScheduledTasks } from '@pikku/core/scheduler'
|
|
12
|
-
|
|
13
|
-
/**
|
|
14
|
-
* Data stored in scheduled job
|
|
15
|
-
*/
|
|
16
|
-
interface ScheduledJobData {
|
|
17
|
-
rpcName: string
|
|
18
|
-
data?: any
|
|
19
|
-
session?: CoreUserSession
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
/** Prefix for pg-boss queues/schedules backing recurring scheduled tasks. */
|
|
23
|
-
const RECURRING_TASK_PREFIX = 'pikku-recurring-scheduled-task_'
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* pg-boss scheduler service implementation
|
|
27
|
-
* Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
|
|
28
|
-
* Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
|
|
29
|
-
*/
|
|
30
|
-
export class PgBossSchedulerService extends SchedulerService {
|
|
31
|
-
private scheduledCronNames: string[] = []
|
|
32
|
-
|
|
33
|
-
constructor(private pgBoss: PgBoss) {
|
|
34
|
-
super()
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* Initialize - no-op as pg-boss is already initialized by factory
|
|
39
|
-
*/
|
|
40
|
-
async init(): Promise<void> {
|
|
41
|
-
// No-op - pg-boss is already initialized by factory
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* Schedule a one-off delayed RPC call
|
|
46
|
-
* Uses pg-boss send() with startAfter for one-time execution
|
|
47
|
-
*/
|
|
48
|
-
async scheduleRPC(
|
|
49
|
-
delay: number | string,
|
|
50
|
-
rpcName: string,
|
|
51
|
-
data?: any,
|
|
52
|
-
session?: CoreUserSession
|
|
53
|
-
): Promise<string> {
|
|
54
|
-
// Parse delay if it's a string
|
|
55
|
-
const delayMs =
|
|
56
|
-
typeof delay === 'string' ? parseDurationString(delay) : delay
|
|
57
|
-
|
|
58
|
-
const jobData: ScheduledJobData = {
|
|
59
|
-
rpcName,
|
|
60
|
-
data,
|
|
61
|
-
session,
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
// Use send() with startAfter for one-off delayed execution
|
|
65
|
-
const taskId = await this.pgBoss.send(
|
|
66
|
-
'pikku-remote-internal-rpc',
|
|
67
|
-
jobData,
|
|
68
|
-
{
|
|
69
|
-
startAfter: new Date(Date.now() + delayMs),
|
|
70
|
-
singletonKey: `${rpcName}-${Date.now()}`, // Ensure uniqueness
|
|
71
|
-
}
|
|
72
|
-
)
|
|
73
|
-
|
|
74
|
-
if (!taskId) {
|
|
75
|
-
throw new Error('Failed to schedule RPC')
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
return taskId
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
/**
|
|
82
|
-
* Unschedule (cancel) a task by ID
|
|
83
|
-
*/
|
|
84
|
-
async unschedule(taskId: string): Promise<boolean> {
|
|
85
|
-
await this.pgBoss.cancel('pikku-remote-internal-rpc', taskId)
|
|
86
|
-
return true
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Get a scheduled task by ID with full details
|
|
91
|
-
*/
|
|
92
|
-
async getTask(taskId: string): Promise<ScheduledTaskInfo | null> {
|
|
93
|
-
const job = await this.pgBoss.getJobById(
|
|
94
|
-
'pikku-remote-internal-rpc',
|
|
95
|
-
taskId
|
|
96
|
-
)
|
|
97
|
-
|
|
98
|
-
if (!job) {
|
|
99
|
-
return null
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
const jobData = job.data as ScheduledJobData
|
|
103
|
-
|
|
104
|
-
return {
|
|
105
|
-
taskId: job.id,
|
|
106
|
-
rpcName: jobData.rpcName,
|
|
107
|
-
scheduledFor: job.startAfter || job.createdOn,
|
|
108
|
-
data: jobData.data,
|
|
109
|
-
session: jobData.session,
|
|
110
|
-
status: job.state as any,
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
/**
|
|
115
|
-
* Get all scheduled tasks with minimal info
|
|
116
|
-
*/
|
|
117
|
-
async getAllTasks(): Promise<ScheduledTaskSummary[]> {
|
|
118
|
-
// pg-boss doesn't have a simple way to list all pending jobs
|
|
119
|
-
// This would require querying the database directly or maintaining a separate index
|
|
120
|
-
// For now, return empty array - can be implemented if needed
|
|
121
|
-
return []
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
/**
|
|
125
|
-
* Close - no-op as pg-boss lifecycle is managed by factory
|
|
126
|
-
*/
|
|
127
|
-
async close(): Promise<void> {
|
|
128
|
-
// No-op - pg-boss lifecycle is managed by factory
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
/**
|
|
132
|
-
* Start recurring scheduled tasks.
|
|
133
|
-
* Registers a pg-boss worker to process recurring jobs via runScheduledTask.
|
|
134
|
-
*/
|
|
135
|
-
async start(): Promise<void> {
|
|
136
|
-
const logger = pikkuState(null, 'package', 'singletonServices')!.logger
|
|
137
|
-
const scheduledTasks = getScheduledTasks()
|
|
138
|
-
|
|
139
|
-
const expectedCronNames = new Set<string>()
|
|
140
|
-
for (const [name, task] of scheduledTasks) {
|
|
141
|
-
const cronName = `${RECURRING_TASK_PREFIX}${name.replace(/[^a-zA-Z0-9_\-.]/g, '_')}`
|
|
142
|
-
expectedCronNames.add(cronName)
|
|
143
|
-
await this.pgBoss.createQueue(cronName)
|
|
144
|
-
await this.pgBoss.schedule(cronName, task.schedule, {
|
|
145
|
-
rpcName: name,
|
|
146
|
-
} as ScheduledJobData)
|
|
147
|
-
this.scheduledCronNames.push(cronName)
|
|
148
|
-
|
|
149
|
-
// Register a worker to process the scheduled jobs
|
|
150
|
-
await this.pgBoss.work<ScheduledJobData>(cronName, async (jobs) => {
|
|
151
|
-
for (const job of jobs) {
|
|
152
|
-
const { rpcName } = job.data
|
|
153
|
-
logger.info(`Running scheduled task: ${rpcName}`)
|
|
154
|
-
await runScheduledTask({ name: rpcName })
|
|
155
|
-
}
|
|
156
|
-
})
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
// Prune orphaned schedules. A task removed from code leaves its pg-boss
|
|
160
|
-
// schedule behind; with no worker registered it keeps producing jobs that
|
|
161
|
-
// pile up as 'created' forever (this is how a removed task flooded a queue
|
|
162
|
-
// with thousands of stuck jobs). Drop any recurring schedule + queue that's
|
|
163
|
-
// no longer registered so task removals self-heal on the next boot.
|
|
164
|
-
try {
|
|
165
|
-
const existing = await this.pgBoss.getSchedules()
|
|
166
|
-
for (const schedule of existing) {
|
|
167
|
-
if (
|
|
168
|
-
schedule.name.startsWith(RECURRING_TASK_PREFIX) &&
|
|
169
|
-
!expectedCronNames.has(schedule.name)
|
|
170
|
-
) {
|
|
171
|
-
await this.pgBoss.unschedule(schedule.name)
|
|
172
|
-
await this.pgBoss.deleteQueue(schedule.name)
|
|
173
|
-
logger.warn(`Pruned orphaned scheduled task queue: ${schedule.name}`)
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
} catch (err) {
|
|
177
|
-
logger.warn(
|
|
178
|
-
`Failed to prune orphaned scheduled tasks: ${err instanceof Error ? err.message : String(err)}`
|
|
179
|
-
)
|
|
180
|
-
}
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* Stop recurring scheduled tasks by unscheduling them from pg-boss.
|
|
185
|
-
*/
|
|
186
|
-
async stop(): Promise<void> {
|
|
187
|
-
for (const cronName of this.scheduledCronNames) {
|
|
188
|
-
await this.pgBoss.unschedule(cronName)
|
|
189
|
-
}
|
|
190
|
-
this.scheduledCronNames = []
|
|
191
|
-
}
|
|
192
|
-
}
|
|
@@ -1,85 +0,0 @@
|
|
|
1
|
-
import { PgBoss, type ConstructorOptions } from 'pg-boss'
|
|
2
|
-
import { PgBossQueueService } from './pg-boss-queue-service.js'
|
|
3
|
-
import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
|
|
4
|
-
import { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Factory class for pg-boss services
|
|
8
|
-
* Manages a single PgBoss instance shared across queue, worker, and scheduler services
|
|
9
|
-
*/
|
|
10
|
-
export class PgBossServiceFactory {
|
|
11
|
-
private pgBoss: PgBoss
|
|
12
|
-
private queueService?: PgBossQueueService
|
|
13
|
-
private queueWorkers?: PgBossQueueWorkers
|
|
14
|
-
private schedulerService?: PgBossSchedulerService
|
|
15
|
-
private initialized = false
|
|
16
|
-
|
|
17
|
-
constructor(optionsOrInstance: ConstructorOptions | string | PgBoss) {
|
|
18
|
-
if (optionsOrInstance instanceof PgBoss) {
|
|
19
|
-
this.pgBoss = optionsOrInstance
|
|
20
|
-
} else {
|
|
21
|
-
const options =
|
|
22
|
-
typeof optionsOrInstance === 'string'
|
|
23
|
-
? { connectionString: optionsOrInstance }
|
|
24
|
-
: optionsOrInstance
|
|
25
|
-
this.pgBoss = new PgBoss(options)
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/**
|
|
30
|
-
* Initialize the pg-boss instance
|
|
31
|
-
*/
|
|
32
|
-
async init(): Promise<void> {
|
|
33
|
-
if (!this.initialized) {
|
|
34
|
-
await this.pgBoss.start()
|
|
35
|
-
this.initialized = true
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Get the queue service for publishing jobs
|
|
41
|
-
*/
|
|
42
|
-
getQueueService(): PgBossQueueService {
|
|
43
|
-
if (!this.queueService) {
|
|
44
|
-
this.queueService = new PgBossQueueService(this.pgBoss)
|
|
45
|
-
}
|
|
46
|
-
return this.queueService
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/**
|
|
50
|
-
* Get the queue workers for processing jobs
|
|
51
|
-
*/
|
|
52
|
-
getQueueWorkers(): PgBossQueueWorkers {
|
|
53
|
-
if (!this.queueWorkers) {
|
|
54
|
-
this.queueWorkers = new PgBossQueueWorkers(this.pgBoss)
|
|
55
|
-
}
|
|
56
|
-
return this.queueWorkers
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Get the scheduler service for managing recurring tasks
|
|
61
|
-
*/
|
|
62
|
-
getSchedulerService(): PgBossSchedulerService {
|
|
63
|
-
if (!this.schedulerService) {
|
|
64
|
-
this.schedulerService = new PgBossSchedulerService(this.pgBoss)
|
|
65
|
-
}
|
|
66
|
-
return this.schedulerService
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
/**
|
|
70
|
-
* Close all services and the pg-boss connection
|
|
71
|
-
*/
|
|
72
|
-
async close(): Promise<void> {
|
|
73
|
-
if (this.initialized) {
|
|
74
|
-
await this.pgBoss.stop()
|
|
75
|
-
this.initialized = false
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
/**
|
|
80
|
-
* Alias for close() - used by stopSingletonServices
|
|
81
|
-
*/
|
|
82
|
-
async stop(): Promise<void> {
|
|
83
|
-
await this.close()
|
|
84
|
-
}
|
|
85
|
-
}
|
package/src/utils.ts
DELETED
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
import type { QueueJob, QueueJobStatus } from '@pikku/core/queue'
|
|
2
|
-
import type { PgBoss, JobWithMetadata } from 'pg-boss'
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* Map pg-boss job state to our queue job status
|
|
6
|
-
*/
|
|
7
|
-
const mapPgBossStateToStatus = (state: string): QueueJobStatus => {
|
|
8
|
-
switch (state) {
|
|
9
|
-
case 'created':
|
|
10
|
-
case 'retry':
|
|
11
|
-
return 'waiting'
|
|
12
|
-
case 'active':
|
|
13
|
-
return 'active'
|
|
14
|
-
case 'completed':
|
|
15
|
-
return 'completed'
|
|
16
|
-
case 'failed':
|
|
17
|
-
return 'failed'
|
|
18
|
-
case 'cancelled':
|
|
19
|
-
return 'failed'
|
|
20
|
-
default:
|
|
21
|
-
return 'waiting'
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Map pg-boss job to QueueJob interface
|
|
27
|
-
*/
|
|
28
|
-
export const mapPgBossJobToQueueJob = <In, Out>(
|
|
29
|
-
pgBossJob: JobWithMetadata<In>,
|
|
30
|
-
pgBossInstance: PgBoss
|
|
31
|
-
): QueueJob<In, Out> => {
|
|
32
|
-
return {
|
|
33
|
-
queueName: pgBossJob.name,
|
|
34
|
-
id: pgBossJob.id || '',
|
|
35
|
-
data: pgBossJob.data,
|
|
36
|
-
status: async () => {
|
|
37
|
-
const jobWithState = await pgBossInstance.getJobById(
|
|
38
|
-
pgBossJob.name,
|
|
39
|
-
pgBossJob.id
|
|
40
|
-
)
|
|
41
|
-
return mapPgBossStateToStatus(jobWithState?.state || 'created')
|
|
42
|
-
},
|
|
43
|
-
waitForCompletion: async (ttl?: number): Promise<Out> => {
|
|
44
|
-
// pg-boss doesn't have a built-in waitForCompletion, so we'll poll
|
|
45
|
-
const startTime = Date.now()
|
|
46
|
-
const timeout = ttl || 30000 // 30 seconds default
|
|
47
|
-
|
|
48
|
-
while (Date.now() - startTime < timeout) {
|
|
49
|
-
const job = await pgBossInstance.getJobById(
|
|
50
|
-
pgBossJob.name,
|
|
51
|
-
pgBossJob.id
|
|
52
|
-
)
|
|
53
|
-
if (job?.state === 'completed') {
|
|
54
|
-
return job.output as Out
|
|
55
|
-
}
|
|
56
|
-
if (job?.state === 'failed' || job?.state === 'cancelled') {
|
|
57
|
-
throw new Error(`Job failed: ${JSON.stringify(job.output)}`)
|
|
58
|
-
}
|
|
59
|
-
// Wait 100ms before polling again
|
|
60
|
-
await new Promise((resolve) => setTimeout(resolve, 100))
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
throw new Error('Job did not complete within timeout')
|
|
64
|
-
},
|
|
65
|
-
metadata: () => {
|
|
66
|
-
return {
|
|
67
|
-
progress: 0, // pg-boss doesn't have built-in progress tracking
|
|
68
|
-
attemptsMade: pgBossJob.retryCount || 0,
|
|
69
|
-
maxAttempts: pgBossJob.retryLimit || 1,
|
|
70
|
-
createdAt: pgBossJob.createdOn
|
|
71
|
-
? new Date(pgBossJob.createdOn)
|
|
72
|
-
: new Date(),
|
|
73
|
-
processedAt: pgBossJob.startedOn
|
|
74
|
-
? new Date(pgBossJob.startedOn)
|
|
75
|
-
: undefined,
|
|
76
|
-
completedAt: pgBossJob.completedOn
|
|
77
|
-
? new Date(pgBossJob.completedOn)
|
|
78
|
-
: undefined,
|
|
79
|
-
failedAt:
|
|
80
|
-
pgBossJob.state === 'failed' && pgBossJob.completedOn
|
|
81
|
-
? new Date(pgBossJob.completedOn)
|
|
82
|
-
: undefined,
|
|
83
|
-
error: (pgBossJob.data as any).error,
|
|
84
|
-
}
|
|
85
|
-
},
|
|
86
|
-
}
|
|
87
|
-
}
|
package/tsconfig.json
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"extends": "../../tsconfig.json",
|
|
3
|
-
"compilerOptions": {
|
|
4
|
-
"rootDir": "./src",
|
|
5
|
-
"module": "Node18",
|
|
6
|
-
"outDir": "dist",
|
|
7
|
-
"target": "esnext",
|
|
8
|
-
"declaration": true
|
|
9
|
-
},
|
|
10
|
-
"include": ["src/**/*.ts"],
|
|
11
|
-
"exclude": ["**/*.test.ts", "node_modules"],
|
|
12
|
-
"references": [
|
|
13
|
-
{
|
|
14
|
-
"path": "../../core/tsconfig.json"
|
|
15
|
-
}
|
|
16
|
-
]
|
|
17
|
-
}
|
package/tsconfig.tsbuildinfo
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"root":["./src/index.ts","./src/pg-boss-queue-service.ts","./src/pg-boss-queue-worker.ts","./src/pg-boss-scheduler-service.ts","./src/pg-boss-service-factory.ts","./src/utils.ts"],"version":"6.0.3"}
|