@pikku/queue-pg-boss 0.12.10 → 0.12.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,31 @@
1
+ ## 0.12.11
2
+
3
+ ### Patch Changes
4
+
5
+ - 4c7a1b5: Run the monorepo's own scripts through bun instead of yarn. What moves is the
6
+ package manager each package's `prepublishOnly` and build scripts invoke, plus
7
+ the two manifest fixes bun needs to resolve the tree: `uWebSockets.js` is
8
+ declared with an explicit `github:` specifier, and `@pikku/uws-handler` marks
9
+ its `uWebSockets.js` peer optional so a bun install of a consumer that brings
10
+ its own uWS app does not try to fetch it from the registry.
11
+
12
+ Three published behaviours change, all of them cases where an isolated
13
+ `node_modules` or bun as the runtime had been papered over by yarn's hoisting:
14
+
15
+ - `@pikku/migrator-sql` turns foreign keys on when it opens a sqlite database
16
+ through bun. `node:sqlite` enforces them by default and `bun:sqlite` does not,
17
+ which silently turned every `ON DELETE CASCADE` into a no-op under
18
+ `bunx --bun pikku`.
19
+ - `@pikku/cli` resolves a deploy provider against the project being deployed
20
+ rather than against wherever the CLI itself is installed, which is what its
21
+ own "is not installed" error asks the user to arrange.
22
+ - `@pikku/cli` treats a specifier a runtime hands straight back — bun does this
23
+ for the modules it implements itself — as not resolved from the project, so
24
+ it falls back rather than loading the runtime's own copy.
25
+
26
+ - Updated dependencies [4c7a1b5]
27
+ - @pikku/core@0.12.108
28
+
1
29
  ## 0.12.10
2
30
 
3
31
  ### Patch Changes
@@ -1,7 +1,7 @@
1
1
  import type { PgBoss, Request as PgBossRequest } from 'pg-boss';
2
2
  import type { QueueJob, QueueService } from '@pikku/core/queue';
3
3
  import type { JobOptions } from '@pikku/core/queue';
4
- export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBossRequest["options"];
4
+ export declare const mapPikkuJobToPgBoss: (options?: JobOptions) => PgBossRequest['options'];
5
5
  /**
6
6
  * pg-boss/PostgreSQL queue service implementation
7
7
  * Supports job results, retries, and PostgreSQL-based queuing
package/package.json CHANGED
@@ -1,6 +1,11 @@
1
1
  {
2
2
  "name": "@pikku/queue-pg-boss",
3
- "version": "0.12.10",
3
+ "version": "0.12.11",
4
+ "repository": {
5
+ "type": "git",
6
+ "url": "git+https://github.com/pikkujs/pikku.git",
7
+ "directory": "packages/services/queue-pg-boss"
8
+ },
4
9
  "description": "pg-boss queue, worker and scheduler services for Pikku, backed by Postgres",
5
10
  "author": "yasser.fadl@gmail.com",
6
11
  "license": "MIT",
@@ -13,18 +18,15 @@
13
18
  "ncu": "npx npm-check-updates",
14
19
  "build": "tsc -b",
15
20
  "release": "npm run build && npm test",
16
- "prepublishOnly": "yarn build"
21
+ "prepublishOnly": "bun run build"
17
22
  },
18
23
  "peerDependencies": {
19
- "@pikku/core": "^0.12.85"
24
+ "@pikku/core": "^0.12.108"
20
25
  },
21
26
  "dependencies": {
22
27
  "pg-boss": "^12.27.0"
23
28
  },
24
- "devDependencies": {
25
- "typescript": "^6.0.3"
26
- },
27
29
  "exports": {
28
30
  ".": "./dist/index.js"
29
31
  }
30
- }
32
+ }
package/run-tests.sh CHANGED
@@ -1,8 +1,5 @@
1
1
  #!/bin/bash
2
2
 
3
- script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4
- cd "$script_dir"
5
-
6
3
  # Enable nullglob to handle cases where no files match the pattern
7
4
  shopt -s nullglob
8
5
 
@@ -37,20 +34,35 @@ files=($(find src -type f -name "*.test.ts"))
37
34
  # Check if any files matched the pattern
38
35
  if [ ${#files[@]} -eq 0 ]; then
39
36
  echo "No test files found matching pattern: $pattern"
37
+ if [ "${STRICT_TEST_DISCOVERY}" = "1" ]; then
38
+ exit 1
39
+ fi
40
40
  exit 0
41
41
  fi
42
42
 
43
- # Construct the node command
44
- node_cmd=(node --import tsx --test)
43
+ # Construct the bun command
44
+ #
45
+ # --parallel gives every test file its own worker process, which is what node's
46
+ # `--test` did. Several suites keep module-level caches (a TypeScript program, a
47
+ # temp fixture directory) that a shared process would leak between files.
48
+ #
49
+ # --timeout because bun caps a test at 5s by default where node had no cap, and
50
+ # the suites that build real TypeScript programs run well past that.
51
+ bun_cmd=(bun test --parallel --timeout 120000)
45
52
 
46
53
  # Append options based on flags
47
54
  if [ "$watch_mode" = true ]; then
48
- node_cmd+=(--watch)
55
+ # --watch reruns in one process; parallel workers have nothing to watch.
56
+ bun_cmd=(bun test --watch --timeout 120000)
49
57
  fi
50
58
 
51
59
  if [ "$coverage_mode" = true ]; then
52
- node_cmd+=(--test-coverage-include="src/**/*.{ts,js}" --test-coverage-exclude="**/dist/**" --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=lcov.info)
60
+ # --coverage-dir=. keeps lcov.info at the package root: CI merges every
61
+ # package's report by prefixing its SF paths with the file's own directory,
62
+ # so a nested coverage/ directory would re-root them one level too deep.
63
+ bun_cmd+=(--coverage --coverage-reporter=lcov --coverage-reporter=text --coverage-dir=.)
64
+ export PIKKU_TEST_COVERAGE=1
53
65
  fi
54
66
 
55
- # Execute the node command with the expanded list of files
56
- "${node_cmd[@]}" "${files[@]}"
67
+ # Execute the bun command with the expanded list of files
68
+ "${bun_cmd[@]}" "${files[@]}"
package/src/index.ts DELETED
@@ -1,19 +0,0 @@
1
- /**
2
- * @module @pikku/queue-pg-boss
3
- *
4
- * PostgreSQL queue service for Pikku using pg-boss
5
- * Provides job results, retries, priorities, and PostgreSQL-based queuing
6
- */
7
-
8
- export { PgBossServiceFactory } from './pg-boss-service-factory.js'
9
- export { PgBossQueueService } from './pg-boss-queue-service.js'
10
- export { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
11
- export { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
12
-
13
- // Re-export core queue types for convenience
14
- export type { QueueJob, QueueService } from '@pikku/core/queue'
15
- export type {
16
- PikkuWorkerConfig,
17
- PikkuJobConfig,
18
- JobOptions,
19
- } from '@pikku/core/queue'
@@ -1,83 +0,0 @@
1
- import { describe, test } from 'node:test'
2
- import assert from 'node:assert/strict'
3
-
4
- import { mapPikkuJobToPgBoss } from './pg-boss-queue-service.js'
5
-
6
- // These guard the contract the workflow engine relies on: a step's resolved
7
- // retry policy (attempts + backoff) must survive translation into pg-boss
8
- // terms, so "workflow owns retries" actually reaches the queue.
9
- describe('mapPikkuJobToPgBoss — retry mapping', () => {
10
- test('attempts → retryLimit (attempts - 1)', () => {
11
- // Default policy: 5 retries → 6 attempts → retryLimit 5.
12
- assert.equal(mapPikkuJobToPgBoss({ attempts: 6 }).retryLimit, 5)
13
- })
14
-
15
- test('attempts: 1 (retries: 0) → retryLimit 0 (pg-boss never re-runs it)', () => {
16
- assert.equal(mapPikkuJobToPgBoss({ attempts: 1 }).retryLimit, 0)
17
- })
18
-
19
- test("backoff 'exponential' → retryBackoff true + 1s base delay", () => {
20
- const opts = mapPikkuJobToPgBoss({ attempts: 6, backoff: 'exponential' })
21
- assert.equal(opts.retryBackoff, true)
22
- // pg-boss multiplies retry_delay by 2^n and the queue default is 0 —
23
- // without a base, "exponential" retries all fire immediately.
24
- assert.equal(opts.retryDelay, 1)
25
- })
26
-
27
- test('sub-second fixed delay rounds up to 1s, not down to immediate', () => {
28
- const opts = mapPikkuJobToPgBoss({
29
- attempts: 4,
30
- backoff: { type: 'fixed', delay: 300 },
31
- })
32
- assert.equal(opts.retryDelay, 1)
33
- })
34
-
35
- test('fixed backoff → retryBackoff false + retryDelay in seconds', () => {
36
- const opts = mapPikkuJobToPgBoss({
37
- attempts: 4,
38
- backoff: { type: 'fixed', delay: 5000 },
39
- })
40
- assert.equal(opts.retryBackoff, false)
41
- // pg-boss takes seconds; our delay is milliseconds.
42
- assert.equal(opts.retryDelay, 5)
43
- })
44
-
45
- test('exponential object backoff → retryBackoff true + retryDelay in seconds', () => {
46
- const opts = mapPikkuJobToPgBoss({
47
- attempts: 4,
48
- backoff: { type: 'exponential', delay: 2000 },
49
- })
50
- assert.equal(opts.retryBackoff, true)
51
- assert.equal(opts.retryDelay, 2)
52
- })
53
-
54
- test('no options → no retry fields set', () => {
55
- const opts = mapPikkuJobToPgBoss(undefined)
56
- assert.equal(opts.retryLimit, undefined)
57
- assert.equal(opts.retryBackoff, undefined)
58
- })
59
- })
60
-
61
- describe('mapPikkuJobToPgBoss — group mapping', () => {
62
- test('group id is forwarded so the worker can enforce a per-group limit', () => {
63
- assert.deepEqual(
64
- mapPikkuJobToPgBoss({ group: { id: 'deployWorkflow' } }).group,
65
- { id: 'deployWorkflow' }
66
- )
67
- })
68
-
69
- test('tier is only set when provided', () => {
70
- const opts = mapPikkuJobToPgBoss({
71
- group: { id: 'deployWorkflow', tier: 'slow' },
72
- })
73
- assert.deepEqual(opts.group, { id: 'deployWorkflow', tier: 'slow' })
74
- assert.equal(
75
- 'tier' in mapPikkuJobToPgBoss({ group: { id: 'a' } }).group!,
76
- false
77
- )
78
- })
79
-
80
- test('no group → no group field', () => {
81
- assert.equal(mapPikkuJobToPgBoss({ attempts: 2 }).group, undefined)
82
- })
83
- })
@@ -1,113 +0,0 @@
1
- import type { PgBoss, Request as PgBossRequest } from 'pg-boss'
2
- import type { QueueJob, QueueService } from '@pikku/core/queue'
3
- import type { JobOptions } from '@pikku/core/queue'
4
- import { mapPgBossJobToQueueJob } from './utils.js'
5
-
6
- export const mapPikkuJobToPgBoss = (
7
- options?: JobOptions
8
- ): PgBossRequest['options'] => {
9
- const pgBossOptions: any = {}
10
-
11
- if (options?.priority !== undefined) {
12
- pgBossOptions.priority = options.priority
13
- }
14
-
15
- if (options?.delay !== undefined) {
16
- pgBossOptions.startAfter = new Date(Date.now() + options.delay)
17
- }
18
-
19
- if (options?.jobId !== undefined) {
20
- pgBossOptions.singletonKey = options.jobId
21
- }
22
-
23
- if (options?.group !== undefined) {
24
- pgBossOptions.group = {
25
- id: options.group.id,
26
- ...(options.group.tier !== undefined ? { tier: options.group.tier } : {}),
27
- }
28
- }
29
-
30
- // Map retry options
31
- if (options?.attempts !== undefined) {
32
- pgBossOptions.retryLimit = options.attempts - 1
33
- }
34
-
35
- if (options?.backoff !== undefined) {
36
- const type =
37
- typeof options.backoff === 'string'
38
- ? options.backoff
39
- : options.backoff.type
40
- const delay =
41
- typeof options.backoff === 'object' ? options.backoff.delay : undefined
42
- pgBossOptions.retryBackoff = type === 'exponential'
43
- if (delay !== undefined) {
44
- // pg-boss uses seconds, we use milliseconds — never round a sub-second
45
- // delay down to 0, which pg-boss treats as retry-immediately
46
- pgBossOptions.retryDelay = Math.max(1, Math.round(delay / 1000))
47
- } else if (type === 'exponential') {
48
- // pg-boss computes exponential backoff as retry_delay * 2^n, and the
49
- // queue-level retry_delay defaults to 0 — without an explicit base every
50
- // "exponential" retry fires immediately. 1s base → ~1s, 2s, 4s, 8s…
51
- pgBossOptions.retryDelay = 1
52
- }
53
- }
54
-
55
- if (options?.removeOnComplete !== undefined) {
56
- pgBossOptions.onComplete = options.removeOnComplete === 0
57
- }
58
-
59
- if (options?.removeOnFail !== undefined) {
60
- // pg-boss doesn't have a direct equivalent, but we can track this for consistency
61
- // Note: pg-boss keeps failed jobs by default
62
- }
63
-
64
- return pgBossOptions as PgBossRequest['options']
65
- }
66
-
67
- /**
68
- * pg-boss/PostgreSQL queue service implementation
69
- * Supports job results, retries, and PostgreSQL-based queuing
70
- */
71
- export class PgBossQueueService implements QueueService {
72
- readonly name = 'pg-boss'
73
- readonly supportsResults = true
74
- protected pgBoss: PgBoss
75
-
76
- constructor(pgBoss: PgBoss) {
77
- this.pgBoss = pgBoss
78
- }
79
-
80
- /**
81
- * Add a job to the queue
82
- */
83
- public async add<In>(
84
- queueName: string,
85
- data: In,
86
- options?: JobOptions
87
- ): Promise<string> {
88
- await this.pgBoss.createQueue(queueName)
89
- const jobId = await this.pgBoss.send(
90
- queueName,
91
- data as any,
92
- mapPikkuJobToPgBoss(options)
93
- )
94
- if (!jobId) {
95
- throw new Error('Failed to create job')
96
- }
97
- return jobId
98
- }
99
-
100
- /**
101
- * Get job by ID
102
- */
103
- async getJob<T, R>(
104
- queueName: string,
105
- jobId: string
106
- ): Promise<QueueJob<T, R> | null> {
107
- const job = await this.pgBoss.getJobById<T>(queueName, jobId)
108
- if (!job) {
109
- return null
110
- }
111
- return mapPgBossJobToQueueJob<T, R>(job, this.pgBoss)
112
- }
113
- }
@@ -1,93 +0,0 @@
1
- import { describe, test } from 'node:test'
2
- import assert from 'node:assert/strict'
3
-
4
- import {
5
- PgBossQueueWorkers,
6
- mapPikkuWorkerToPgBoss,
7
- } from './pg-boss-queue-worker.js'
8
-
9
- describe('PgBossQueueWorkers', () => {
10
- test('requires a logger (explicit or from singleton services)', async () => {
11
- const workers = new PgBossQueueWorkers({} as any)
12
-
13
- await assert.rejects(() => workers.registerQueues(), {
14
- message:
15
- 'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first',
16
- })
17
- })
18
- })
19
-
20
- describe('mapPikkuWorkerToPgBoss', () => {
21
- test('parallelism maps to independent workers (localConcurrency), never batch fetching', () => {
22
- // batchSize: 1 keeps each worker single-job so a slow job can never hold its
23
- // batch siblings — head-of-line blocking is impossible.
24
- assert.deepEqual(mapPikkuWorkerToPgBoss(), {
25
- localConcurrency: 10,
26
- batchSize: 1,
27
- })
28
-
29
- assert.deepEqual(mapPikkuWorkerToPgBoss({ batchSize: 25 }), {
30
- localConcurrency: 25,
31
- batchSize: 1,
32
- })
33
- })
34
-
35
- test('pollInterval is converted from ms to seconds', () => {
36
- assert.deepEqual(mapPikkuWorkerToPgBoss({ pollInterval: 3000 }), {
37
- localConcurrency: 10,
38
- batchSize: 1,
39
- pollingIntervalSeconds: 3,
40
- })
41
- })
42
-
43
- test('groupConcurrency caps how much of the worker one group can take', () => {
44
- assert.deepEqual(mapPikkuWorkerToPgBoss({ groupConcurrency: 2 }), {
45
- localConcurrency: 10,
46
- batchSize: 1,
47
- localGroupConcurrency: 2,
48
- })
49
- })
50
-
51
- test('tiered groupConcurrency passes tiers through', () => {
52
- assert.deepEqual(
53
- mapPikkuWorkerToPgBoss({
54
- batchSize: 20,
55
- groupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
56
- }),
57
- {
58
- localConcurrency: 20,
59
- batchSize: 1,
60
- localGroupConcurrency: { default: 2, tiers: { slowWorkflow: 1 } },
61
- }
62
- )
63
- })
64
-
65
- test('group limits are clamped to the worker concurrency', () => {
66
- // pg-boss asserts every group limit is <= localConcurrency and throws at
67
- // worker start otherwise — clamp so an over-eager config degrades to "this
68
- // group may use the whole worker" instead of failing to start.
69
- assert.deepEqual(
70
- mapPikkuWorkerToPgBoss({
71
- batchSize: 4,
72
- groupConcurrency: { default: 10, tiers: { greedy: 99 } },
73
- }),
74
- {
75
- localConcurrency: 4,
76
- batchSize: 1,
77
- localGroupConcurrency: { default: 4, tiers: { greedy: 4 } },
78
- }
79
- )
80
-
81
- assert.deepEqual(
82
- mapPikkuWorkerToPgBoss({ batchSize: 3, groupConcurrency: 8 }),
83
- { localConcurrency: 3, batchSize: 1, localGroupConcurrency: 3 }
84
- )
85
- })
86
-
87
- test('no group config means no group limits', () => {
88
- assert.equal(
89
- 'localGroupConcurrency' in mapPikkuWorkerToPgBoss({ batchSize: 5 }),
90
- false
91
- )
92
- })
93
- })
@@ -1,231 +0,0 @@
1
- import type { PgBoss, WorkOptions } from 'pg-boss'
2
- import type {
3
- QueueWorkers,
4
- PikkuWorkerConfig,
5
- QueueConfigMapping,
6
- ConfigValidationResult,
7
- } from '@pikku/core/queue'
8
- import { runQueueJob } from '@pikku/core/queue'
9
- import type { Logger } from '@pikku/core/services'
10
- import {
11
- registerQueueWorkers,
12
- QueueJobFailedError,
13
- QueueJobDiscardedError,
14
- } from '@pikku/core/queue'
15
- import { pikkuState } from '@pikku/core/state'
16
- import { mapPgBossJobToQueueJob } from './utils.js'
17
-
18
- export const mapPikkuWorkerToPgBoss = (
19
- workerConfig?: PikkuWorkerConfig
20
- ): WorkOptions => {
21
- // Map the requested parallelism to pg-boss `localConcurrency` (independent
22
- // per-job workers), NOT `batchSize`. A batch worker hands the whole fetched
23
- // batch to one handler and only fetches again once that handler resolves — so
24
- // a single slow job blocks every sibling in its batch AND stalls the next
25
- // fetch. On a shared queue that is head-of-line blocking: one long job can
26
- // freeze the entire worker and starve everything queued behind it.
27
- // `localConcurrency` spawns N pollers that each fetch+process a single job and
28
- // refill the instant they finish, so a slow job never holds up the others.
29
- const localConcurrency = workerConfig?.batchSize ?? 10
30
- const workerOptions: WorkOptions = {
31
- localConcurrency,
32
- batchSize: 1,
33
- }
34
-
35
- if (workerConfig?.pollInterval !== undefined) {
36
- workerOptions.pollingIntervalSeconds = Math.round(
37
- workerConfig.pollInterval / 1000
38
- )
39
- }
40
-
41
- // Per-group fairness. `localGroupConcurrency` is in-memory and free — groups
42
- // already at capacity are excluded from the fetch itself, so an at-capacity
43
- // group costs nothing rather than being fetched and restored. pg-boss asserts
44
- // every limit is <= localConcurrency, so clamp instead of letting a config
45
- // that reads as "let this tier use more" throw at worker start.
46
- if (workerConfig?.groupConcurrency !== undefined) {
47
- const clamp = (limit: number) => Math.min(limit, localConcurrency)
48
- workerOptions.localGroupConcurrency =
49
- typeof workerConfig.groupConcurrency === 'number'
50
- ? clamp(workerConfig.groupConcurrency)
51
- : {
52
- default: clamp(workerConfig.groupConcurrency.default),
53
- ...(workerConfig.groupConcurrency.tiers
54
- ? {
55
- tiers: Object.fromEntries(
56
- Object.entries(workerConfig.groupConcurrency.tiers).map(
57
- ([tier, limit]) => [tier, clamp(limit)]
58
- )
59
- ),
60
- }
61
- : {}),
62
- }
63
- }
64
-
65
- return workerOptions
66
- }
67
-
68
- /**
69
- * pg-boss/PostgreSQL queue worker implementation
70
- * Supports job results, retries, and PostgreSQL-based queuing
71
- */
72
- export class PgBossQueueWorkers implements QueueWorkers {
73
- readonly name = 'pg-boss'
74
- readonly supportsResults = true
75
-
76
- /**
77
- * Configuration mapping rules for pg-boss
78
- * This defines how Pikku worker configs map to pg-boss configurations
79
- */
80
- readonly configMappings: QueueConfigMapping = {
81
- // Configurations that are directly supported
82
- supported: {
83
- batchSize: {
84
- queueProperty: 'localConcurrency',
85
- description: 'Number of jobs to process concurrently',
86
- },
87
- groupConcurrency: {
88
- queueProperty: 'localGroupConcurrency',
89
- description:
90
- 'Maximum concurrent jobs per group, keeping one shared queue fair across groups',
91
- },
92
- pollInterval: {
93
- queueProperty: 'pollingIntervalSeconds',
94
- transform: (value: number) => Math.round(value / 1000),
95
- description:
96
- 'How often to poll for new jobs (converted from ms to seconds)',
97
- },
98
- },
99
-
100
- // Configurations that are not supported but have reasonable explanations
101
- unsupported: {
102
- name: {
103
- reason: 'Worker names are not supported in pg-boss',
104
- explanation:
105
- 'pg-boss identifies workers by their queue name and process, not custom names',
106
- },
107
- autorun: {
108
- reason: 'Autorun is not configurable in pg-boss (always enabled)',
109
- explanation:
110
- 'pg-boss workers automatically start processing when created',
111
- },
112
- lockDuration: {
113
- reason: 'Lock duration is managed differently in pg-boss',
114
- explanation:
115
- 'pg-boss uses job-level expiration instead of worker-level locks',
116
- },
117
- drainDelay: {
118
- reason: 'Drain delay is not configurable in pg-boss',
119
- explanation: 'pg-boss handles graceful shutdown internally',
120
- },
121
- maxStalledCount: {
122
- reason: 'Max stalled count is not configurable in pg-boss',
123
- explanation:
124
- 'pg-boss handles stalled jobs through its built-in retry mechanism',
125
- },
126
- prefetch: {
127
- reason: 'Prefetch is managed internally by pg-boss',
128
- explanation:
129
- 'pg-boss optimizes job fetching automatically based on batch size',
130
- },
131
- visibilityTimeout: {
132
- reason: "Visibility timeout concept doesn't apply to pg-boss",
133
- explanation:
134
- 'pg-boss uses PostgreSQL locks and job expiration instead of visibility timeout',
135
- },
136
- },
137
-
138
- // Configurations that have partial support or workarounds
139
- fallbacks: {
140
- removeOnComplete: {
141
- reason: 'Job retention is managed through pg-boss archival system',
142
- explanation:
143
- 'pg-boss automatically archives completed jobs based on database settings',
144
- fallbackValue: 'Managed by pg-boss database archival',
145
- },
146
- removeOnFail: {
147
- reason:
148
- 'Failed job retention is managed through pg-boss archival system',
149
- explanation:
150
- 'pg-boss automatically archives failed jobs based on database settings',
151
- fallbackValue: 'Managed by pg-boss database archival',
152
- },
153
- },
154
- }
155
-
156
- private pgBoss: PgBoss
157
- private activeWorkers = new Map<string, string>()
158
-
159
- constructor(pgBoss: PgBoss) {
160
- this.pgBoss = pgBoss
161
- }
162
-
163
- /**
164
- * Scan state and register all compatible processors
165
- */
166
- async registerQueues(
167
- logger?: Logger
168
- ): Promise<Record<string, ConfigValidationResult[]>> {
169
- if (!logger) {
170
- logger = pikkuState(null, 'package', 'singletonServices')?.logger
171
- }
172
- if (!logger) {
173
- throw new Error(
174
- 'Logger is required for registerQueues — pass it explicitly or ensure singleton services are initialized first'
175
- )
176
- }
177
-
178
- return await registerQueueWorkers(
179
- this.configMappings,
180
- logger,
181
- async (queueName, processor) => {
182
- await this.pgBoss.createQueue(queueName)
183
- const workerId = await this.pgBoss.work(
184
- processor.name,
185
- {
186
- ...mapPikkuWorkerToPgBoss(processor.config),
187
- includeMetadata: true,
188
- },
189
- async (jobs) => {
190
- if (!jobs || jobs.length === 0) {
191
- logger.warn(`No jobs received for queue ${queueName}`)
192
- return
193
- }
194
- // Process all jobs in parallel
195
- await Promise.all(
196
- jobs.map(async (job) => {
197
- try {
198
- await runQueueJob({
199
- job: mapPgBossJobToQueueJob(job, this.pgBoss),
200
- })
201
- } catch (error: unknown) {
202
- if (error instanceof QueueJobFailedError) {
203
- // Mark this specific job as failed
204
- await this.pgBoss.fail(queueName, job.id, {
205
- message: error.message,
206
- })
207
- } else if (error instanceof QueueJobDiscardedError) {
208
- // For pg-boss, complete the job successfully to discard it
209
- logger.info(
210
- `PgBoss job ${job.id} discarded: ${error.message}`
211
- )
212
- await this.pgBoss.complete(queueName, job.id)
213
- } else {
214
- // For other errors, fail the job
215
- await this.pgBoss.fail(queueName, job.id, {
216
- message:
217
- error instanceof Error
218
- ? error.message
219
- : 'Unknown error',
220
- })
221
- }
222
- }
223
- })
224
- )
225
- }
226
- )
227
- this.activeWorkers.set(queueName, workerId)
228
- }
229
- )
230
- }
231
- }
@@ -1,192 +0,0 @@
1
- import type { PgBoss } from 'pg-boss'
2
- import { runScheduledTask } from '@pikku/core/scheduler'
3
- import { SchedulerService } from '@pikku/core/services'
4
- import type { CoreUserSession } from '@pikku/core/types'
5
- import type {
6
- ScheduledTaskInfo,
7
- ScheduledTaskSummary,
8
- } from '@pikku/core/services'
9
- import { parseDurationString } from '@pikku/core/utils'
10
- import { pikkuState } from '@pikku/core/state'
11
- import { getScheduledTasks } from '@pikku/core/scheduler'
12
-
13
- /**
14
- * Data stored in scheduled job
15
- */
16
- interface ScheduledJobData {
17
- rpcName: string
18
- data?: any
19
- session?: CoreUserSession
20
- }
21
-
22
- /** Prefix for pg-boss queues/schedules backing recurring scheduled tasks. */
23
- const RECURRING_TASK_PREFIX = 'pikku-recurring-scheduled-task_'
24
-
25
- /**
26
- * pg-boss scheduler service implementation
27
- * Uses pg-boss's schedule() API to store recurring jobs in PostgreSQL.
28
- * Handles both one-off delayed RPCs and recurring cron-scheduled tasks.
29
- */
30
- export class PgBossSchedulerService extends SchedulerService {
31
- private scheduledCronNames: string[] = []
32
-
33
- constructor(private pgBoss: PgBoss) {
34
- super()
35
- }
36
-
37
- /**
38
- * Initialize - no-op as pg-boss is already initialized by factory
39
- */
40
- async init(): Promise<void> {
41
- // No-op - pg-boss is already initialized by factory
42
- }
43
-
44
- /**
45
- * Schedule a one-off delayed RPC call
46
- * Uses pg-boss send() with startAfter for one-time execution
47
- */
48
- async scheduleRPC(
49
- delay: number | string,
50
- rpcName: string,
51
- data?: any,
52
- session?: CoreUserSession
53
- ): Promise<string> {
54
- // Parse delay if it's a string
55
- const delayMs =
56
- typeof delay === 'string' ? parseDurationString(delay) : delay
57
-
58
- const jobData: ScheduledJobData = {
59
- rpcName,
60
- data,
61
- session,
62
- }
63
-
64
- // Use send() with startAfter for one-off delayed execution
65
- const taskId = await this.pgBoss.send(
66
- 'pikku-remote-internal-rpc',
67
- jobData,
68
- {
69
- startAfter: new Date(Date.now() + delayMs),
70
- singletonKey: `${rpcName}-${Date.now()}`, // Ensure uniqueness
71
- }
72
- )
73
-
74
- if (!taskId) {
75
- throw new Error('Failed to schedule RPC')
76
- }
77
-
78
- return taskId
79
- }
80
-
81
- /**
82
- * Unschedule (cancel) a task by ID
83
- */
84
- async unschedule(taskId: string): Promise<boolean> {
85
- await this.pgBoss.cancel('pikku-remote-internal-rpc', taskId)
86
- return true
87
- }
88
-
89
- /**
90
- * Get a scheduled task by ID with full details
91
- */
92
- async getTask(taskId: string): Promise<ScheduledTaskInfo | null> {
93
- const job = await this.pgBoss.getJobById(
94
- 'pikku-remote-internal-rpc',
95
- taskId
96
- )
97
-
98
- if (!job) {
99
- return null
100
- }
101
-
102
- const jobData = job.data as ScheduledJobData
103
-
104
- return {
105
- taskId: job.id,
106
- rpcName: jobData.rpcName,
107
- scheduledFor: job.startAfter || job.createdOn,
108
- data: jobData.data,
109
- session: jobData.session,
110
- status: job.state as any,
111
- }
112
- }
113
-
114
- /**
115
- * Get all scheduled tasks with minimal info
116
- */
117
- async getAllTasks(): Promise<ScheduledTaskSummary[]> {
118
- // pg-boss doesn't have a simple way to list all pending jobs
119
- // This would require querying the database directly or maintaining a separate index
120
- // For now, return empty array - can be implemented if needed
121
- return []
122
- }
123
-
124
- /**
125
- * Close - no-op as pg-boss lifecycle is managed by factory
126
- */
127
- async close(): Promise<void> {
128
- // No-op - pg-boss lifecycle is managed by factory
129
- }
130
-
131
- /**
132
- * Start recurring scheduled tasks.
133
- * Registers a pg-boss worker to process recurring jobs via runScheduledTask.
134
- */
135
- async start(): Promise<void> {
136
- const logger = pikkuState(null, 'package', 'singletonServices')!.logger
137
- const scheduledTasks = getScheduledTasks()
138
-
139
- const expectedCronNames = new Set<string>()
140
- for (const [name, task] of scheduledTasks) {
141
- const cronName = `${RECURRING_TASK_PREFIX}${name.replace(/[^a-zA-Z0-9_\-.]/g, '_')}`
142
- expectedCronNames.add(cronName)
143
- await this.pgBoss.createQueue(cronName)
144
- await this.pgBoss.schedule(cronName, task.schedule, {
145
- rpcName: name,
146
- } as ScheduledJobData)
147
- this.scheduledCronNames.push(cronName)
148
-
149
- // Register a worker to process the scheduled jobs
150
- await this.pgBoss.work<ScheduledJobData>(cronName, async (jobs) => {
151
- for (const job of jobs) {
152
- const { rpcName } = job.data
153
- logger.info(`Running scheduled task: ${rpcName}`)
154
- await runScheduledTask({ name: rpcName })
155
- }
156
- })
157
- }
158
-
159
- // Prune orphaned schedules. A task removed from code leaves its pg-boss
160
- // schedule behind; with no worker registered it keeps producing jobs that
161
- // pile up as 'created' forever (this is how a removed task flooded a queue
162
- // with thousands of stuck jobs). Drop any recurring schedule + queue that's
163
- // no longer registered so task removals self-heal on the next boot.
164
- try {
165
- const existing = await this.pgBoss.getSchedules()
166
- for (const schedule of existing) {
167
- if (
168
- schedule.name.startsWith(RECURRING_TASK_PREFIX) &&
169
- !expectedCronNames.has(schedule.name)
170
- ) {
171
- await this.pgBoss.unschedule(schedule.name)
172
- await this.pgBoss.deleteQueue(schedule.name)
173
- logger.warn(`Pruned orphaned scheduled task queue: ${schedule.name}`)
174
- }
175
- }
176
- } catch (err) {
177
- logger.warn(
178
- `Failed to prune orphaned scheduled tasks: ${err instanceof Error ? err.message : String(err)}`
179
- )
180
- }
181
- }
182
-
183
- /**
184
- * Stop recurring scheduled tasks by unscheduling them from pg-boss.
185
- */
186
- async stop(): Promise<void> {
187
- for (const cronName of this.scheduledCronNames) {
188
- await this.pgBoss.unschedule(cronName)
189
- }
190
- this.scheduledCronNames = []
191
- }
192
- }
@@ -1,85 +0,0 @@
1
- import { PgBoss, type ConstructorOptions } from 'pg-boss'
2
- import { PgBossQueueService } from './pg-boss-queue-service.js'
3
- import { PgBossQueueWorkers } from './pg-boss-queue-worker.js'
4
- import { PgBossSchedulerService } from './pg-boss-scheduler-service.js'
5
-
6
- /**
7
- * Factory class for pg-boss services
8
- * Manages a single PgBoss instance shared across queue, worker, and scheduler services
9
- */
10
- export class PgBossServiceFactory {
11
- private pgBoss: PgBoss
12
- private queueService?: PgBossQueueService
13
- private queueWorkers?: PgBossQueueWorkers
14
- private schedulerService?: PgBossSchedulerService
15
- private initialized = false
16
-
17
- constructor(optionsOrInstance: ConstructorOptions | string | PgBoss) {
18
- if (optionsOrInstance instanceof PgBoss) {
19
- this.pgBoss = optionsOrInstance
20
- } else {
21
- const options =
22
- typeof optionsOrInstance === 'string'
23
- ? { connectionString: optionsOrInstance }
24
- : optionsOrInstance
25
- this.pgBoss = new PgBoss(options)
26
- }
27
- }
28
-
29
- /**
30
- * Initialize the pg-boss instance
31
- */
32
- async init(): Promise<void> {
33
- if (!this.initialized) {
34
- await this.pgBoss.start()
35
- this.initialized = true
36
- }
37
- }
38
-
39
- /**
40
- * Get the queue service for publishing jobs
41
- */
42
- getQueueService(): PgBossQueueService {
43
- if (!this.queueService) {
44
- this.queueService = new PgBossQueueService(this.pgBoss)
45
- }
46
- return this.queueService
47
- }
48
-
49
- /**
50
- * Get the queue workers for processing jobs
51
- */
52
- getQueueWorkers(): PgBossQueueWorkers {
53
- if (!this.queueWorkers) {
54
- this.queueWorkers = new PgBossQueueWorkers(this.pgBoss)
55
- }
56
- return this.queueWorkers
57
- }
58
-
59
- /**
60
- * Get the scheduler service for managing recurring tasks
61
- */
62
- getSchedulerService(): PgBossSchedulerService {
63
- if (!this.schedulerService) {
64
- this.schedulerService = new PgBossSchedulerService(this.pgBoss)
65
- }
66
- return this.schedulerService
67
- }
68
-
69
- /**
70
- * Close all services and the pg-boss connection
71
- */
72
- async close(): Promise<void> {
73
- if (this.initialized) {
74
- await this.pgBoss.stop()
75
- this.initialized = false
76
- }
77
- }
78
-
79
- /**
80
- * Alias for close() - used by stopSingletonServices
81
- */
82
- async stop(): Promise<void> {
83
- await this.close()
84
- }
85
- }
package/src/utils.ts DELETED
@@ -1,87 +0,0 @@
1
- import type { QueueJob, QueueJobStatus } from '@pikku/core/queue'
2
- import type { PgBoss, JobWithMetadata } from 'pg-boss'
3
-
4
- /**
5
- * Map pg-boss job state to our queue job status
6
- */
7
- const mapPgBossStateToStatus = (state: string): QueueJobStatus => {
8
- switch (state) {
9
- case 'created':
10
- case 'retry':
11
- return 'waiting'
12
- case 'active':
13
- return 'active'
14
- case 'completed':
15
- return 'completed'
16
- case 'failed':
17
- return 'failed'
18
- case 'cancelled':
19
- return 'failed'
20
- default:
21
- return 'waiting'
22
- }
23
- }
24
-
25
- /**
26
- * Map pg-boss job to QueueJob interface
27
- */
28
- export const mapPgBossJobToQueueJob = <In, Out>(
29
- pgBossJob: JobWithMetadata<In>,
30
- pgBossInstance: PgBoss
31
- ): QueueJob<In, Out> => {
32
- return {
33
- queueName: pgBossJob.name,
34
- id: pgBossJob.id || '',
35
- data: pgBossJob.data,
36
- status: async () => {
37
- const jobWithState = await pgBossInstance.getJobById(
38
- pgBossJob.name,
39
- pgBossJob.id
40
- )
41
- return mapPgBossStateToStatus(jobWithState?.state || 'created')
42
- },
43
- waitForCompletion: async (ttl?: number): Promise<Out> => {
44
- // pg-boss doesn't have a built-in waitForCompletion, so we'll poll
45
- const startTime = Date.now()
46
- const timeout = ttl || 30000 // 30 seconds default
47
-
48
- while (Date.now() - startTime < timeout) {
49
- const job = await pgBossInstance.getJobById(
50
- pgBossJob.name,
51
- pgBossJob.id
52
- )
53
- if (job?.state === 'completed') {
54
- return job.output as Out
55
- }
56
- if (job?.state === 'failed' || job?.state === 'cancelled') {
57
- throw new Error(`Job failed: ${JSON.stringify(job.output)}`)
58
- }
59
- // Wait 100ms before polling again
60
- await new Promise((resolve) => setTimeout(resolve, 100))
61
- }
62
-
63
- throw new Error('Job did not complete within timeout')
64
- },
65
- metadata: () => {
66
- return {
67
- progress: 0, // pg-boss doesn't have built-in progress tracking
68
- attemptsMade: pgBossJob.retryCount || 0,
69
- maxAttempts: pgBossJob.retryLimit || 1,
70
- createdAt: pgBossJob.createdOn
71
- ? new Date(pgBossJob.createdOn)
72
- : new Date(),
73
- processedAt: pgBossJob.startedOn
74
- ? new Date(pgBossJob.startedOn)
75
- : undefined,
76
- completedAt: pgBossJob.completedOn
77
- ? new Date(pgBossJob.completedOn)
78
- : undefined,
79
- failedAt:
80
- pgBossJob.state === 'failed' && pgBossJob.completedOn
81
- ? new Date(pgBossJob.completedOn)
82
- : undefined,
83
- error: (pgBossJob.data as any).error,
84
- }
85
- },
86
- }
87
- }
package/tsconfig.json DELETED
@@ -1,17 +0,0 @@
1
- {
2
- "extends": "../../tsconfig.json",
3
- "compilerOptions": {
4
- "rootDir": "./src",
5
- "module": "Node18",
6
- "outDir": "dist",
7
- "target": "esnext",
8
- "declaration": true
9
- },
10
- "include": ["src/**/*.ts"],
11
- "exclude": ["**/*.test.ts", "node_modules"],
12
- "references": [
13
- {
14
- "path": "../../core/tsconfig.json"
15
- }
16
- ]
17
- }
@@ -1 +0,0 @@
1
- {"root":["./src/index.ts","./src/pg-boss-queue-service.ts","./src/pg-boss-queue-worker.ts","./src/pg-boss-scheduler-service.ts","./src/pg-boss-service-factory.ts","./src/utils.ts"],"version":"6.0.3"}