@open-mercato/cli 0.6.6-develop.5586.1.c9ed1d68a8 → 0.6.6-develop.5594.1.30cd738303
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/lib/events-single-delivery.js +27 -0
- package/dist/lib/events-single-delivery.js.map +7 -0
- package/dist/lib/worker-connection-budget.js +63 -0
- package/dist/lib/worker-connection-budget.js.map +7 -0
- package/dist/mercato.js +43 -2
- package/dist/mercato.js.map +2 -2
- package/package.json +5 -5
- package/src/__tests__/mercato.test.ts +23 -0
- package/src/lib/__tests__/events-single-delivery.test.ts +68 -0
- package/src/lib/__tests__/worker-connection-budget.test.ts +115 -0
- package/src/lib/events-single-delivery.ts +69 -0
- package/src/lib/worker-connection-budget.ts +131 -0
- package/src/mercato.ts +81 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/cli",
|
|
3
|
-
"version": "0.6.6-develop.
|
|
3
|
+
"version": "0.6.6-develop.5594.1.30cd738303",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"exports": {
|
|
@@ -59,8 +59,8 @@
|
|
|
59
59
|
"@mikro-orm/decorators": "^7.1.4",
|
|
60
60
|
"@mikro-orm/migrations": "^7.1.4",
|
|
61
61
|
"@mikro-orm/postgresql": "^7.1.4",
|
|
62
|
-
"@open-mercato/queue": "0.6.6-develop.
|
|
63
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
62
|
+
"@open-mercato/queue": "0.6.6-develop.5594.1.30cd738303",
|
|
63
|
+
"@open-mercato/shared": "0.6.6-develop.5594.1.30cd738303",
|
|
64
64
|
"cross-spawn": "^7.0.6",
|
|
65
65
|
"pg": "8.21.0",
|
|
66
66
|
"semver": "^7.8.4",
|
|
@@ -70,10 +70,10 @@
|
|
|
70
70
|
"typescript": "^6.0.3"
|
|
71
71
|
},
|
|
72
72
|
"peerDependencies": {
|
|
73
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
73
|
+
"@open-mercato/shared": "0.6.6-develop.5594.1.30cd738303"
|
|
74
74
|
},
|
|
75
75
|
"devDependencies": {
|
|
76
|
-
"@open-mercato/shared": "0.6.6-develop.
|
|
76
|
+
"@open-mercato/shared": "0.6.6-develop.5594.1.30cd738303",
|
|
77
77
|
"@types/jest": "^30.0.0",
|
|
78
78
|
"jest": "^30.4.2",
|
|
79
79
|
"ts-jest": "^29.4.11"
|
|
@@ -642,6 +642,8 @@ describe('server dev managed process exits', () => {
|
|
|
642
642
|
const originalLazy = process.env.OM_AUTO_SPAWN_WORKERS_LAZY
|
|
643
643
|
const originalLazyScheduler = process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
644
644
|
const originalGenerateWatchMode = process.env.OM_DEV_GENERATE_WATCH_MODE
|
|
645
|
+
const originalSingleDelivery = process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
646
|
+
const originalExternalWorker = process.env.OM_EVENTS_EXTERNAL_WORKER
|
|
645
647
|
|
|
646
648
|
beforeEach(() => {
|
|
647
649
|
jest.restoreAllMocks()
|
|
@@ -650,6 +652,12 @@ describe('server dev managed process exits', () => {
|
|
|
650
652
|
process.env.AUTO_SPAWN_WORKERS = 'true'
|
|
651
653
|
delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY
|
|
652
654
|
delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
655
|
+
// These tests toggle AUTO_SPAWN_WORKERS to exercise scheduler/Next exits, not
|
|
656
|
+
// the events single-delivery guard. Acknowledge an external events worker so
|
|
657
|
+
// the (default-on) guard stays quiet, and start each test from a clean
|
|
658
|
+
// single-delivery env since the guard rewrites it in place.
|
|
659
|
+
delete process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
660
|
+
process.env.OM_EVENTS_EXTERNAL_WORKER = 'true'
|
|
653
661
|
// These tests stub the resolver to an empty object; the in-process
|
|
654
662
|
// generate watcher's default checksum function would error on the
|
|
655
663
|
// missing methods. Force the legacy out-of-process mode so the
|
|
@@ -680,6 +688,10 @@ describe('server dev managed process exits', () => {
|
|
|
680
688
|
else process.env.OM_AUTO_SPAWN_WORKERS_LAZY = originalLazy
|
|
681
689
|
if (originalLazyScheduler === undefined) delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
682
690
|
else process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY = originalLazyScheduler
|
|
691
|
+
if (originalSingleDelivery === undefined) delete process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
692
|
+
else process.env.OM_EVENTS_SINGLE_DELIVERY = originalSingleDelivery
|
|
693
|
+
if (originalExternalWorker === undefined) delete process.env.OM_EVENTS_EXTERNAL_WORKER
|
|
694
|
+
else process.env.OM_EVENTS_EXTERNAL_WORKER = originalExternalWorker
|
|
683
695
|
})
|
|
684
696
|
|
|
685
697
|
it('skips scheduler auto-start when the module is not enabled', async () => {
|
|
@@ -909,6 +921,8 @@ describe('server start managed process exits', () => {
|
|
|
909
921
|
const originalAutoSpawnWorkers = process.env.AUTO_SPAWN_WORKERS
|
|
910
922
|
const originalLazy = process.env.OM_AUTO_SPAWN_WORKERS_LAZY
|
|
911
923
|
const originalLazyScheduler = process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
924
|
+
const originalSingleDelivery = process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
925
|
+
const originalExternalWorker = process.env.OM_EVENTS_EXTERNAL_WORKER
|
|
912
926
|
|
|
913
927
|
beforeEach(() => {
|
|
914
928
|
jest.restoreAllMocks()
|
|
@@ -917,6 +931,11 @@ describe('server start managed process exits', () => {
|
|
|
917
931
|
process.env.AUTO_SPAWN_WORKERS = 'true'
|
|
918
932
|
delete process.env.OM_AUTO_SPAWN_WORKERS_LAZY
|
|
919
933
|
delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
934
|
+
// Not exercising the events single-delivery guard here — acknowledge an
|
|
935
|
+
// external worker so the default-on guard stays quiet, and reset the
|
|
936
|
+
// guard-rewritten single-delivery env between tests.
|
|
937
|
+
delete process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
938
|
+
process.env.OM_EVENTS_EXTERNAL_WORKER = 'true'
|
|
920
939
|
})
|
|
921
940
|
|
|
922
941
|
afterEach(() => {
|
|
@@ -938,6 +957,10 @@ describe('server start managed process exits', () => {
|
|
|
938
957
|
else process.env.OM_AUTO_SPAWN_WORKERS_LAZY = originalLazy
|
|
939
958
|
if (originalLazyScheduler === undefined) delete process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY
|
|
940
959
|
else process.env.OM_AUTO_SPAWN_SCHEDULER_LAZY = originalLazyScheduler
|
|
960
|
+
if (originalSingleDelivery === undefined) delete process.env.OM_EVENTS_SINGLE_DELIVERY
|
|
961
|
+
else process.env.OM_EVENTS_SINGLE_DELIVERY = originalSingleDelivery
|
|
962
|
+
if (originalExternalWorker === undefined) delete process.env.OM_EVENTS_EXTERNAL_WORKER
|
|
963
|
+
else process.env.OM_EVENTS_EXTERNAL_WORKER = originalExternalWorker
|
|
941
964
|
})
|
|
942
965
|
|
|
943
966
|
it('skips scheduler auto-start when the module is not enabled', async () => {
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import {
|
|
2
|
+
applyEventsSingleDeliveryGuard,
|
|
3
|
+
reconcileEventsSingleDelivery,
|
|
4
|
+
} from '../events-single-delivery'
|
|
5
|
+
|
|
6
|
+
describe('reconcileEventsSingleDelivery', () => {
|
|
7
|
+
it('keeps single-delivery on when workers auto-spawn (default request)', () => {
|
|
8
|
+
expect(reconcileEventsSingleDelivery({}, 'eager')).toEqual({ effective: true })
|
|
9
|
+
expect(reconcileEventsSingleDelivery({}, 'lazy')).toEqual({ effective: true })
|
|
10
|
+
})
|
|
11
|
+
|
|
12
|
+
it('falls back to inline dual-dispatch (with a warning) when no worker runs', () => {
|
|
13
|
+
const result = reconcileEventsSingleDelivery({}, 'off')
|
|
14
|
+
expect(result.effective).toBe(false)
|
|
15
|
+
expect(result.warning).toContain('OM_EVENTS_SINGLE_DELIVERY')
|
|
16
|
+
expect(result.warning).toContain('OM_EVENTS_EXTERNAL_WORKER')
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
it('keeps single-delivery on with no auto-spawn when an external worker is acknowledged', () => {
|
|
20
|
+
expect(
|
|
21
|
+
reconcileEventsSingleDelivery({ OM_EVENTS_EXTERNAL_WORKER: 'true' }, 'off'),
|
|
22
|
+
).toEqual({ effective: true })
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
it('respects an explicit legacy opt-out regardless of worker availability', () => {
|
|
26
|
+
expect(
|
|
27
|
+
reconcileEventsSingleDelivery({ OM_EVENTS_SINGLE_DELIVERY: 'false' }, 'eager'),
|
|
28
|
+
).toEqual({ effective: false })
|
|
29
|
+
})
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
describe('applyEventsSingleDeliveryGuard', () => {
|
|
33
|
+
it('writes the reconciled value into both process and runtime env', () => {
|
|
34
|
+
const processEnv: NodeJS.ProcessEnv = {}
|
|
35
|
+
const runtimeEnv: NodeJS.ProcessEnv = {}
|
|
36
|
+
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
37
|
+
|
|
38
|
+
const result = applyEventsSingleDeliveryGuard({
|
|
39
|
+
processEnv,
|
|
40
|
+
runtimeEnv,
|
|
41
|
+
autoSpawnWorkersMode: 'off',
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
expect(result.effective).toBe(false)
|
|
45
|
+
expect(processEnv.OM_EVENTS_SINGLE_DELIVERY).toBe('false')
|
|
46
|
+
expect(runtimeEnv.OM_EVENTS_SINGLE_DELIVERY).toBe('false')
|
|
47
|
+
expect(errorSpy).toHaveBeenCalledTimes(1)
|
|
48
|
+
errorSpy.mockRestore()
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
it('writes true (and stays quiet) when workers are available', () => {
|
|
52
|
+
const processEnv: NodeJS.ProcessEnv = {}
|
|
53
|
+
const runtimeEnv: NodeJS.ProcessEnv = {}
|
|
54
|
+
const errorSpy = jest.spyOn(console, 'error').mockImplementation(() => {})
|
|
55
|
+
|
|
56
|
+
const result = applyEventsSingleDeliveryGuard({
|
|
57
|
+
processEnv,
|
|
58
|
+
runtimeEnv,
|
|
59
|
+
autoSpawnWorkersMode: 'eager',
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
expect(result.effective).toBe(true)
|
|
63
|
+
expect(processEnv.OM_EVENTS_SINGLE_DELIVERY).toBe('true')
|
|
64
|
+
expect(runtimeEnv.OM_EVENTS_SINGLE_DELIVERY).toBe('true')
|
|
65
|
+
expect(errorSpy).not.toHaveBeenCalled()
|
|
66
|
+
errorSpy.mockRestore()
|
|
67
|
+
})
|
|
68
|
+
})
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import {
|
|
2
|
+
planWorkerConcurrency,
|
|
3
|
+
resolveWorkerConnectionBudget,
|
|
4
|
+
} from '../worker-connection-budget'
|
|
5
|
+
|
|
6
|
+
describe('resolveWorkerConnectionBudget', () => {
|
|
7
|
+
it('defaults to the pool max when no override is set', () => {
|
|
8
|
+
expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, 20)).toBe(20)
|
|
9
|
+
})
|
|
10
|
+
|
|
11
|
+
it('honors a positive OM_WORKERS_DB_CONNECTION_BUDGET override', () => {
|
|
12
|
+
expect(
|
|
13
|
+
resolveWorkerConnectionBudget(
|
|
14
|
+
{ OM_WORKERS_DB_CONNECTION_BUDGET: '8' } as unknown as NodeJS.ProcessEnv,
|
|
15
|
+
20,
|
|
16
|
+
),
|
|
17
|
+
).toBe(8)
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
it('falls back to the pool max for non-numeric or non-positive overrides', () => {
|
|
21
|
+
for (const value of ['0', '-3', 'abc', '']) {
|
|
22
|
+
expect(
|
|
23
|
+
resolveWorkerConnectionBudget(
|
|
24
|
+
{ OM_WORKERS_DB_CONNECTION_BUDGET: value } as unknown as NodeJS.ProcessEnv,
|
|
25
|
+
16,
|
|
26
|
+
),
|
|
27
|
+
).toBe(16)
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
it('clamps an invalid pool max to at least 1', () => {
|
|
32
|
+
expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, 0)).toBe(1)
|
|
33
|
+
expect(resolveWorkerConnectionBudget({} as NodeJS.ProcessEnv, Number.NaN)).toBe(1)
|
|
34
|
+
})
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
describe('planWorkerConcurrency', () => {
|
|
38
|
+
it('passes concurrency through untouched when it fits the budget', () => {
|
|
39
|
+
const plan = planWorkerConcurrency(
|
|
40
|
+
[
|
|
41
|
+
{ queue: 'events', concurrency: 5 },
|
|
42
|
+
{ queue: 'vector-indexing', concurrency: 2 },
|
|
43
|
+
{ queue: 'fulltext-indexing', concurrency: 2 },
|
|
44
|
+
],
|
|
45
|
+
20,
|
|
46
|
+
)
|
|
47
|
+
expect(plan.clamped).toBe(false)
|
|
48
|
+
expect(plan.totalEffective).toBe(9)
|
|
49
|
+
expect(plan.entries.map((entry) => entry.effective)).toEqual([5, 2, 2])
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
it('scales down to exactly the budget when over-subscribed', () => {
|
|
53
|
+
const plan = planWorkerConcurrency(
|
|
54
|
+
[
|
|
55
|
+
{ queue: 'events', concurrency: 10 },
|
|
56
|
+
{ queue: 'vector-indexing', concurrency: 10 },
|
|
57
|
+
{ queue: 'fulltext-indexing', concurrency: 10 },
|
|
58
|
+
],
|
|
59
|
+
12,
|
|
60
|
+
)
|
|
61
|
+
expect(plan.clamped).toBe(true)
|
|
62
|
+
expect(plan.belowQueueFloor).toBe(false)
|
|
63
|
+
expect(plan.totalEffective).toBe(12)
|
|
64
|
+
for (const entry of plan.entries) {
|
|
65
|
+
expect(entry.effective).toBeGreaterThanOrEqual(1)
|
|
66
|
+
expect(entry.effective).toBeLessThanOrEqual(entry.requested)
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
it('keeps a floor of 1 per queue and never exceeds the request', () => {
|
|
71
|
+
const plan = planWorkerConcurrency(
|
|
72
|
+
[
|
|
73
|
+
{ queue: 'events', concurrency: 16 },
|
|
74
|
+
{ queue: 'vector-indexing', concurrency: 2 },
|
|
75
|
+
{ queue: 'fulltext-indexing', concurrency: 2 },
|
|
76
|
+
],
|
|
77
|
+
6,
|
|
78
|
+
)
|
|
79
|
+
const byQueue = Object.fromEntries(plan.entries.map((entry) => [entry.queue, entry.effective]))
|
|
80
|
+
expect(byQueue['vector-indexing']).toBeGreaterThanOrEqual(1)
|
|
81
|
+
expect(byQueue['fulltext-indexing']).toBeGreaterThanOrEqual(1)
|
|
82
|
+
// The largest requester absorbs most of the remaining budget.
|
|
83
|
+
expect(byQueue['events']).toBeGreaterThan(byQueue['vector-indexing'])
|
|
84
|
+
expect(plan.totalEffective).toBe(6)
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
it('flags belowQueueFloor when the budget is smaller than the queue count', () => {
|
|
88
|
+
const plan = planWorkerConcurrency(
|
|
89
|
+
[
|
|
90
|
+
{ queue: 'a', concurrency: 4 },
|
|
91
|
+
{ queue: 'b', concurrency: 4 },
|
|
92
|
+
{ queue: 'c', concurrency: 4 },
|
|
93
|
+
],
|
|
94
|
+
2,
|
|
95
|
+
)
|
|
96
|
+
expect(plan.clamped).toBe(true)
|
|
97
|
+
expect(plan.belowQueueFloor).toBe(true)
|
|
98
|
+
// Floor of 1 wins even though it exceeds the budget.
|
|
99
|
+
expect(plan.entries.every((entry) => entry.effective === 1)).toBe(true)
|
|
100
|
+
expect(plan.totalEffective).toBe(3)
|
|
101
|
+
})
|
|
102
|
+
|
|
103
|
+
it('treats zero/negative requested concurrency as a floor of 1', () => {
|
|
104
|
+
const plan = planWorkerConcurrency(
|
|
105
|
+
[
|
|
106
|
+
{ queue: 'events', concurrency: 0 },
|
|
107
|
+
{ queue: 'vector-indexing', concurrency: -5 },
|
|
108
|
+
],
|
|
109
|
+
20,
|
|
110
|
+
)
|
|
111
|
+
expect(plan.entries.map((entry) => entry.requested)).toEqual([1, 1])
|
|
112
|
+
expect(plan.totalEffective).toBe(2)
|
|
113
|
+
expect(plan.clamped).toBe(false)
|
|
114
|
+
})
|
|
115
|
+
})
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { parseBooleanWithDefault } from '@open-mercato/shared/lib/boolean'
|
|
2
|
+
import type { AutoSpawnWorkersMode } from './auto-spawn-workers'
|
|
3
|
+
|
|
4
|
+
// Keep these env names in sync with the runtime source of truth,
|
|
5
|
+
// `@open-mercato/events/single-delivery`. The CLI cannot import the events
|
|
6
|
+
// package (no dependency edge), so the reconcile logic is mirrored here for the
|
|
7
|
+
// server-bootstrap guard only; the bus and worker own the runtime behavior.
|
|
8
|
+
const EVENTS_SINGLE_DELIVERY_ENV = 'OM_EVENTS_SINGLE_DELIVERY'
|
|
9
|
+
const EVENTS_EXTERNAL_WORKER_ENV = 'OM_EVENTS_EXTERNAL_WORKER'
|
|
10
|
+
|
|
11
|
+
type EnvSource = NodeJS.ProcessEnv | Record<string, string | undefined>
|
|
12
|
+
|
|
13
|
+
export type SingleDeliveryReconciliation = {
|
|
14
|
+
effective: boolean
|
|
15
|
+
warning?: string
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Server-bootstrap guard for the default-on single-delivery dispatch.
|
|
20
|
+
*
|
|
21
|
+
* With single-delivery on, persistent subscribers are skipped inline and ONLY
|
|
22
|
+
* the events worker dispatches them. A process that runs NO events worker
|
|
23
|
+
* (auto-spawn off and no acknowledged external worker) would skip those
|
|
24
|
+
* subscribers with nothing to drain the queue — silently dropping notifications,
|
|
25
|
+
* queued emails, and indexing. This fails safe by disabling single-delivery for
|
|
26
|
+
* such a process (back to inline dual-dispatch) and returning a loud warning.
|
|
27
|
+
*
|
|
28
|
+
* Transient worker downtime is NOT this guard's concern: the durable queue holds
|
|
29
|
+
* the job until a worker returns. Only the "no worker at all" config is guarded.
|
|
30
|
+
*/
|
|
31
|
+
export function reconcileEventsSingleDelivery(
|
|
32
|
+
env: EnvSource,
|
|
33
|
+
autoSpawnWorkersMode: AutoSpawnWorkersMode,
|
|
34
|
+
): SingleDeliveryReconciliation {
|
|
35
|
+
const requested = parseBooleanWithDefault(env[EVENTS_SINGLE_DELIVERY_ENV], true)
|
|
36
|
+
if (!requested) return { effective: false }
|
|
37
|
+
const externalWorker = parseBooleanWithDefault(env[EVENTS_EXTERNAL_WORKER_ENV], false)
|
|
38
|
+
const workersAvailable = autoSpawnWorkersMode !== 'off' || externalWorker
|
|
39
|
+
if (workersAvailable) return { effective: true }
|
|
40
|
+
return {
|
|
41
|
+
effective: false,
|
|
42
|
+
warning:
|
|
43
|
+
`[events] ${EVENTS_SINGLE_DELIVERY_ENV} is on (default) but this process auto-spawns no events worker ` +
|
|
44
|
+
`(AUTO_SPAWN_WORKERS=off) and ${EVENTS_EXTERNAL_WORKER_ENV} is not set. Persistent subscribers would be ` +
|
|
45
|
+
`skipped inline with nothing to drain the queue, silently dropping notifications, queued emails, and ` +
|
|
46
|
+
`indexing. Falling back to legacy inline dual-dispatch for safety. To keep single-delivery, run an events ` +
|
|
47
|
+
`worker (\`mercato queue worker events\`) and set ${EVENTS_EXTERNAL_WORKER_ENV}=true, or enable ` +
|
|
48
|
+
`AUTO_SPAWN_WORKERS.`,
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Applies {@link reconcileEventsSingleDelivery} and writes the effective value
|
|
54
|
+
* into both the current process env (for the in-process app/bus) and the spawned
|
|
55
|
+
* worker/child env (so a child worker reads the same value the bus does). Logs
|
|
56
|
+
* the warning once when single-delivery is disabled for safety.
|
|
57
|
+
*/
|
|
58
|
+
export function applyEventsSingleDeliveryGuard(args: {
|
|
59
|
+
processEnv: NodeJS.ProcessEnv
|
|
60
|
+
runtimeEnv: NodeJS.ProcessEnv
|
|
61
|
+
autoSpawnWorkersMode: AutoSpawnWorkersMode
|
|
62
|
+
}): SingleDeliveryReconciliation {
|
|
63
|
+
const result = reconcileEventsSingleDelivery(args.processEnv, args.autoSpawnWorkersMode)
|
|
64
|
+
if (result.warning) console.error(result.warning)
|
|
65
|
+
const value = result.effective ? 'true' : 'false'
|
|
66
|
+
args.processEnv[EVENTS_SINGLE_DELIVERY_ENV] = value
|
|
67
|
+
args.runtimeEnv[EVENTS_SINGLE_DELIVERY_ENV] = value
|
|
68
|
+
return result
|
|
69
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bound the DB-connection demand of background queue workers.
|
|
3
|
+
*
|
|
4
|
+
* Since #2970/#3011 each worker job builds its own request container, so its own
|
|
5
|
+
* `EntityManager` checks out a dedicated pooled DB connection for the job's
|
|
6
|
+
* duration. The peak connection demand of `worker --all` is therefore the sum of
|
|
7
|
+
* every queue's concurrency. Nothing previously bounded that sum against the DB
|
|
8
|
+
* pool (`DB_POOL_MAX`) or the database's global `max_connections`, so a storm of
|
|
9
|
+
* slow/failing jobs could over-subscribe connections — thrashing on the worker's
|
|
10
|
+
* own acquire timeout and, across the worker + web processes, starving the
|
|
11
|
+
* request/onboarding path that shares the same database.
|
|
12
|
+
*
|
|
13
|
+
* These helpers derive a per-queue effective-concurrency plan whose total never
|
|
14
|
+
* exceeds a connection budget (defaulting to the resolved pool max), while
|
|
15
|
+
* guaranteeing every queue keeps at least one worker.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export type WorkerQueueConcurrency = {
|
|
19
|
+
queue: string
|
|
20
|
+
concurrency: number
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type WorkerConcurrencyPlanEntry = {
|
|
24
|
+
queue: string
|
|
25
|
+
requested: number
|
|
26
|
+
effective: number
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export type WorkerConcurrencyPlan = {
|
|
30
|
+
/** Connection budget the plan was fitted to. */
|
|
31
|
+
budget: number
|
|
32
|
+
/** Sum of requested concurrency across all queues. */
|
|
33
|
+
totalRequested: number
|
|
34
|
+
/** Sum of effective concurrency the plan grants. */
|
|
35
|
+
totalEffective: number
|
|
36
|
+
/** True when any queue's concurrency was reduced to fit the budget. */
|
|
37
|
+
clamped: boolean
|
|
38
|
+
/**
|
|
39
|
+
* True when the budget is smaller than the number of queues, so the per-queue
|
|
40
|
+
* floor of 1 forces the total above the budget. The caller should surface this
|
|
41
|
+
* as a misconfiguration (raise `DB_POOL_MAX` or the budget).
|
|
42
|
+
*/
|
|
43
|
+
belowQueueFloor: boolean
|
|
44
|
+
entries: WorkerConcurrencyPlanEntry[]
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function toPositiveInt(value: number | undefined, fallback: number): number {
|
|
48
|
+
if (value === undefined || !Number.isFinite(value)) return fallback
|
|
49
|
+
const floored = Math.floor(value)
|
|
50
|
+
return floored > 0 ? floored : fallback
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Resolve the connection budget for background workers. Defaults to the resolved
|
|
55
|
+
* DB pool max so the worker never tries to use more connections than its own pool
|
|
56
|
+
* can hand out; override with `OM_WORKERS_DB_CONNECTION_BUDGET` (positive int).
|
|
57
|
+
*/
|
|
58
|
+
export function resolveWorkerConnectionBudget(
|
|
59
|
+
env: NodeJS.ProcessEnv,
|
|
60
|
+
poolMax: number,
|
|
61
|
+
): number {
|
|
62
|
+
const safePoolMax = toPositiveInt(poolMax, 1)
|
|
63
|
+
const raw = env.OM_WORKERS_DB_CONNECTION_BUDGET
|
|
64
|
+
if (!raw) return safePoolMax
|
|
65
|
+
const parsed = Number.parseInt(raw, 10)
|
|
66
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : safePoolMax
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Fit per-queue concurrency to a connection budget.
|
|
71
|
+
*
|
|
72
|
+
* - Every queue keeps a floor of 1 worker (no queue is starved).
|
|
73
|
+
* - No queue exceeds its requested concurrency.
|
|
74
|
+
* - When the sum exceeds the budget, leftover capacity is distributed greedily to
|
|
75
|
+
* the queues furthest below their request, so the total matches the budget
|
|
76
|
+
* exactly (when `budget >= queueCount`) and the allocation is deterministic.
|
|
77
|
+
*/
|
|
78
|
+
export function planWorkerConcurrency(
|
|
79
|
+
queues: WorkerQueueConcurrency[],
|
|
80
|
+
budget: number,
|
|
81
|
+
): WorkerConcurrencyPlan {
|
|
82
|
+
const safeBudget = toPositiveInt(budget, 1)
|
|
83
|
+
const requested = queues.map((entry) => ({
|
|
84
|
+
queue: entry.queue,
|
|
85
|
+
requested: toPositiveInt(entry.concurrency, 1),
|
|
86
|
+
}))
|
|
87
|
+
const totalRequested = requested.reduce((sum, entry) => sum + entry.requested, 0)
|
|
88
|
+
|
|
89
|
+
if (totalRequested <= safeBudget) {
|
|
90
|
+
return {
|
|
91
|
+
budget: safeBudget,
|
|
92
|
+
totalRequested,
|
|
93
|
+
totalEffective: totalRequested,
|
|
94
|
+
clamped: false,
|
|
95
|
+
belowQueueFloor: false,
|
|
96
|
+
entries: requested.map((entry) => ({ ...entry, effective: entry.requested })),
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// Floor every queue at 1, then water-fill the remaining budget to the queues
|
|
101
|
+
// with the largest unmet request first.
|
|
102
|
+
const effective = requested.map(() => 1)
|
|
103
|
+
let used = effective.length
|
|
104
|
+
while (used < safeBudget) {
|
|
105
|
+
let bestIndex = -1
|
|
106
|
+
let bestDeficit = 0
|
|
107
|
+
for (let index = 0; index < requested.length; index += 1) {
|
|
108
|
+
const deficit = requested[index].requested - effective[index]
|
|
109
|
+
if (deficit > bestDeficit) {
|
|
110
|
+
bestDeficit = deficit
|
|
111
|
+
bestIndex = index
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (bestIndex === -1) break
|
|
115
|
+
effective[bestIndex] += 1
|
|
116
|
+
used += 1
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const totalEffective = effective.reduce((sum, value) => sum + value, 0)
|
|
120
|
+
return {
|
|
121
|
+
budget: safeBudget,
|
|
122
|
+
totalRequested,
|
|
123
|
+
totalEffective,
|
|
124
|
+
clamped: true,
|
|
125
|
+
belowQueueFloor: requested.length > safeBudget,
|
|
126
|
+
entries: requested.map((entry, index) => ({
|
|
127
|
+
...entry,
|
|
128
|
+
effective: effective[index],
|
|
129
|
+
})),
|
|
130
|
+
}
|
|
131
|
+
}
|
package/src/mercato.ts
CHANGED
|
@@ -15,7 +15,13 @@ import {
|
|
|
15
15
|
resolveLazyRestart,
|
|
16
16
|
} from './lib/auto-spawn-workers'
|
|
17
17
|
import { startLazyWorkerSupervisor } from './lib/queue-worker-supervisor'
|
|
18
|
+
import { applyEventsSingleDeliveryGuard } from './lib/events-single-delivery'
|
|
18
19
|
import { createPerJobWorkerHandler } from './lib/worker-job-handler'
|
|
20
|
+
import {
|
|
21
|
+
planWorkerConcurrency,
|
|
22
|
+
resolveWorkerConnectionBudget,
|
|
23
|
+
type WorkerConcurrencyPlan,
|
|
24
|
+
} from './lib/worker-connection-budget'
|
|
19
25
|
import {
|
|
20
26
|
resolveAutoSpawnSchedulerMode,
|
|
21
27
|
resolveLazySchedulerPollMs,
|
|
@@ -525,6 +531,46 @@ function formatQueueWorkerLabel(queueNames: string[]): string {
|
|
|
525
531
|
return `Queue worker (${preview})`
|
|
526
532
|
}
|
|
527
533
|
|
|
534
|
+
/**
|
|
535
|
+
* Fit the requested per-queue worker concurrency to the worker process's DB
|
|
536
|
+
* connection budget and log the resolved plan. Since each job runs in its own
|
|
537
|
+
* request container (one pooled connection per in-flight job), the sum of worker
|
|
538
|
+
* concurrency is the worker's peak connection demand — it MUST stay within the
|
|
539
|
+
* pool so background jobs cannot starve the request/onboarding path that shares
|
|
540
|
+
* the same database.
|
|
541
|
+
*/
|
|
542
|
+
async function resolveWorkerBudgetPlan(
|
|
543
|
+
requestedByQueue: { queue: string; concurrency: number }[],
|
|
544
|
+
): Promise<WorkerConcurrencyPlan> {
|
|
545
|
+
const { resolvePoolConfig } = await import('@open-mercato/shared/lib/db/mikro')
|
|
546
|
+
const poolMax = resolvePoolConfig(process.env).poolMax
|
|
547
|
+
const budget = resolveWorkerConnectionBudget(process.env, poolMax)
|
|
548
|
+
const plan = planWorkerConcurrency(requestedByQueue, budget)
|
|
549
|
+
|
|
550
|
+
console.log(
|
|
551
|
+
`[worker] DB connection budget: ${plan.budget} (pool max ${poolMax}); ` +
|
|
552
|
+
`requested Σconcurrency ${plan.totalRequested}, effective ${plan.totalEffective}`,
|
|
553
|
+
)
|
|
554
|
+
if (plan.clamped) {
|
|
555
|
+
const perQueue = plan.entries
|
|
556
|
+
.map((entry) => `${entry.queue}=${entry.effective}/${entry.requested}`)
|
|
557
|
+
.join(', ')
|
|
558
|
+
console.warn(
|
|
559
|
+
`[worker] Worker concurrency clamped to fit the DB connection budget (${plan.budget}): ${perQueue}. ` +
|
|
560
|
+
`Raise DB_POOL_MAX or set OM_WORKERS_DB_CONNECTION_BUDGET to change this. ` +
|
|
561
|
+
`Keep web_pool_max + worker_pool_max + overhead <= Postgres max_connections.`,
|
|
562
|
+
)
|
|
563
|
+
}
|
|
564
|
+
if (plan.belowQueueFloor) {
|
|
565
|
+
console.warn(
|
|
566
|
+
`[worker] DB connection budget (${plan.budget}) is smaller than the number of queues ` +
|
|
567
|
+
`(${requestedByQueue.length}); every queue runs at concurrency 1 and total demand ` +
|
|
568
|
+
`(${plan.totalEffective}) still exceeds the budget. Raise DB_POOL_MAX.`,
|
|
569
|
+
)
|
|
570
|
+
}
|
|
571
|
+
return plan
|
|
572
|
+
}
|
|
573
|
+
|
|
528
574
|
function lookupModuleCommand(
|
|
529
575
|
allModules: Module[],
|
|
530
576
|
moduleName: string,
|
|
@@ -1520,12 +1566,31 @@ export async function run(argv = process.argv) {
|
|
|
1520
1566
|
}
|
|
1521
1567
|
|
|
1522
1568
|
const { createRequestContainer } = await import('@open-mercato/shared/lib/di/container')
|
|
1569
|
+
|
|
1570
|
+
// Fit Σconcurrency to the worker's DB connection budget before
|
|
1571
|
+
// starting any worker, so the per-job containers (one connection each)
|
|
1572
|
+
// can never over-subscribe the pool the request path shares.
|
|
1573
|
+
const requestedByQueue = discoveredQueues.map((queue) => {
|
|
1574
|
+
const queueWorkers = allWorkers.filter((w) => w.queue === queue)
|
|
1575
|
+
return {
|
|
1576
|
+
queue,
|
|
1577
|
+
concurrency: concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1),
|
|
1578
|
+
}
|
|
1579
|
+
})
|
|
1580
|
+
const budgetPlan = await resolveWorkerBudgetPlan(requestedByQueue)
|
|
1581
|
+
const effectiveByQueue = new Map(
|
|
1582
|
+
budgetPlan.entries.map((entry) => [entry.queue, entry.effective]),
|
|
1583
|
+
)
|
|
1584
|
+
|
|
1523
1585
|
console.log(`[worker] Starting workers for all queues: ${discoveredQueues.join(', ')}`)
|
|
1524
1586
|
|
|
1525
1587
|
// Start all queue workers in background mode
|
|
1526
1588
|
const workerPromises = discoveredQueues.map(async (queue) => {
|
|
1527
1589
|
const queueWorkers = allWorkers.filter((w) => w.queue === queue)
|
|
1528
|
-
const concurrency =
|
|
1590
|
+
const concurrency =
|
|
1591
|
+
effectiveByQueue.get(queue) ??
|
|
1592
|
+
concurrencyOverride ??
|
|
1593
|
+
Math.max(...queueWorkers.map((w) => w.concurrency), 1)
|
|
1529
1594
|
|
|
1530
1595
|
console.log(`[worker] Starting "${queue}" with ${queueWorkers.length} handler(s), concurrency: ${concurrency}`)
|
|
1531
1596
|
|
|
@@ -1552,7 +1617,11 @@ export async function run(argv = process.argv) {
|
|
|
1552
1617
|
if (queueWorkers.length > 0) {
|
|
1553
1618
|
// Use discovered workers
|
|
1554
1619
|
const { createRequestContainer } = await import('@open-mercato/shared/lib/di/container')
|
|
1555
|
-
const
|
|
1620
|
+
const requested = concurrencyOverride ?? Math.max(...queueWorkers.map((w) => w.concurrency), 1)
|
|
1621
|
+
// Bound a single-queue run to the connection budget too, so it can
|
|
1622
|
+
// never check out more pooled connections than the worker pool holds.
|
|
1623
|
+
const budgetPlan = await resolveWorkerBudgetPlan([{ queue: queueName!, concurrency: requested }])
|
|
1624
|
+
const concurrency = budgetPlan.entries[0]?.effective ?? requested
|
|
1556
1625
|
|
|
1557
1626
|
console.log(`[worker] Found ${queueWorkers.length} worker(s) for queue "${queueName}"`)
|
|
1558
1627
|
|
|
@@ -1993,6 +2062,12 @@ export async function run(argv = process.argv) {
|
|
|
1993
2062
|
envReloader.reload()
|
|
1994
2063
|
const runtimeEnv = buildServerProcessEnvironment(process.env)
|
|
1995
2064
|
const autoSpawnWorkersMode = resolveAutoSpawnWorkersMode(process.env)
|
|
2065
|
+
// Guard the default-on events single-delivery: if this process runs
|
|
2066
|
+
// no events worker, fall back to safe inline dual-dispatch so
|
|
2067
|
+
// persistent side effects are never silently dropped. Mutates both
|
|
2068
|
+
// process.env (in-process bus) and runtimeEnv (spawned workers) so
|
|
2069
|
+
// they agree.
|
|
2070
|
+
applyEventsSingleDeliveryGuard({ processEnv: process.env, runtimeEnv, autoSpawnWorkersMode })
|
|
1996
2071
|
const autoSpawnSchedulerMode = resolveAutoSpawnSchedulerMode(process.env)
|
|
1997
2072
|
const queueStrategy = process.env.QUEUE_STRATEGY || 'local'
|
|
1998
2073
|
const schedulerCommand = lookupModuleCommand(getCliModules(), 'scheduler', 'start')
|
|
@@ -2146,6 +2221,10 @@ export async function run(argv = process.argv) {
|
|
|
2146
2221
|
const autoSpawnSchedulerMode = resolveAutoSpawnSchedulerMode(process.env)
|
|
2147
2222
|
const queueStrategy = process.env.QUEUE_STRATEGY || 'local'
|
|
2148
2223
|
const runtimeEnv = buildServerProcessEnvironment(process.env)
|
|
2224
|
+
// Guard the default-on events single-delivery (see the dev `server`
|
|
2225
|
+
// command): fall back to safe inline dual-dispatch when this process
|
|
2226
|
+
// runs no events worker, keeping process.env and runtimeEnv in sync.
|
|
2227
|
+
applyEventsSingleDeliveryGuard({ processEnv: process.env, runtimeEnv, autoSpawnWorkersMode })
|
|
2149
2228
|
// Throws on single-instance strategies under a multi-instance topology,
|
|
2150
2229
|
// aborting before the start lock is acquired or any process is spawned.
|
|
2151
2230
|
assertSingleInstanceStrategies(runtimeEnv)
|