@open-mercato/queue 0.6.7-develop.6828.1.ab1620a63e → 0.6.7-develop.6842.1.8cd8a8883c
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/pending-probe.js +4 -2
- package/dist/pending-probe.js.map +2 -2
- package/dist/strategies/async.js +38 -8
- package/dist/strategies/async.js.map +2 -2
- package/dist/strategies/local.js +14 -7
- package/dist/strategies/local.js.map +2 -2
- package/dist/tracing.js +27 -0
- package/dist/tracing.js.map +7 -0
- package/dist/worker/runner.js +13 -0
- package/dist/worker/runner.js.map +2 -2
- package/jest.config.cjs +1 -0
- package/package.json +8 -3
- package/src/__tests__/async.strategy.test.ts +30 -5
- package/src/__tests__/async.telemetry.test.ts +123 -0
- package/src/__tests__/factory.test.ts +5 -1
- package/src/__tests__/pending-probe.test.ts +40 -1
- package/src/__tests__/tracing.test.ts +123 -0
- package/src/__tests__/worker-shutdown-telemetry.test.ts +80 -0
- package/src/pending-probe.ts +4 -2
- package/src/strategies/async.ts +67 -12
- package/src/strategies/local.ts +11 -6
- package/src/tracing.ts +58 -0
- package/src/types.ts +2 -0
- package/src/worker/runner.ts +24 -0
|
@@ -18,6 +18,7 @@ const workerOn = jest.fn()
|
|
|
18
18
|
|
|
19
19
|
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
20
20
|
getRedisUrlOrThrow: jest.fn(),
|
|
21
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
21
22
|
}))
|
|
22
23
|
|
|
23
24
|
jest.mock('bullmq', () => {
|
|
@@ -60,7 +61,7 @@ describe('Queue - async strategy', () => {
|
|
|
60
61
|
getRedisUrlOrThrowMock.mockReturnValue('rediss://default:secret@example.com:6380/1')
|
|
61
62
|
})
|
|
62
63
|
|
|
63
|
-
it('passes
|
|
64
|
+
it('passes parsed Redis connection fields to BullMQ for env-based async config', async () => {
|
|
64
65
|
const queue = createQueue<{ value: number }>('test-queue', 'async', {
|
|
65
66
|
concurrency: 3,
|
|
66
67
|
})
|
|
@@ -69,19 +70,35 @@ describe('Queue - async strategy', () => {
|
|
|
69
70
|
await queue.process(async () => {})
|
|
70
71
|
|
|
71
72
|
expect(queueCtor).toHaveBeenCalledWith('test-queue', {
|
|
72
|
-
connection: {
|
|
73
|
+
connection: {
|
|
74
|
+
host: 'example.com',
|
|
75
|
+
port: 6380,
|
|
76
|
+
username: 'default',
|
|
77
|
+
password: 'secret',
|
|
78
|
+
db: 1,
|
|
79
|
+
tls: {},
|
|
80
|
+
family: undefined,
|
|
81
|
+
},
|
|
73
82
|
})
|
|
74
83
|
expect(workerCtor).toHaveBeenCalledWith(
|
|
75
84
|
'test-queue',
|
|
76
85
|
expect.any(Function),
|
|
77
86
|
{
|
|
78
|
-
connection: {
|
|
87
|
+
connection: {
|
|
88
|
+
host: 'example.com',
|
|
89
|
+
port: 6380,
|
|
90
|
+
username: 'default',
|
|
91
|
+
password: 'secret',
|
|
92
|
+
db: 1,
|
|
93
|
+
tls: {},
|
|
94
|
+
family: undefined,
|
|
95
|
+
},
|
|
79
96
|
concurrency: 3,
|
|
80
97
|
},
|
|
81
98
|
)
|
|
82
99
|
})
|
|
83
100
|
|
|
84
|
-
it('preserves
|
|
101
|
+
it('preserves URL connection semantics when converting to BullMQ fields', async () => {
|
|
85
102
|
const queue = createQueue<{ value: number }>('test-queue', 'async', {
|
|
86
103
|
connection: {
|
|
87
104
|
url: 'rediss://user:secret@example.com:6380/4?family=6',
|
|
@@ -91,7 +108,15 @@ describe('Queue - async strategy', () => {
|
|
|
91
108
|
await queue.enqueue({ value: 42 })
|
|
92
109
|
|
|
93
110
|
expect(queueCtor).toHaveBeenCalledWith('test-queue', {
|
|
94
|
-
connection: {
|
|
111
|
+
connection: {
|
|
112
|
+
host: 'example.com',
|
|
113
|
+
port: 6380,
|
|
114
|
+
username: 'user',
|
|
115
|
+
password: 'secret',
|
|
116
|
+
db: 4,
|
|
117
|
+
tls: {},
|
|
118
|
+
family: 6,
|
|
119
|
+
},
|
|
95
120
|
})
|
|
96
121
|
})
|
|
97
122
|
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// An OTLP backend must be active for the async strategy to delegate tracing to
|
|
2
|
+
// bullmq-otel. Set before any module reads the (memoized) telemetry env, and
|
|
3
|
+
// restored afterwards — process.env is shared across test files in the same
|
|
4
|
+
// jest worker, and a leaked 'otlp' backend breaks sibling telemetry tests.
|
|
5
|
+
const originalTelemetryBackend = process.env.TELEMETRY_BACKEND
|
|
6
|
+
process.env.TELEMETRY_BACKEND = 'otlp'
|
|
7
|
+
|
|
8
|
+
afterAll(() => {
|
|
9
|
+
if (originalTelemetryBackend === undefined) delete process.env.TELEMETRY_BACKEND
|
|
10
|
+
else process.env.TELEMETRY_BACKEND = originalTelemetryBackend
|
|
11
|
+
})
|
|
12
|
+
|
|
13
|
+
import { createQueue } from '../factory'
|
|
14
|
+
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
15
|
+
import {
|
|
16
|
+
registerTelemetryRuntime,
|
|
17
|
+
resetTelemetryRuntime,
|
|
18
|
+
} from '@open-mercato/shared/lib/telemetry/runtime'
|
|
19
|
+
|
|
20
|
+
const queueCtor = jest.fn()
|
|
21
|
+
const workerCtor = jest.fn()
|
|
22
|
+
const queueAdd = jest.fn(async () => ({ id: 'bull-job-id' }))
|
|
23
|
+
|
|
24
|
+
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
25
|
+
getRedisUrlOrThrow: jest.fn(),
|
|
26
|
+
parseRedisUrl: jest.requireActual('@open-mercato/shared/lib/redis/connection').parseRedisUrl,
|
|
27
|
+
}))
|
|
28
|
+
|
|
29
|
+
jest.mock('bullmq', () => {
|
|
30
|
+
class MockQueue<T> {
|
|
31
|
+
constructor(name: string, opts: unknown) {
|
|
32
|
+
queueCtor(name, opts)
|
|
33
|
+
}
|
|
34
|
+
add = queueAdd as unknown as (name: string, data: T, opts?: unknown) => Promise<{ id?: string }>
|
|
35
|
+
close = jest.fn(async () => {})
|
|
36
|
+
obliterate = jest.fn(async () => {})
|
|
37
|
+
getJobCounts = jest.fn(async () => ({ waiting: 0, active: 0, completed: 0, failed: 0 }))
|
|
38
|
+
}
|
|
39
|
+
class MockWorker<T> {
|
|
40
|
+
constructor(
|
|
41
|
+
name: string,
|
|
42
|
+
_processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
|
|
43
|
+
opts: unknown,
|
|
44
|
+
) {
|
|
45
|
+
workerCtor(name, _processor, opts)
|
|
46
|
+
}
|
|
47
|
+
on = jest.fn()
|
|
48
|
+
close = jest.fn(async () => {})
|
|
49
|
+
}
|
|
50
|
+
return { Queue: MockQueue, Worker: MockWorker }
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
class MockBullMQOtel {
|
|
54
|
+
constructor(public readonly tracerName: string) {}
|
|
55
|
+
}
|
|
56
|
+
jest.mock('bullmq-otel', () => ({ BullMQOtel: MockBullMQOtel }))
|
|
57
|
+
|
|
58
|
+
describe('Queue - async strategy telemetry wiring', () => {
|
|
59
|
+
beforeEach(() => {
|
|
60
|
+
jest.clearAllMocks()
|
|
61
|
+
;(getRedisUrlOrThrow as jest.MockedFunction<typeof getRedisUrlOrThrow>).mockReturnValue(
|
|
62
|
+
'rediss://default:secret@example.com:6380/1',
|
|
63
|
+
)
|
|
64
|
+
registerTelemetryRuntime({
|
|
65
|
+
canUseGlobalTracePropagation: () => true,
|
|
66
|
+
captureTraceContext: () => ({}),
|
|
67
|
+
continueTrace: (_carrier, _name, fn) => fn(),
|
|
68
|
+
recordHttpDuration: () => {},
|
|
69
|
+
reportError: () => {},
|
|
70
|
+
shutdown: async () => {},
|
|
71
|
+
})
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
afterEach(() => {
|
|
75
|
+
resetTelemetryRuntime()
|
|
76
|
+
})
|
|
77
|
+
|
|
78
|
+
it('wires bullmq-otel into BOTH the queue and worker when they resolve concurrently', async () => {
|
|
79
|
+
const queue = createQueue<{ value: number }>('trace-queue', 'async', { concurrency: 3 })
|
|
80
|
+
|
|
81
|
+
// Resolve enqueue (Queue) and process (Worker) concurrently: both hit the
|
|
82
|
+
// shared telemetry resolution at once. The memoized in-flight promise must
|
|
83
|
+
// hand both the SAME bullmq-otel instance — never one with, one without.
|
|
84
|
+
await Promise.all([queue.enqueue({ value: 1 }), queue.process(async () => {})])
|
|
85
|
+
|
|
86
|
+
const queueOpts = queueCtor.mock.calls[0]?.[1] as { telemetry?: unknown }
|
|
87
|
+
const workerOpts = workerCtor.mock.calls[0]?.[2] as { telemetry?: unknown }
|
|
88
|
+
expect(queueOpts.telemetry).toBeInstanceOf(MockBullMQOtel)
|
|
89
|
+
expect(workerOpts.telemetry).toBeInstanceOf(MockBullMQOtel)
|
|
90
|
+
expect(queueOpts.telemetry).toBe(workerOpts.telemetry)
|
|
91
|
+
})
|
|
92
|
+
|
|
93
|
+
it('omits the metadata._trace carrier when bullmq-otel owns propagation', async () => {
|
|
94
|
+
const queue = createQueue<{ value: number }>('trace-queue', 'async')
|
|
95
|
+
|
|
96
|
+
await queue.enqueue({ value: 42 })
|
|
97
|
+
|
|
98
|
+
const jobData = queueAdd.mock.calls[0]?.[1] as Record<string, unknown>
|
|
99
|
+
expect(jobData).not.toHaveProperty('metadata')
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it('uses the dedicated carrier when global propagation is not explicitly trusted', async () => {
|
|
103
|
+
resetTelemetryRuntime()
|
|
104
|
+
registerTelemetryRuntime({
|
|
105
|
+
canUseGlobalTracePropagation: () => false,
|
|
106
|
+
captureTraceContext: () => ({ traceparent: 'secure-carrier' }),
|
|
107
|
+
continueTrace: (_carrier, _name, fn) => fn(),
|
|
108
|
+
recordHttpDuration: () => {},
|
|
109
|
+
reportError: () => {},
|
|
110
|
+
shutdown: async () => {},
|
|
111
|
+
})
|
|
112
|
+
const queue = createQueue<{ value: number }>('secure-trace-queue', 'async')
|
|
113
|
+
|
|
114
|
+
await queue.enqueue({ value: 7 })
|
|
115
|
+
|
|
116
|
+
const queueOpts = queueCtor.mock.calls[0]?.[1] as { telemetry?: unknown }
|
|
117
|
+
const jobData = queueAdd.mock.calls[0]?.[1] as {
|
|
118
|
+
metadata?: { _trace?: { traceparent?: string } }
|
|
119
|
+
}
|
|
120
|
+
expect(queueOpts.telemetry).toBeUndefined()
|
|
121
|
+
expect(jobData.metadata?._trace?.traceparent).toBe('secure-carrier')
|
|
122
|
+
})
|
|
123
|
+
})
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { resolveQueueStrategy, createModuleQueue } from '../factory'
|
|
2
|
-
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
2
|
+
import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
|
|
3
3
|
|
|
4
4
|
jest.mock('@open-mercato/shared/lib/redis/connection', () => ({
|
|
5
5
|
getRedisUrlOrThrow: jest.fn(),
|
|
6
|
+
parseRedisUrl: jest.fn(),
|
|
6
7
|
}))
|
|
7
8
|
|
|
8
9
|
jest.mock('bullmq', () => {
|
|
@@ -58,10 +59,12 @@ describe('resolveQueueStrategy', () => {
|
|
|
58
59
|
describe('createModuleQueue', () => {
|
|
59
60
|
const originalEnv = process.env.QUEUE_STRATEGY
|
|
60
61
|
const getRedisUrlOrThrowMock = getRedisUrlOrThrow as jest.MockedFunction<typeof getRedisUrlOrThrow>
|
|
62
|
+
const parseRedisUrlMock = parseRedisUrl as jest.MockedFunction<typeof parseRedisUrl>
|
|
61
63
|
|
|
62
64
|
beforeEach(() => {
|
|
63
65
|
jest.clearAllMocks()
|
|
64
66
|
getRedisUrlOrThrowMock.mockReturnValue('redis://localhost:6379')
|
|
67
|
+
parseRedisUrlMock.mockReturnValue({ host: 'localhost', port: 6379 })
|
|
65
68
|
})
|
|
66
69
|
|
|
67
70
|
afterEach(() => {
|
|
@@ -85,6 +88,7 @@ describe('createModuleQueue', () => {
|
|
|
85
88
|
expect(queue.strategy).toBe('async')
|
|
86
89
|
expect(queue.name).toBe('test-queue')
|
|
87
90
|
expect(getRedisUrlOrThrowMock).toHaveBeenCalledWith('QUEUE')
|
|
91
|
+
expect(parseRedisUrlMock).toHaveBeenCalledWith('redis://localhost:6379')
|
|
88
92
|
})
|
|
89
93
|
|
|
90
94
|
it('passes concurrency to local strategy', () => {
|
|
@@ -2,7 +2,22 @@ import fs from 'node:fs'
|
|
|
2
2
|
import os from 'node:os'
|
|
3
3
|
import path from 'node:path'
|
|
4
4
|
import { createQueue } from '../factory'
|
|
5
|
-
import { getQueuePendingProbe } from '../pending-probe'
|
|
5
|
+
import { __resetPendingProbeBullMQCache, getQueuePendingProbe } from '../pending-probe'
|
|
6
|
+
|
|
7
|
+
const asyncQueueConstructor = jest.fn()
|
|
8
|
+
const asyncQueueClose = jest.fn(async () => {})
|
|
9
|
+
const asyncQueueGetJobCounts = jest.fn(async () => ({ waiting: 1, delayed: 0, active: 0 }))
|
|
10
|
+
|
|
11
|
+
jest.mock('bullmq', () => ({
|
|
12
|
+
Queue: class MockQueue {
|
|
13
|
+
constructor(name: string, options: unknown) {
|
|
14
|
+
asyncQueueConstructor(name, options)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
getJobCounts = asyncQueueGetJobCounts
|
|
18
|
+
close = asyncQueueClose
|
|
19
|
+
},
|
|
20
|
+
}))
|
|
6
21
|
|
|
7
22
|
describe('getQueuePendingProbe — local strategy', () => {
|
|
8
23
|
const origCwd = process.cwd()
|
|
@@ -95,6 +110,11 @@ describe('getQueuePendingProbe — local strategy', () => {
|
|
|
95
110
|
})
|
|
96
111
|
|
|
97
112
|
describe('getQueuePendingProbe — async strategy', () => {
|
|
113
|
+
beforeEach(() => {
|
|
114
|
+
jest.clearAllMocks()
|
|
115
|
+
__resetPendingProbeBullMQCache()
|
|
116
|
+
})
|
|
117
|
+
|
|
98
118
|
it('reports an error when QUEUE Redis URL is unset and no connection override is provided', async () => {
|
|
99
119
|
const original = process.env.QUEUE_REDIS_URL
|
|
100
120
|
const fallback = process.env.REDIS_URL
|
|
@@ -109,4 +129,23 @@ describe('getQueuePendingProbe — async strategy', () => {
|
|
|
109
129
|
if (fallback !== undefined) process.env.REDIS_URL = fallback
|
|
110
130
|
}
|
|
111
131
|
})
|
|
132
|
+
|
|
133
|
+
it('converts a URL override to BullMQ connection fields', async () => {
|
|
134
|
+
const probe = await getQueuePendingProbe('async-probe', 'async', {
|
|
135
|
+
connection: { url: 'rediss://probe:secret@example.com:6380/3?family=6' },
|
|
136
|
+
})
|
|
137
|
+
|
|
138
|
+
expect(probe).toEqual(expect.objectContaining({ error: false, ready: 1 }))
|
|
139
|
+
expect(asyncQueueConstructor).toHaveBeenCalledWith('async-probe', {
|
|
140
|
+
connection: {
|
|
141
|
+
host: 'example.com',
|
|
142
|
+
port: 6380,
|
|
143
|
+
username: 'probe',
|
|
144
|
+
password: 'secret',
|
|
145
|
+
db: 3,
|
|
146
|
+
tls: {},
|
|
147
|
+
family: 6,
|
|
148
|
+
},
|
|
149
|
+
})
|
|
150
|
+
})
|
|
112
151
|
})
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { attachTraceMetadata, runJobInTrace } from '../tracing'
|
|
5
|
+
import { createLocalQueue } from '../strategies/local'
|
|
6
|
+
import { registerProvider, initTelemetry, shutdownTelemetry } from '@open-mercato/telemetry'
|
|
7
|
+
import type { LogRecord, MetricPoint, Span, SpanOptions, TelemetryProvider, TraceCarrier } from '@open-mercato/telemetry'
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Verifies the enqueue → worker trace handoff: the active context is captured
|
|
11
|
+
* onto job metadata at enqueue, and the worker continues that trace at dispatch.
|
|
12
|
+
* Uses a recording provider under the explicitly enabled console seam rather
|
|
13
|
+
* than the real OTLP SDK.
|
|
14
|
+
*/
|
|
15
|
+
const spanNames: string[] = []
|
|
16
|
+
const remoteCarriers: TraceCarrier[] = []
|
|
17
|
+
|
|
18
|
+
function noopSpan(): Span {
|
|
19
|
+
return { setAttribute() {}, setAttributes() {}, recordException() {}, setStatus() {}, end() {} }
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const recordingProvider: TelemetryProvider = {
|
|
23
|
+
name: 'console',
|
|
24
|
+
supports: ['traces'],
|
|
25
|
+
async start() {},
|
|
26
|
+
async shutdown() {},
|
|
27
|
+
runInSpan<T>(name: string, _o: SpanOptions, fn: (s: Span) => T): T {
|
|
28
|
+
spanNames.push(name)
|
|
29
|
+
return fn(noopSpan())
|
|
30
|
+
},
|
|
31
|
+
activeSpan: () => undefined,
|
|
32
|
+
activeTraceContext: () => undefined,
|
|
33
|
+
inject: (carrier) => {
|
|
34
|
+
carrier.traceparent = 'test-traceparent'
|
|
35
|
+
},
|
|
36
|
+
runInRemoteSpan<T>(carrier: TraceCarrier, name: string, _o: SpanOptions, fn: (s: Span) => T): T {
|
|
37
|
+
remoteCarriers.push(carrier)
|
|
38
|
+
spanNames.push(name)
|
|
39
|
+
return fn(noopSpan())
|
|
40
|
+
},
|
|
41
|
+
emitLog: (_r: LogRecord) => {},
|
|
42
|
+
recordMetric: (_p: MetricPoint) => {},
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
beforeAll(async () => {
|
|
46
|
+
process.env.TELEMETRY_BACKEND = 'console'
|
|
47
|
+
registerProvider(recordingProvider)
|
|
48
|
+
await initTelemetry()
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
afterAll(async () => {
|
|
52
|
+
await shutdownTelemetry()
|
|
53
|
+
delete process.env.TELEMETRY_BACKEND
|
|
54
|
+
})
|
|
55
|
+
|
|
56
|
+
describe('queue trace propagation', () => {
|
|
57
|
+
it('attaches the active trace carrier to job metadata at enqueue', () => {
|
|
58
|
+
const metadata = attachTraceMetadata(undefined)
|
|
59
|
+
expect(metadata).toEqual({ _trace: { traceparent: 'test-traceparent' } })
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
it('preserves existing metadata while attaching the trace carrier', () => {
|
|
63
|
+
const metadata = attachTraceMetadata({ foo: 'bar' })
|
|
64
|
+
expect(metadata).toMatchObject({ foo: 'bar', _trace: { traceparent: 'test-traceparent' } })
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
it('continues the producer trace from job metadata at dispatch', async () => {
|
|
68
|
+
const result = await runJobInTrace('orders-process', { _trace: { traceparent: 'tp-123' } }, () =>
|
|
69
|
+
Promise.resolve('done'),
|
|
70
|
+
)
|
|
71
|
+
expect(result).toBe('done')
|
|
72
|
+
expect(remoteCarriers).toContainEqual({ traceparent: 'tp-123' })
|
|
73
|
+
expect(spanNames).toContain('queue.orders-process')
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
it('runs jobs without a carrier under a fresh span (no crash)', async () => {
|
|
77
|
+
const result = await runJobInTrace('orders-process', undefined, () => Promise.resolve(42))
|
|
78
|
+
expect(result).toBe(42)
|
|
79
|
+
})
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* End-to-end through the REAL local strategy (file write + read + dispatch),
|
|
84
|
+
* proving the headline acceptance criterion: a queued job continues the
|
|
85
|
+
* enqueuing request's trace. This exercises the actual enqueue/dispatch wiring,
|
|
86
|
+
* not just the helpers above.
|
|
87
|
+
*/
|
|
88
|
+
describe('queue trace propagation (real local strategy)', () => {
|
|
89
|
+
let baseDir: string
|
|
90
|
+
|
|
91
|
+
beforeEach(() => {
|
|
92
|
+
baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'om-queue-trace-'))
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
afterEach(() => {
|
|
96
|
+
fs.rmSync(baseDir, { recursive: true, force: true })
|
|
97
|
+
})
|
|
98
|
+
|
|
99
|
+
it('persists the trace carrier on enqueue and continues it on dispatch', async () => {
|
|
100
|
+
const queue = createLocalQueue<{ orderId: string }>('orders-process', { baseDir })
|
|
101
|
+
|
|
102
|
+
await queue.enqueue({ orderId: 'o-1' })
|
|
103
|
+
|
|
104
|
+
// The carrier is written to the job's metadata — NOT the user payload.
|
|
105
|
+
const stored = JSON.parse(
|
|
106
|
+
fs.readFileSync(path.join(baseDir, 'orders-process', 'queue.json'), 'utf8'),
|
|
107
|
+
) as Array<{ payload: unknown; metadata?: Record<string, unknown> }>
|
|
108
|
+
expect(stored[0].metadata).toEqual({ _trace: { traceparent: 'test-traceparent' } })
|
|
109
|
+
expect(stored[0].payload).toEqual({ orderId: 'o-1' })
|
|
110
|
+
|
|
111
|
+
let handlerRan = false
|
|
112
|
+
await queue.process((job) => {
|
|
113
|
+
handlerRan = true
|
|
114
|
+
// The handler still sees only its payload; the carrier is invisible to it.
|
|
115
|
+
expect(job.payload).toEqual({ orderId: 'o-1' })
|
|
116
|
+
})
|
|
117
|
+
|
|
118
|
+
expect(handlerRan).toBe(true)
|
|
119
|
+
// The worker continued the producer's trace under a `queue.<name>` span.
|
|
120
|
+
expect(remoteCarriers).toContainEqual({ traceparent: 'test-traceparent' })
|
|
121
|
+
expect(spanNames).toContain('queue.orders-process')
|
|
122
|
+
})
|
|
123
|
+
})
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import path from 'node:path'
|
|
2
|
+
import fs from 'node:fs'
|
|
3
|
+
import os from 'node:os'
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Regression: the worker's graceful shutdown used to close the queues
|
|
7
|
+
* and call process.exit() without flushing telemetry. A worker never returns
|
|
8
|
+
* from run(), so the CLI's post-run shutdownTelemetry() is unreachable — the
|
|
9
|
+
* BatchSpanProcessor's buffered tail (~5s of spans/logs) was dropped on every
|
|
10
|
+
* restart/redeploy. The shutdown handler must flush BEFORE exiting.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
registerTelemetryRuntime,
|
|
15
|
+
resetTelemetryRuntime,
|
|
16
|
+
type TelemetryRuntime,
|
|
17
|
+
} from '@open-mercato/shared/lib/telemetry/runtime'
|
|
18
|
+
|
|
19
|
+
import { runWorker } from '../worker/runner'
|
|
20
|
+
|
|
21
|
+
const mockCallOrder: string[] = []
|
|
22
|
+
const runtime: TelemetryRuntime = {
|
|
23
|
+
canUseGlobalTracePropagation: () => false,
|
|
24
|
+
captureTraceContext: () => ({}),
|
|
25
|
+
continueTrace: (_carrier, _name, fn) => fn(),
|
|
26
|
+
recordHttpDuration: () => {},
|
|
27
|
+
reportError: () => {},
|
|
28
|
+
shutdown: jest.fn(async () => {
|
|
29
|
+
mockCallOrder.push('flush')
|
|
30
|
+
}),
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe('worker shutdown flushes telemetry', () => {
|
|
34
|
+
let tmpDir: string
|
|
35
|
+
let cwd: string
|
|
36
|
+
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'worker-shutdown-'))
|
|
39
|
+
cwd = process.cwd()
|
|
40
|
+
process.chdir(tmpDir)
|
|
41
|
+
mockCallOrder.length = 0
|
|
42
|
+
delete process.env.TELEMETRY_BACKEND
|
|
43
|
+
registerTelemetryRuntime(runtime)
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
afterEach(() => {
|
|
47
|
+
process.chdir(cwd)
|
|
48
|
+
resetTelemetryRuntime()
|
|
49
|
+
fs.rmSync(tmpDir, { recursive: true, force: true })
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
it('SIGTERM flushes telemetry before process.exit', async () => {
|
|
53
|
+
const exitSpy = jest.spyOn(process, 'exit').mockImplementation(((code?: number) => {
|
|
54
|
+
mockCallOrder.push(`exit:${code ?? 0}`)
|
|
55
|
+
return undefined as never
|
|
56
|
+
}) as never)
|
|
57
|
+
const logSpy = jest.spyOn(console, 'log').mockImplementation(() => {})
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
await runWorker({
|
|
61
|
+
queueName: 'shutdown-flush-test',
|
|
62
|
+
handler: async () => {},
|
|
63
|
+
strategy: 'local',
|
|
64
|
+
background: true,
|
|
65
|
+
gracefulShutdown: true,
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
process.emit('SIGTERM')
|
|
69
|
+
// The shutdown handler is async (close → flush → exit); let it settle.
|
|
70
|
+
await new Promise((resolve) => setTimeout(resolve, 100))
|
|
71
|
+
|
|
72
|
+
expect(mockCallOrder).toContain('flush')
|
|
73
|
+
expect(mockCallOrder).toContain('exit:0')
|
|
74
|
+
expect(mockCallOrder.indexOf('flush')).toBeLessThan(mockCallOrder.indexOf('exit:0'))
|
|
75
|
+
} finally {
|
|
76
|
+
exitSpy.mockRestore()
|
|
77
|
+
logSpy.mockRestore()
|
|
78
|
+
}
|
|
79
|
+
})
|
|
80
|
+
})
|
package/src/pending-probe.ts
CHANGED
|
@@ -149,14 +149,16 @@ async function probeAsyncQueue(
|
|
|
149
149
|
return errorResult(queueName, 'async', new Error('bullmq is not installed'))
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
-
const { getRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')
|
|
152
|
+
const { getRedisUrl, parseRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')
|
|
153
153
|
let connection = options?.connection
|
|
154
154
|
if (!connection) {
|
|
155
155
|
const url = getRedisUrl('QUEUE')
|
|
156
156
|
if (!url) {
|
|
157
157
|
return errorResult(queueName, 'async', new Error('QUEUE Redis URL is not configured'))
|
|
158
158
|
}
|
|
159
|
-
connection =
|
|
159
|
+
connection = parseRedisUrl(url)
|
|
160
|
+
} else if (connection.url) {
|
|
161
|
+
connection = parseRedisUrl(connection.url)
|
|
160
162
|
}
|
|
161
163
|
|
|
162
164
|
let queue: InstanceType<BullMQModuleShape['Queue']> | null = null
|
package/src/strategies/async.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { Queue, QueuedJob, JobHandler, AsyncQueueOptions, ProcessResult, EnqueueOptions, QueueJobScope } from '../types'
|
|
2
|
-
import { getRedisUrlOrThrow } from '@open-mercato/shared/lib/redis/connection'
|
|
2
|
+
import { getRedisUrlOrThrow, parseRedisUrl } from '@open-mercato/shared/lib/redis/connection'
|
|
3
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
4
|
+
import { attachTraceMetadata, runJobInTrace } from '../tracing'
|
|
3
5
|
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
4
6
|
|
|
5
7
|
const packageLogger = createLogger('queue')
|
|
@@ -7,13 +9,13 @@ const packageLogger = createLogger('queue')
|
|
|
7
9
|
// BullMQ interface types - we define the shape we use to maintain type safety
|
|
8
10
|
// while keeping bullmq as an optional peer dependency
|
|
9
11
|
type ConnectionOptions = {
|
|
10
|
-
url?: string
|
|
11
12
|
host?: string
|
|
12
13
|
port?: number
|
|
13
14
|
username?: string
|
|
14
15
|
password?: string
|
|
15
16
|
db?: number
|
|
16
17
|
tls?: Record<string, unknown>
|
|
18
|
+
family?: number
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
interface BullQueueInterface<T> {
|
|
@@ -43,14 +45,23 @@ interface BullWorkerInterface {
|
|
|
43
45
|
}
|
|
44
46
|
|
|
45
47
|
interface BullMQModule {
|
|
46
|
-
Queue: new <T>(name: string, opts: { connection: ConnectionOptions }) => BullQueueInterface<T>
|
|
48
|
+
Queue: new <T>(name: string, opts: { connection: ConnectionOptions; telemetry?: unknown }) => BullQueueInterface<T>
|
|
47
49
|
Worker: new <T>(
|
|
48
50
|
name: string,
|
|
49
51
|
processor: (job: { id?: string; data: T; attemptsMade: number }) => Promise<void>,
|
|
50
|
-
opts: {
|
|
52
|
+
opts: {
|
|
53
|
+
connection: ConnectionOptions
|
|
54
|
+
concurrency: number
|
|
55
|
+
telemetry?: unknown
|
|
56
|
+
lockDuration?: number
|
|
57
|
+
maxStalledCount?: number
|
|
58
|
+
}
|
|
51
59
|
) => BullWorkerInterface
|
|
52
60
|
}
|
|
53
61
|
|
|
62
|
+
/** The `bullmq-otel` package (optional). Loaded only when an OTLP backend is active. */
|
|
63
|
+
type BullMQOtelModule = { BullMQOtel: new (tracerName: string) => object }
|
|
64
|
+
|
|
54
65
|
const REMOVABLE_JOB_STATES = ['waiting', 'delayed', 'prioritized', 'paused', 'waiting-children']
|
|
55
66
|
|
|
56
67
|
function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
|
|
@@ -69,13 +80,13 @@ function payloadMatchesScope(payload: unknown, scope: QueueJobScope): boolean {
|
|
|
69
80
|
/**
|
|
70
81
|
* Resolves Redis connection options from various sources.
|
|
71
82
|
*
|
|
72
|
-
* BullMQ expects
|
|
73
|
-
*
|
|
74
|
-
*
|
|
83
|
+
* BullMQ expects ioredis connection fields rather than a nested URL string.
|
|
84
|
+
* Parse URL-based configuration at this boundary while keeping the public
|
|
85
|
+
* queue API compatible with existing `{ url }` callers.
|
|
75
86
|
*/
|
|
76
87
|
function resolveConnection(options?: AsyncQueueOptions['connection']): ConnectionOptions {
|
|
77
88
|
if (options?.url) {
|
|
78
|
-
return
|
|
89
|
+
return parseRedisUrl(options.url)
|
|
79
90
|
}
|
|
80
91
|
|
|
81
92
|
if (options?.host) {
|
|
@@ -86,10 +97,11 @@ function resolveConnection(options?: AsyncQueueOptions['connection']): Connectio
|
|
|
86
97
|
password: options.password,
|
|
87
98
|
db: options.db,
|
|
88
99
|
tls: options.tls,
|
|
100
|
+
family: options.family,
|
|
89
101
|
}
|
|
90
102
|
}
|
|
91
103
|
|
|
92
|
-
return
|
|
104
|
+
return parseRedisUrl(getRedisUrlOrThrow('QUEUE'))
|
|
93
105
|
}
|
|
94
106
|
|
|
95
107
|
/**
|
|
@@ -119,6 +131,10 @@ export function createAsyncQueue<T = unknown>(
|
|
|
119
131
|
let bullQueue: BullQueueInterface<QueuedJob<T>> | null = null
|
|
120
132
|
let bullWorker: BullWorkerInterface | null = null
|
|
121
133
|
let bullmqModule: BullMQModule | null = null
|
|
134
|
+
// Resolved once: a BullMQOtel instance (delegate async tracing to BullMQ) or
|
|
135
|
+
// undefined (use our own metadata._trace carrier instead). Memoized as the
|
|
136
|
+
// in-flight promise so concurrent first-time callers share one resolution.
|
|
137
|
+
let telemetryPromise: Promise<object | undefined> | null = null
|
|
122
138
|
|
|
123
139
|
// -------------------------------------------------------------------------
|
|
124
140
|
// Lazy BullMQ initialization
|
|
@@ -137,10 +153,35 @@ export function createAsyncQueue<T = unknown>(
|
|
|
137
153
|
return bullmqModule
|
|
138
154
|
}
|
|
139
155
|
|
|
156
|
+
/**
|
|
157
|
+
* When an OTLP backend is active, delegate async-queue tracing to `bullmq-otel`
|
|
158
|
+
* (richer BullMQ-internal spans: add / process / wait / attempts). Returns
|
|
159
|
+
* `undefined` — meaning "use our own `metadata._trace` carrier" — when telemetry
|
|
160
|
+
* is off, a non-OTEL backend is selected, or `bullmq-otel` isn't installed. (The
|
|
161
|
+
* `local` strategy always uses our carrier; it isn't BullMQ, so `bullmq-otel`
|
|
162
|
+
* cannot instrument it.)
|
|
163
|
+
*/
|
|
164
|
+
async function getQueueTelemetry(): Promise<object | undefined> {
|
|
165
|
+
if (!telemetryPromise) {
|
|
166
|
+
telemetryPromise = (async () => {
|
|
167
|
+
if (!getTelemetryRuntime()?.canUseGlobalTracePropagation()) return undefined
|
|
168
|
+
try {
|
|
169
|
+
const mod = (await import('bullmq-otel')) as unknown as BullMQOtelModule
|
|
170
|
+
return new mod.BullMQOtel('open-mercato')
|
|
171
|
+
} catch {
|
|
172
|
+
packageLogger.warn('bullmq-otel not available; using built-in trace carrier', { queue: name })
|
|
173
|
+
return undefined
|
|
174
|
+
}
|
|
175
|
+
})()
|
|
176
|
+
}
|
|
177
|
+
return telemetryPromise
|
|
178
|
+
}
|
|
179
|
+
|
|
140
180
|
async function getQueue(): Promise<BullQueueInterface<QueuedJob<T>>> {
|
|
141
181
|
if (!bullQueue) {
|
|
142
182
|
const { Queue: BullQueueClass } = await getBullMQ()
|
|
143
|
-
|
|
183
|
+
const telemetry = await getQueueTelemetry()
|
|
184
|
+
bullQueue = new BullQueueClass<QueuedJob<T>>(name, { connection, ...(telemetry ? { telemetry } : {}) })
|
|
144
185
|
}
|
|
145
186
|
return bullQueue
|
|
146
187
|
}
|
|
@@ -151,10 +192,14 @@ export function createAsyncQueue<T = unknown>(
|
|
|
151
192
|
|
|
152
193
|
async function enqueue(data: T, options?: EnqueueOptions): Promise<string> {
|
|
153
194
|
const queue = await getQueue()
|
|
195
|
+
// When bullmq-otel handles propagation, don't also attach our carrier.
|
|
196
|
+
const telemetry = await getQueueTelemetry()
|
|
197
|
+
const metadata = telemetry ? undefined : attachTraceMetadata(undefined)
|
|
154
198
|
const jobData: QueuedJob<T> = {
|
|
155
199
|
id: crypto.randomUUID(),
|
|
156
200
|
payload: data,
|
|
157
201
|
createdAt: new Date().toISOString(),
|
|
202
|
+
...(metadata ? { metadata } : {}),
|
|
158
203
|
}
|
|
159
204
|
|
|
160
205
|
const job = await queue.add(jobData.id, jobData, {
|
|
@@ -170,21 +215,31 @@ export function createAsyncQueue<T = unknown>(
|
|
|
170
215
|
|
|
171
216
|
async function process(handler: JobHandler<T>): Promise<ProcessResult> {
|
|
172
217
|
const { Worker } = await getBullMQ()
|
|
218
|
+
const telemetry = await getQueueTelemetry()
|
|
173
219
|
|
|
174
220
|
// Create worker that processes jobs
|
|
175
221
|
bullWorker = new Worker<QueuedJob<T>>(
|
|
176
222
|
name,
|
|
177
223
|
async (job) => {
|
|
178
224
|
const jobData = job.data
|
|
179
|
-
|
|
225
|
+
const ctx = {
|
|
180
226
|
jobId: job.id ?? jobData.id,
|
|
181
227
|
attemptNumber: job.attemptsMade + 1,
|
|
182
228
|
queueName: name,
|
|
183
|
-
}
|
|
229
|
+
}
|
|
230
|
+
// With bullmq-otel active, BullMQ owns the process span and active
|
|
231
|
+
// context (the handler's pg/undici spans nest under it). Otherwise
|
|
232
|
+
// continue the trace from our own carrier.
|
|
233
|
+
if (telemetry) {
|
|
234
|
+
await handler(jobData, ctx)
|
|
235
|
+
} else {
|
|
236
|
+
await runJobInTrace(name, jobData.metadata, () => handler(jobData, ctx))
|
|
237
|
+
}
|
|
184
238
|
},
|
|
185
239
|
{
|
|
186
240
|
connection,
|
|
187
241
|
concurrency,
|
|
242
|
+
...(telemetry ? { telemetry } : {}),
|
|
188
243
|
...(lockDuration !== undefined ? { lockDuration } : {}),
|
|
189
244
|
...(maxStalledCount !== undefined ? { maxStalledCount } : {}),
|
|
190
245
|
}
|