@open-mercato/queue 0.6.1-develop.3167.1.7aedd7b2ba → 0.6.1-develop.3179.1.9ee0441adc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +2 -2
- package/dist/pending-probe.js +127 -0
- package/dist/pending-probe.js.map +7 -0
- package/package.json +2 -2
- package/src/__tests__/pending-probe.test.ts +112 -0
- package/src/index.ts +4 -0
- package/src/pending-probe.ts +215 -0
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
[build:queue] found
|
|
1
|
+
[build:queue] found 8 entry points
|
|
2
2
|
[build:queue] built successfully
|
package/dist/index.js
CHANGED
|
@@ -2,10 +2,12 @@ export * from "./types.js";
|
|
|
2
2
|
import { createQueue, createModuleQueue, resolveQueueStrategy } from "./factory.js";
|
|
3
3
|
export * from "./worker/registry.js";
|
|
4
4
|
import { runWorker, createRoutedHandler } from "./worker/runner.js";
|
|
5
|
+
import { getQueuePendingProbe } from "./pending-probe.js";
|
|
5
6
|
export {
|
|
6
7
|
createModuleQueue,
|
|
7
8
|
createQueue,
|
|
8
9
|
createRoutedHandler,
|
|
10
|
+
getQueuePendingProbe,
|
|
9
11
|
resolveQueueStrategy,
|
|
10
12
|
runWorker
|
|
11
13
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/index.ts"],
|
|
4
|
-
"sourcesContent": ["/**\n * @open-mercato/queue\n *\n * Multi-strategy job queue package supporting local (file-based) and async (BullMQ) strategies.\n *\n * @example\n * ```typescript\n * import { createQueue } from '@open-mercato/queue'\n *\n * // Create a local queue\n * const queue = createQueue<{ userId: string }>('my-queue', 'local')\n *\n * // Enqueue a job\n * await queue.enqueue({ userId: '123' })\n *\n * // Process jobs\n * await queue.process(async (job, ctx) => {\n * console.log(`Processing job ${ctx.jobId}:`, job.payload)\n * })\n * ```\n */\n\nexport * from './types'\nexport { createQueue, createModuleQueue, resolveQueueStrategy } from './factory'\n\n// Worker utilities\nexport * from './worker/registry'\nexport { runWorker, createRoutedHandler } from './worker/runner'\n"],
|
|
5
|
-
"mappings": "AAsBA,cAAc;AACd,SAAS,aAAa,mBAAmB,4BAA4B;AAGrE,cAAc;AACd,SAAS,WAAW,2BAA2B;",
|
|
4
|
+
"sourcesContent": ["/**\n * @open-mercato/queue\n *\n * Multi-strategy job queue package supporting local (file-based) and async (BullMQ) strategies.\n *\n * @example\n * ```typescript\n * import { createQueue } from '@open-mercato/queue'\n *\n * // Create a local queue\n * const queue = createQueue<{ userId: string }>('my-queue', 'local')\n *\n * // Enqueue a job\n * await queue.enqueue({ userId: '123' })\n *\n * // Process jobs\n * await queue.process(async (job, ctx) => {\n * console.log(`Processing job ${ctx.jobId}:`, job.payload)\n * })\n * ```\n */\n\nexport * from './types'\nexport { createQueue, createModuleQueue, resolveQueueStrategy } from './factory'\n\n// Worker utilities\nexport * from './worker/registry'\nexport { runWorker, createRoutedHandler } from './worker/runner'\n\n// Lightweight pending-job probes used by the lazy auto-spawn supervisor.\nexport { getQueuePendingProbe } from './pending-probe'\nexport type { QueuePendingProbeOptions, QueuePendingProbeResult } from './pending-probe'\n"],
|
|
5
|
+
"mappings": "AAsBA,cAAc;AACd,SAAS,aAAa,mBAAmB,4BAA4B;AAGrE,cAAc;AACd,SAAS,WAAW,2BAA2B;AAG/C,SAAS,4BAA4B;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
const DEFAULT_LOCAL_QUEUE_BASE_DIR = ".mercato/queue";
|
|
4
|
+
const fsp = fs.promises;
|
|
5
|
+
function emptyResult(queueName, strategy) {
|
|
6
|
+
return { queueName, strategy, ready: 0, delayedFuture: 0, active: 0, error: false };
|
|
7
|
+
}
|
|
8
|
+
function errorResult(queueName, strategy, err) {
|
|
9
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
10
|
+
return {
|
|
11
|
+
queueName,
|
|
12
|
+
strategy,
|
|
13
|
+
ready: 0,
|
|
14
|
+
delayedFuture: 0,
|
|
15
|
+
active: 0,
|
|
16
|
+
error: true,
|
|
17
|
+
errorMessage: message
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
async function probeLocalQueue(queueName, options) {
|
|
21
|
+
const nodeProcess = globalThis.process;
|
|
22
|
+
const envBaseDir = nodeProcess?.env?.QUEUE_BASE_DIR;
|
|
23
|
+
const baseDir = options?.baseDir ?? path.resolve(envBaseDir || DEFAULT_LOCAL_QUEUE_BASE_DIR);
|
|
24
|
+
const queueFile = path.join(baseDir, queueName, "queue.json");
|
|
25
|
+
let raw;
|
|
26
|
+
try {
|
|
27
|
+
raw = await fsp.readFile(queueFile, "utf8");
|
|
28
|
+
} catch (err) {
|
|
29
|
+
const fsErr = err;
|
|
30
|
+
if (fsErr?.code === "ENOENT") {
|
|
31
|
+
return emptyResult(queueName, "local");
|
|
32
|
+
}
|
|
33
|
+
return errorResult(queueName, "local", err);
|
|
34
|
+
}
|
|
35
|
+
let parsed;
|
|
36
|
+
try {
|
|
37
|
+
parsed = JSON.parse(raw);
|
|
38
|
+
} catch (err) {
|
|
39
|
+
return errorResult(queueName, "local", err);
|
|
40
|
+
}
|
|
41
|
+
if (!Array.isArray(parsed)) {
|
|
42
|
+
return emptyResult(queueName, "local");
|
|
43
|
+
}
|
|
44
|
+
const now = Date.now();
|
|
45
|
+
let ready = 0;
|
|
46
|
+
let delayedFuture = 0;
|
|
47
|
+
for (const entry of parsed) {
|
|
48
|
+
if (!entry || typeof entry !== "object") continue;
|
|
49
|
+
const availableAt = entry.availableAt;
|
|
50
|
+
if (typeof availableAt !== "string" || availableAt.length === 0) {
|
|
51
|
+
ready++;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
const ts = Date.parse(availableAt);
|
|
55
|
+
if (!Number.isFinite(ts) || ts <= now) {
|
|
56
|
+
ready++;
|
|
57
|
+
} else {
|
|
58
|
+
delayedFuture++;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return { queueName, strategy: "local", ready, delayedFuture, active: 0, error: false };
|
|
62
|
+
}
|
|
63
|
+
let cachedBullMQ;
|
|
64
|
+
async function loadBullMQ() {
|
|
65
|
+
if (cachedBullMQ !== void 0) return cachedBullMQ;
|
|
66
|
+
try {
|
|
67
|
+
cachedBullMQ = await import("bullmq");
|
|
68
|
+
} catch {
|
|
69
|
+
cachedBullMQ = null;
|
|
70
|
+
}
|
|
71
|
+
return cachedBullMQ;
|
|
72
|
+
}
|
|
73
|
+
async function probeAsyncQueue(queueName, options) {
|
|
74
|
+
const bullmq = await loadBullMQ();
|
|
75
|
+
if (!bullmq) {
|
|
76
|
+
return errorResult(queueName, "async", new Error("bullmq is not installed"));
|
|
77
|
+
}
|
|
78
|
+
const { getRedisUrl } = await import("@open-mercato/shared/lib/redis/connection");
|
|
79
|
+
let connection = options?.connection;
|
|
80
|
+
if (!connection) {
|
|
81
|
+
const url = getRedisUrl("QUEUE");
|
|
82
|
+
if (!url) {
|
|
83
|
+
return errorResult(queueName, "async", new Error("QUEUE Redis URL is not configured"));
|
|
84
|
+
}
|
|
85
|
+
connection = { url };
|
|
86
|
+
}
|
|
87
|
+
let queue = null;
|
|
88
|
+
try {
|
|
89
|
+
queue = new bullmq.Queue(queueName, { connection });
|
|
90
|
+
const counts = await queue.getJobCounts("waiting", "delayed", "active");
|
|
91
|
+
const waiting = counts.waiting ?? 0;
|
|
92
|
+
const delayed = counts.delayed ?? 0;
|
|
93
|
+
const active = counts.active ?? 0;
|
|
94
|
+
return {
|
|
95
|
+
queueName,
|
|
96
|
+
strategy: "async",
|
|
97
|
+
ready: waiting,
|
|
98
|
+
delayedFuture: delayed,
|
|
99
|
+
active,
|
|
100
|
+
error: false
|
|
101
|
+
};
|
|
102
|
+
} catch (err) {
|
|
103
|
+
return errorResult(queueName, "async", err);
|
|
104
|
+
} finally {
|
|
105
|
+
if (queue) {
|
|
106
|
+
try {
|
|
107
|
+
await queue.close();
|
|
108
|
+
} catch {
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
async function getQueuePendingProbe(queueName, strategy, options) {
|
|
114
|
+
const resolvedStrategy = strategy ?? (process.env.QUEUE_STRATEGY === "async" ? "async" : "local");
|
|
115
|
+
if (resolvedStrategy === "async") {
|
|
116
|
+
return probeAsyncQueue(queueName, options);
|
|
117
|
+
}
|
|
118
|
+
return probeLocalQueue(queueName, options);
|
|
119
|
+
}
|
|
120
|
+
function __resetPendingProbeBullMQCache() {
|
|
121
|
+
cachedBullMQ = void 0;
|
|
122
|
+
}
|
|
123
|
+
export {
|
|
124
|
+
__resetPendingProbeBullMQCache,
|
|
125
|
+
getQueuePendingProbe
|
|
126
|
+
};
|
|
127
|
+
//# sourceMappingURL=pending-probe.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../src/pending-probe.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Lightweight pending-job probe helpers.\n *\n * Used by the lazy worker supervisor to detect whether a queue has a\n * ready-to-process job before spawning a long-lived worker process for it.\n *\n * The probes MUST NOT:\n * - call `queue.process()` or otherwise install handlers\n * - import any module worker handler code\n * - create BullMQ `Worker` instances\n *\n * Probes are best-effort and fail-soft: a probe error returns \"no pending\"\n * so the supervisor can keep polling instead of crashing the runtime.\n */\n\nimport fs from 'node:fs'\nimport path from 'node:path'\nimport type { QueueStrategyType, RedisConnectionOptions } from './types'\n\nexport type QueuePendingProbeOptions = {\n /** Async strategy: Redis connection override (mirrors `AsyncQueueOptions['connection']`). */\n connection?: RedisConnectionOptions\n /** Local strategy: override `QUEUE_BASE_DIR` for test isolation. */\n baseDir?: string\n}\n\nexport type QueuePendingProbeResult = {\n queueName: string\n strategy: QueueStrategyType\n /** Number of jobs ready to process now (no delay or delay already elapsed). */\n ready: number\n /** Number of jobs scheduled for the future (still waiting on `availableAt`). */\n delayedFuture: number\n /** Number of jobs currently being processed. May be unavailable for some strategies. */\n active: number\n /**\n * True when the probe could not query the underlying storage at all\n * (filesystem error, Redis unreachable, optional dependency missing, etc.).\n * The supervisor treats `error: true` as \"do not start worker yet\".\n */\n error: boolean\n errorMessage?: string\n}\n\nconst DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'\n\nconst fsp = fs.promises\n\nfunction emptyResult(queueName: string, strategy: QueueStrategyType): QueuePendingProbeResult {\n return { queueName, strategy, ready: 0, delayedFuture: 0, active: 0, error: false }\n}\n\nfunction errorResult(\n queueName: string,\n strategy: QueueStrategyType,\n err: unknown,\n): QueuePendingProbeResult {\n const message = err instanceof Error ? err.message : String(err)\n return {\n queueName,\n strategy,\n ready: 0,\n delayedFuture: 0,\n active: 0,\n error: true,\n errorMessage: message,\n }\n}\n\nasync function probeLocalQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process\n const envBaseDir = nodeProcess?.env?.QUEUE_BASE_DIR\n const baseDir = options?.baseDir\n ?? path.resolve(envBaseDir || DEFAULT_LOCAL_QUEUE_BASE_DIR)\n const queueFile = path.join(baseDir, queueName, 'queue.json')\n\n let raw: string\n try {\n raw = await fsp.readFile(queueFile, 'utf8')\n } catch (err) {\n const fsErr = err as NodeJS.ErrnoException\n if (fsErr?.code === 'ENOENT') {\n return emptyResult(queueName, 'local')\n }\n return errorResult(queueName, 'local', err)\n }\n\n let parsed: unknown\n try {\n parsed = JSON.parse(raw)\n } catch (err) {\n return errorResult(queueName, 'local', err)\n }\n\n if (!Array.isArray(parsed)) {\n return emptyResult(queueName, 'local')\n }\n\n const now = Date.now()\n let ready = 0\n let delayedFuture = 0\n\n for (const entry of parsed) {\n if (!entry || typeof entry !== 'object') continue\n const availableAt = (entry as { availableAt?: unknown }).availableAt\n if (typeof availableAt !== 'string' || availableAt.length === 0) {\n ready++\n continue\n }\n const ts = Date.parse(availableAt)\n if (!Number.isFinite(ts) || ts <= now) {\n ready++\n } else {\n delayedFuture++\n }\n }\n\n return { queueName, strategy: 'local', ready, delayedFuture, active: 0, error: false }\n}\n\ntype BullMQModuleShape = {\n Queue: new <T>(name: string, opts: { connection: RedisConnectionOptions }) => {\n getJobCounts: (...states: string[]) => Promise<Record<string, number>>\n close: () => Promise<void>\n }\n}\n\nlet cachedBullMQ: BullMQModuleShape | null | undefined\n\nasync function loadBullMQ(): Promise<BullMQModuleShape | null> {\n if (cachedBullMQ !== undefined) return cachedBullMQ\n try {\n cachedBullMQ = (await import('bullmq')) as unknown as BullMQModuleShape\n } catch {\n cachedBullMQ = null\n }\n return cachedBullMQ\n}\n\nasync function probeAsyncQueue(\n queueName: string,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const bullmq = await loadBullMQ()\n if (!bullmq) {\n return errorResult(queueName, 'async', new Error('bullmq is not installed'))\n }\n\n const { getRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')\n let connection = options?.connection\n if (!connection) {\n const url = getRedisUrl('QUEUE')\n if (!url) {\n return errorResult(queueName, 'async', new Error('QUEUE Redis URL is not configured'))\n }\n connection = { url }\n }\n\n let queue: InstanceType<BullMQModuleShape['Queue']> | null = null\n try {\n queue = new bullmq.Queue(queueName, { connection })\n const counts = await queue.getJobCounts('waiting', 'delayed', 'active')\n const waiting = counts.waiting ?? 0\n const delayed = counts.delayed ?? 0\n const active = counts.active ?? 0\n return {\n queueName,\n strategy: 'async',\n ready: waiting,\n delayedFuture: delayed,\n active,\n error: false,\n }\n } catch (err) {\n return errorResult(queueName, 'async', err)\n } finally {\n if (queue) {\n try {\n await queue.close()\n } catch {\n /* swallow shutdown errors \u2014 probe must not throw on cleanup */\n }\n }\n }\n}\n\n/**\n * Read-only pending-job probe for a queue.\n *\n * `strategy` defaults to the value of `QUEUE_STRATEGY` (via\n * `resolveQueueStrategy`). The probe never installs handlers and never\n * starts a BullMQ Worker, so it is safe to call from a lightweight\n * supervisor process that watches many queues.\n */\nexport async function getQueuePendingProbe(\n queueName: string,\n strategy?: QueueStrategyType,\n options?: QueuePendingProbeOptions,\n): Promise<QueuePendingProbeResult> {\n const resolvedStrategy: QueueStrategyType = strategy\n ?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')\n\n if (resolvedStrategy === 'async') {\n return probeAsyncQueue(queueName, options)\n }\n return probeLocalQueue(queueName, options)\n}\n\n/** Reset the cached bullmq module reference. Test-only. */\nexport function __resetPendingProbeBullMQCache(): void {\n cachedBullMQ = undefined\n}\n"],
|
|
5
|
+
"mappings": "AAeA,OAAO,QAAQ;AACf,OAAO,UAAU;AA4BjB,MAAM,+BAA+B;AAErC,MAAM,MAAM,GAAG;AAEf,SAAS,YAAY,WAAmB,UAAsD;AAC5F,SAAO,EAAE,WAAW,UAAU,OAAO,GAAG,eAAe,GAAG,QAAQ,GAAG,OAAO,MAAM;AACpF;AAEA,SAAS,YACP,WACA,UACA,KACyB;AACzB,QAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,OAAO;AAAA,IACP,eAAe;AAAA,IACf,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,cAAc;AAAA,EAChB;AACF;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,cAAe,WAAgE;AACrF,QAAM,aAAa,aAAa,KAAK;AACrC,QAAM,UAAU,SAAS,WACpB,KAAK,QAAQ,cAAc,4BAA4B;AAC5D,QAAM,YAAY,KAAK,KAAK,SAAS,WAAW,YAAY;AAE5D,MAAI;AACJ,MAAI;AACF,UAAM,MAAM,IAAI,SAAS,WAAW,MAAM;AAAA,EAC5C,SAAS,KAAK;AACZ,UAAM,QAAQ;AACd,QAAI,OAAO,SAAS,UAAU;AAC5B,aAAO,YAAY,WAAW,OAAO;AAAA,IACvC;AACA,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,GAAG;AAAA,EACzB,SAAS,KAAK;AACZ,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C;AAEA,MAAI,CAAC,MAAM,QAAQ,MAAM,GAAG;AAC1B,WAAO,YAAY,WAAW,OAAO;AAAA,EACvC;AAEA,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,QAAQ;AACZ,MAAI,gBAAgB;AAEpB,aAAW,SAAS,QAAQ;AAC1B,QAAI,CAAC,SAAS,OAAO,UAAU,SAAU;AACzC,UAAM,cAAe,MAAoC;AACzD,QAAI,OAAO,gBAAgB,YAAY,YAAY,WAAW,GAAG;AAC/D;AACA;AAAA,IACF;AACA,UAAM,KAAK,KAAK,MAAM,WAAW;AACjC,QAAI,CAAC,OAAO,SAAS,EAAE,KAAK,MAAM,KAAK;AACrC;AAAA,IACF,OAAO;AACL;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,WAAW,UAAU,SAAS,OAAO,eAAe,QAAQ,GAAG,OAAO,MAAM;AACvF;AASA,IAAI;AAEJ,eAAe,aAAgD;AAC7D,MAAI,iBAAiB,OAAW,QAAO;AACvC,MAAI;AACF,mBAAgB,MAAM,OAAO,QAAQ;AAAA,EACvC,QAAQ;AACN,mBAAe;AAAA,EACjB;AACA,SAAO;AACT;AAEA,eAAe,gBACb,WACA,SACkC;AAClC,QAAM,SAAS,MAAM,WAAW;AAChC,MAAI,CAAC,QAAQ;AACX,WAAO,YAAY,WAAW,SAAS,IAAI,MAAM,yBAAyB,CAAC;AAAA,EAC7E;AAEA,QAAM,EAAE,YAAY,IAAI,MAAM,OAAO,2CAA2C;AAChF,MAAI,aAAa,SAAS;AAC1B,MAAI,CAAC,YAAY;AACf,UAAM,MAAM,YAAY,OAAO;AAC/B,QAAI,CAAC,KAAK;AACR,aAAO,YAAY,WAAW,SAAS,IAAI,MAAM,mCAAmC,CAAC;AAAA,IACvF;AACA,iBAAa,EAAE,IAAI;AAAA,EACrB;AAEA,MAAI,QAAyD;AAC7D,MAAI;AACF,YAAQ,IAAI,OAAO,MAAM,WAAW,EAAE,WAAW,CAAC;AAClD,UAAM,SAAS,MAAM,MAAM,aAAa,WAAW,WAAW,QAAQ;AACtE,UAAM,UAAU,OAAO,WAAW;AAClC,UAAM,UAAU,OAAO,WAAW;AAClC,UAAM,SAAS,OAAO,UAAU;AAChC,WAAO;AAAA,MACL;AAAA,MACA,UAAU;AAAA,MACV,OAAO;AAAA,MACP,eAAe;AAAA,MACf;AAAA,MACA,OAAO;AAAA,IACT;AAAA,EACF,SAAS,KAAK;AACZ,WAAO,YAAY,WAAW,SAAS,GAAG;AAAA,EAC5C,UAAE;AACA,QAAI,OAAO;AACT,UAAI;AACF,cAAM,MAAM,MAAM;AAAA,MACpB,QAAQ;AAAA,MAER;AAAA,IACF;AAAA,EACF;AACF;AAUA,eAAsB,qBACpB,WACA,UACA,SACkC;AAClC,QAAM,mBAAsC,aACtC,QAAQ,IAAI,mBAAmB,UAAU,UAAU;AAEzD,MAAI,qBAAqB,SAAS;AAChC,WAAO,gBAAgB,WAAW,OAAO;AAAA,EAC3C;AACA,SAAO,gBAAgB,WAAW,OAAO;AAC3C;AAGO,SAAS,iCAAuC;AACrD,iBAAe;AACjB;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/queue",
|
|
3
|
-
"version": "0.6.1-develop.
|
|
3
|
+
"version": "0.6.1-develop.3179.1.9ee0441adc",
|
|
4
4
|
"description": "Multi-strategy job queue with local and BullMQ support",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -50,7 +50,7 @@
|
|
|
50
50
|
"access": "public"
|
|
51
51
|
},
|
|
52
52
|
"dependencies": {
|
|
53
|
-
"@open-mercato/shared": "0.6.1-develop.
|
|
53
|
+
"@open-mercato/shared": "0.6.1-develop.3179.1.9ee0441adc"
|
|
54
54
|
},
|
|
55
55
|
"repository": {
|
|
56
56
|
"type": "git",
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { createQueue } from '../factory'
|
|
5
|
+
import { getQueuePendingProbe } from '../pending-probe'
|
|
6
|
+
|
|
7
|
+
describe('getQueuePendingProbe — local strategy', () => {
|
|
8
|
+
const origCwd = process.cwd()
|
|
9
|
+
let tmp: string
|
|
10
|
+
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'queue-probe-test-'))
|
|
13
|
+
process.chdir(tmp)
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
afterEach(() => {
|
|
17
|
+
process.chdir(origCwd)
|
|
18
|
+
try {
|
|
19
|
+
fs.rmSync(tmp, { recursive: true, force: true })
|
|
20
|
+
} catch {
|
|
21
|
+
/* ignore cleanup errors */
|
|
22
|
+
}
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
it('returns zero when the queue does not exist yet', async () => {
|
|
26
|
+
const probe = await getQueuePendingProbe('does-not-exist', 'local', { baseDir: tmp })
|
|
27
|
+
expect(probe.error).toBe(false)
|
|
28
|
+
expect(probe.ready).toBe(0)
|
|
29
|
+
expect(probe.delayedFuture).toBe(0)
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
it('counts ready jobs without invoking handlers', async () => {
|
|
33
|
+
const queue = createQueue<{ value: number }>('probe-queue', 'local', { baseDir: tmp })
|
|
34
|
+
await queue.enqueue({ value: 1 })
|
|
35
|
+
await queue.enqueue({ value: 2 })
|
|
36
|
+
await queue.close()
|
|
37
|
+
|
|
38
|
+
const probe = await getQueuePendingProbe('probe-queue', 'local', { baseDir: tmp })
|
|
39
|
+
expect(probe.error).toBe(false)
|
|
40
|
+
expect(probe.ready).toBe(2)
|
|
41
|
+
expect(probe.delayedFuture).toBe(0)
|
|
42
|
+
})
|
|
43
|
+
|
|
44
|
+
it('treats future-delayed jobs as not ready', async () => {
|
|
45
|
+
const queue = createQueue<{ value: number }>('probe-queue', 'local', { baseDir: tmp })
|
|
46
|
+
await queue.enqueue({ value: 1 }, { delayMs: 60_000 })
|
|
47
|
+
await queue.close()
|
|
48
|
+
|
|
49
|
+
const probe = await getQueuePendingProbe('probe-queue', 'local', { baseDir: tmp })
|
|
50
|
+
expect(probe.ready).toBe(0)
|
|
51
|
+
expect(probe.delayedFuture).toBe(1)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
it('treats already-elapsed delayed jobs as ready', async () => {
|
|
55
|
+
const queueDir = path.join(tmp, 'probe-queue')
|
|
56
|
+
fs.mkdirSync(queueDir, { recursive: true })
|
|
57
|
+
const past = new Date(Date.now() - 1000).toISOString()
|
|
58
|
+
fs.writeFileSync(
|
|
59
|
+
path.join(queueDir, 'queue.json'),
|
|
60
|
+
JSON.stringify([
|
|
61
|
+
{
|
|
62
|
+
id: 'past-job',
|
|
63
|
+
payload: { value: 1 },
|
|
64
|
+
createdAt: new Date().toISOString(),
|
|
65
|
+
availableAt: past,
|
|
66
|
+
},
|
|
67
|
+
]),
|
|
68
|
+
'utf8',
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
const probe = await getQueuePendingProbe('probe-queue', 'local', { baseDir: tmp })
|
|
72
|
+
expect(probe.ready).toBe(1)
|
|
73
|
+
expect(probe.delayedFuture).toBe(0)
|
|
74
|
+
})
|
|
75
|
+
|
|
76
|
+
it('returns a soft error when queue.json is corrupt', async () => {
|
|
77
|
+
const queueDir = path.join(tmp, 'corrupt-queue')
|
|
78
|
+
fs.mkdirSync(queueDir, { recursive: true })
|
|
79
|
+
fs.writeFileSync(path.join(queueDir, 'queue.json'), '{ not valid json', 'utf8')
|
|
80
|
+
|
|
81
|
+
const probe = await getQueuePendingProbe('corrupt-queue', 'local', { baseDir: tmp })
|
|
82
|
+
expect(probe.error).toBe(true)
|
|
83
|
+
expect(probe.ready).toBe(0)
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
it('does not run the worker handler', async () => {
|
|
87
|
+
const queue = createQueue<{ value: number }>('handler-check', 'local', { baseDir: tmp })
|
|
88
|
+
const handler = jest.fn()
|
|
89
|
+
await queue.enqueue({ value: 1 })
|
|
90
|
+
await queue.close()
|
|
91
|
+
|
|
92
|
+
await getQueuePendingProbe('handler-check', 'local', { baseDir: tmp })
|
|
93
|
+
expect(handler).not.toHaveBeenCalled()
|
|
94
|
+
})
|
|
95
|
+
})
|
|
96
|
+
|
|
97
|
+
describe('getQueuePendingProbe — async strategy', () => {
|
|
98
|
+
it('reports an error when QUEUE Redis URL is unset and no connection override is provided', async () => {
|
|
99
|
+
const original = process.env.QUEUE_REDIS_URL
|
|
100
|
+
const fallback = process.env.REDIS_URL
|
|
101
|
+
delete process.env.QUEUE_REDIS_URL
|
|
102
|
+
delete process.env.REDIS_URL
|
|
103
|
+
try {
|
|
104
|
+
const probe = await getQueuePendingProbe('no-redis-queue', 'async')
|
|
105
|
+
// Either bullmq is missing (cached load failed) or Redis URL missing — both surface as error
|
|
106
|
+
expect(probe.error).toBe(true)
|
|
107
|
+
} finally {
|
|
108
|
+
if (original !== undefined) process.env.QUEUE_REDIS_URL = original
|
|
109
|
+
if (fallback !== undefined) process.env.REDIS_URL = fallback
|
|
110
|
+
}
|
|
111
|
+
})
|
|
112
|
+
})
|
package/src/index.ts
CHANGED
|
@@ -26,3 +26,7 @@ export { createQueue, createModuleQueue, resolveQueueStrategy } from './factory'
|
|
|
26
26
|
// Worker utilities
|
|
27
27
|
export * from './worker/registry'
|
|
28
28
|
export { runWorker, createRoutedHandler } from './worker/runner'
|
|
29
|
+
|
|
30
|
+
// Lightweight pending-job probes used by the lazy auto-spawn supervisor.
|
|
31
|
+
export { getQueuePendingProbe } from './pending-probe'
|
|
32
|
+
export type { QueuePendingProbeOptions, QueuePendingProbeResult } from './pending-probe'
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight pending-job probe helpers.
|
|
3
|
+
*
|
|
4
|
+
* Used by the lazy worker supervisor to detect whether a queue has a
|
|
5
|
+
* ready-to-process job before spawning a long-lived worker process for it.
|
|
6
|
+
*
|
|
7
|
+
* The probes MUST NOT:
|
|
8
|
+
* - call `queue.process()` or otherwise install handlers
|
|
9
|
+
* - import any module worker handler code
|
|
10
|
+
* - create BullMQ `Worker` instances
|
|
11
|
+
*
|
|
12
|
+
* Probes are best-effort and fail-soft: a probe error returns "no pending"
|
|
13
|
+
* so the supervisor can keep polling instead of crashing the runtime.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import fs from 'node:fs'
|
|
17
|
+
import path from 'node:path'
|
|
18
|
+
import type { QueueStrategyType, RedisConnectionOptions } from './types'
|
|
19
|
+
|
|
20
|
+
export type QueuePendingProbeOptions = {
|
|
21
|
+
/** Async strategy: Redis connection override (mirrors `AsyncQueueOptions['connection']`). */
|
|
22
|
+
connection?: RedisConnectionOptions
|
|
23
|
+
/** Local strategy: override `QUEUE_BASE_DIR` for test isolation. */
|
|
24
|
+
baseDir?: string
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type QueuePendingProbeResult = {
|
|
28
|
+
queueName: string
|
|
29
|
+
strategy: QueueStrategyType
|
|
30
|
+
/** Number of jobs ready to process now (no delay or delay already elapsed). */
|
|
31
|
+
ready: number
|
|
32
|
+
/** Number of jobs scheduled for the future (still waiting on `availableAt`). */
|
|
33
|
+
delayedFuture: number
|
|
34
|
+
/** Number of jobs currently being processed. May be unavailable for some strategies. */
|
|
35
|
+
active: number
|
|
36
|
+
/**
|
|
37
|
+
* True when the probe could not query the underlying storage at all
|
|
38
|
+
* (filesystem error, Redis unreachable, optional dependency missing, etc.).
|
|
39
|
+
* The supervisor treats `error: true` as "do not start worker yet".
|
|
40
|
+
*/
|
|
41
|
+
error: boolean
|
|
42
|
+
errorMessage?: string
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const DEFAULT_LOCAL_QUEUE_BASE_DIR = '.mercato/queue'
|
|
46
|
+
|
|
47
|
+
const fsp = fs.promises
|
|
48
|
+
|
|
49
|
+
function emptyResult(queueName: string, strategy: QueueStrategyType): QueuePendingProbeResult {
|
|
50
|
+
return { queueName, strategy, ready: 0, delayedFuture: 0, active: 0, error: false }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function errorResult(
|
|
54
|
+
queueName: string,
|
|
55
|
+
strategy: QueueStrategyType,
|
|
56
|
+
err: unknown,
|
|
57
|
+
): QueuePendingProbeResult {
|
|
58
|
+
const message = err instanceof Error ? err.message : String(err)
|
|
59
|
+
return {
|
|
60
|
+
queueName,
|
|
61
|
+
strategy,
|
|
62
|
+
ready: 0,
|
|
63
|
+
delayedFuture: 0,
|
|
64
|
+
active: 0,
|
|
65
|
+
error: true,
|
|
66
|
+
errorMessage: message,
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function probeLocalQueue(
|
|
71
|
+
queueName: string,
|
|
72
|
+
options?: QueuePendingProbeOptions,
|
|
73
|
+
): Promise<QueuePendingProbeResult> {
|
|
74
|
+
const nodeProcess = (globalThis as typeof globalThis & { process?: NodeJS.Process }).process
|
|
75
|
+
const envBaseDir = nodeProcess?.env?.QUEUE_BASE_DIR
|
|
76
|
+
const baseDir = options?.baseDir
|
|
77
|
+
?? path.resolve(envBaseDir || DEFAULT_LOCAL_QUEUE_BASE_DIR)
|
|
78
|
+
const queueFile = path.join(baseDir, queueName, 'queue.json')
|
|
79
|
+
|
|
80
|
+
let raw: string
|
|
81
|
+
try {
|
|
82
|
+
raw = await fsp.readFile(queueFile, 'utf8')
|
|
83
|
+
} catch (err) {
|
|
84
|
+
const fsErr = err as NodeJS.ErrnoException
|
|
85
|
+
if (fsErr?.code === 'ENOENT') {
|
|
86
|
+
return emptyResult(queueName, 'local')
|
|
87
|
+
}
|
|
88
|
+
return errorResult(queueName, 'local', err)
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
let parsed: unknown
|
|
92
|
+
try {
|
|
93
|
+
parsed = JSON.parse(raw)
|
|
94
|
+
} catch (err) {
|
|
95
|
+
return errorResult(queueName, 'local', err)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (!Array.isArray(parsed)) {
|
|
99
|
+
return emptyResult(queueName, 'local')
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const now = Date.now()
|
|
103
|
+
let ready = 0
|
|
104
|
+
let delayedFuture = 0
|
|
105
|
+
|
|
106
|
+
for (const entry of parsed) {
|
|
107
|
+
if (!entry || typeof entry !== 'object') continue
|
|
108
|
+
const availableAt = (entry as { availableAt?: unknown }).availableAt
|
|
109
|
+
if (typeof availableAt !== 'string' || availableAt.length === 0) {
|
|
110
|
+
ready++
|
|
111
|
+
continue
|
|
112
|
+
}
|
|
113
|
+
const ts = Date.parse(availableAt)
|
|
114
|
+
if (!Number.isFinite(ts) || ts <= now) {
|
|
115
|
+
ready++
|
|
116
|
+
} else {
|
|
117
|
+
delayedFuture++
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return { queueName, strategy: 'local', ready, delayedFuture, active: 0, error: false }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
type BullMQModuleShape = {
|
|
125
|
+
Queue: new <T>(name: string, opts: { connection: RedisConnectionOptions }) => {
|
|
126
|
+
getJobCounts: (...states: string[]) => Promise<Record<string, number>>
|
|
127
|
+
close: () => Promise<void>
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
let cachedBullMQ: BullMQModuleShape | null | undefined
|
|
132
|
+
|
|
133
|
+
async function loadBullMQ(): Promise<BullMQModuleShape | null> {
|
|
134
|
+
if (cachedBullMQ !== undefined) return cachedBullMQ
|
|
135
|
+
try {
|
|
136
|
+
cachedBullMQ = (await import('bullmq')) as unknown as BullMQModuleShape
|
|
137
|
+
} catch {
|
|
138
|
+
cachedBullMQ = null
|
|
139
|
+
}
|
|
140
|
+
return cachedBullMQ
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function probeAsyncQueue(
|
|
144
|
+
queueName: string,
|
|
145
|
+
options?: QueuePendingProbeOptions,
|
|
146
|
+
): Promise<QueuePendingProbeResult> {
|
|
147
|
+
const bullmq = await loadBullMQ()
|
|
148
|
+
if (!bullmq) {
|
|
149
|
+
return errorResult(queueName, 'async', new Error('bullmq is not installed'))
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const { getRedisUrl } = await import('@open-mercato/shared/lib/redis/connection')
|
|
153
|
+
let connection = options?.connection
|
|
154
|
+
if (!connection) {
|
|
155
|
+
const url = getRedisUrl('QUEUE')
|
|
156
|
+
if (!url) {
|
|
157
|
+
return errorResult(queueName, 'async', new Error('QUEUE Redis URL is not configured'))
|
|
158
|
+
}
|
|
159
|
+
connection = { url }
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
let queue: InstanceType<BullMQModuleShape['Queue']> | null = null
|
|
163
|
+
try {
|
|
164
|
+
queue = new bullmq.Queue(queueName, { connection })
|
|
165
|
+
const counts = await queue.getJobCounts('waiting', 'delayed', 'active')
|
|
166
|
+
const waiting = counts.waiting ?? 0
|
|
167
|
+
const delayed = counts.delayed ?? 0
|
|
168
|
+
const active = counts.active ?? 0
|
|
169
|
+
return {
|
|
170
|
+
queueName,
|
|
171
|
+
strategy: 'async',
|
|
172
|
+
ready: waiting,
|
|
173
|
+
delayedFuture: delayed,
|
|
174
|
+
active,
|
|
175
|
+
error: false,
|
|
176
|
+
}
|
|
177
|
+
} catch (err) {
|
|
178
|
+
return errorResult(queueName, 'async', err)
|
|
179
|
+
} finally {
|
|
180
|
+
if (queue) {
|
|
181
|
+
try {
|
|
182
|
+
await queue.close()
|
|
183
|
+
} catch {
|
|
184
|
+
/* swallow shutdown errors — probe must not throw on cleanup */
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Read-only pending-job probe for a queue.
|
|
192
|
+
*
|
|
193
|
+
* `strategy` defaults to the value of `QUEUE_STRATEGY` (via
|
|
194
|
+
* `resolveQueueStrategy`). The probe never installs handlers and never
|
|
195
|
+
* starts a BullMQ Worker, so it is safe to call from a lightweight
|
|
196
|
+
* supervisor process that watches many queues.
|
|
197
|
+
*/
|
|
198
|
+
export async function getQueuePendingProbe(
|
|
199
|
+
queueName: string,
|
|
200
|
+
strategy?: QueueStrategyType,
|
|
201
|
+
options?: QueuePendingProbeOptions,
|
|
202
|
+
): Promise<QueuePendingProbeResult> {
|
|
203
|
+
const resolvedStrategy: QueueStrategyType = strategy
|
|
204
|
+
?? (process.env.QUEUE_STRATEGY === 'async' ? 'async' : 'local')
|
|
205
|
+
|
|
206
|
+
if (resolvedStrategy === 'async') {
|
|
207
|
+
return probeAsyncQueue(queueName, options)
|
|
208
|
+
}
|
|
209
|
+
return probeLocalQueue(queueName, options)
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Reset the cached bullmq module reference. Test-only. */
|
|
213
|
+
export function __resetPendingProbeBullMQCache(): void {
|
|
214
|
+
cachedBullMQ = undefined
|
|
215
|
+
}
|