@open-mercato/webhooks 0.10.1-develop.8075.1.f2d38008c0 → 0.10.1-develop.8098.1.f4decc8a57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/dist/modules/webhooks/__integration__/TC-WEBHOOK-015.spec.js +75 -0
- package/dist/modules/webhooks/__integration__/TC-WEBHOOK-015.spec.js.map +7 -0
- package/dist/modules/webhooks/lib/delivery.js +71 -2
- package/dist/modules/webhooks/lib/delivery.js.map +2 -2
- package/dist/modules/webhooks/lib/queue.js +4 -0
- package/dist/modules/webhooks/lib/queue.js.map +2 -2
- package/package.json +12 -9
- package/src/modules/webhooks/__integration__/TC-WEBHOOK-015.spec.ts +101 -0
- package/src/modules/webhooks/lib/__tests__/delivery-claim.integration.test.ts +291 -0
- package/src/modules/webhooks/lib/__tests__/delivery-claim.test.ts +191 -0
- package/src/modules/webhooks/lib/__tests__/delivery-custom-headers.test.ts +1 -0
- package/src/modules/webhooks/lib/__tests__/delivery-url-safety.test.ts +1 -0
- package/src/modules/webhooks/lib/__tests__/subscription-cache-invalidation.test.ts +2 -0
- package/src/modules/webhooks/lib/delivery.ts +93 -2
- package/src/modules/webhooks/lib/queue.ts +4 -0
package/.turbo/turbo-build.log
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
[build:webhooks] found
|
|
1
|
+
[build:webhooks] found 82 entry points
|
|
2
2
|
[build:webhooks] built successfully
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { createServer } from "node:http";
|
|
2
|
+
import { expect, test } from "@playwright/test";
|
|
3
|
+
import { apiRequest, getAuthToken } from "@open-mercato/core/helpers/integration/api";
|
|
4
|
+
import { readJsonSafe } from "@open-mercato/core/helpers/integration/generalFixtures";
|
|
5
|
+
import { drainIntegrationQueue } from "@open-mercato/core/helpers/integration/queue";
|
|
6
|
+
import { readIntegrationEnvFlag } from "@open-mercato/core/helpers/integration/standaloneEnv";
|
|
7
|
+
import {
|
|
8
|
+
createWebhookFixture,
|
|
9
|
+
deleteWebhookIfExists,
|
|
10
|
+
getDeliveryDetail
|
|
11
|
+
} from "./helpers/fixtures.js";
|
|
12
|
+
const PRIVATE_URLS_ALLOWED = readIntegrationEnvFlag("OM_WEBHOOKS_ALLOW_PRIVATE_URLS");
|
|
13
|
+
const DELIVERY_COUNT = 3;
|
|
14
|
+
async function startReceiver() {
|
|
15
|
+
const sentByPath = /* @__PURE__ */ new Map();
|
|
16
|
+
const server = createServer((req, res) => {
|
|
17
|
+
req.resume();
|
|
18
|
+
req.on("end", () => {
|
|
19
|
+
const path = req.url ?? "";
|
|
20
|
+
const sent = sentByPath.get(path) ?? [];
|
|
21
|
+
const status = sent.length === 0 ? 400 : 200;
|
|
22
|
+
sentByPath.set(path, [...sent, status]);
|
|
23
|
+
res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify({ ok: status === 200 }));
|
|
24
|
+
});
|
|
25
|
+
});
|
|
26
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
27
|
+
const { port } = server.address();
|
|
28
|
+
return {
|
|
29
|
+
server,
|
|
30
|
+
baseUrl: `http://127.0.0.1:${port}`,
|
|
31
|
+
statuses: (path) => [...sentByPath.get(path) ?? []]
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
test.describe("TC-WEBHOOK-015: delivery jobs do not send a delivery twice", () => {
|
|
35
|
+
test("each retried delivery reaches the receiver exactly once more", async ({ request }) => {
|
|
36
|
+
test.skip(!PRIVATE_URLS_ALLOWED, "The loopback receiver needs OM_WEBHOOKS_ALLOW_PRIVATE_URLS");
|
|
37
|
+
test.setTimeout(12e4);
|
|
38
|
+
const token = await getAuthToken(request);
|
|
39
|
+
const receiver = await startReceiver();
|
|
40
|
+
const webhookIds = [];
|
|
41
|
+
try {
|
|
42
|
+
const deliveries = [];
|
|
43
|
+
for (let index = 0; index < DELIVERY_COUNT; index += 1) {
|
|
44
|
+
const path = `/tc-webhook-015-${index}`;
|
|
45
|
+
const created = await createWebhookFixture(request, token, {
|
|
46
|
+
name: `Webhook delivery claim ${index} ${Date.now()}`,
|
|
47
|
+
url: `${receiver.baseUrl}${path}`,
|
|
48
|
+
subscribedEvents: ["catalog.product.updated"]
|
|
49
|
+
});
|
|
50
|
+
webhookIds.push(created.id);
|
|
51
|
+
const testResponse = await apiRequest(request, "POST", `/api/webhooks/${created.id}/test`, { token, data: {} });
|
|
52
|
+
expect(testResponse.status()).toBe(200);
|
|
53
|
+
const testBody = await readJsonSafe(testResponse);
|
|
54
|
+
const deliveryId = testBody?.delivery.id;
|
|
55
|
+
expect(testBody?.delivery.status).toBe("expired");
|
|
56
|
+
expect(deliveryId).toEqual(expect.any(String));
|
|
57
|
+
deliveries.push({ path, deliveryId: String(deliveryId) });
|
|
58
|
+
}
|
|
59
|
+
const retries = await Promise.all(deliveries.map(({ deliveryId }) => apiRequest(request, "POST", `/api/webhooks/deliveries/${deliveryId}/retry`, { token })));
|
|
60
|
+
for (const retry of retries) expect(retry.status()).toBe(200);
|
|
61
|
+
for (const { deliveryId } of deliveries) {
|
|
62
|
+
await expect.poll(async () => (await getDeliveryDetail(request, token, deliveryId)).status, { timeout: 3e4 }).toBe("delivered");
|
|
63
|
+
}
|
|
64
|
+
await drainIntegrationQueue("webhook-deliveries");
|
|
65
|
+
for (const { path, deliveryId } of deliveries) {
|
|
66
|
+
expect(receiver.statuses(path)).toEqual([400, 200]);
|
|
67
|
+
expect((await getDeliveryDetail(request, token, deliveryId)).attemptNumber).toBe(2);
|
|
68
|
+
}
|
|
69
|
+
} finally {
|
|
70
|
+
for (const webhookId of webhookIds) await deleteWebhookIfExists(request, token, webhookId);
|
|
71
|
+
await new Promise((resolve) => receiver.server.close(() => resolve()));
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
});
|
|
75
|
+
//# sourceMappingURL=TC-WEBHOOK-015.spec.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/modules/webhooks/__integration__/TC-WEBHOOK-015.spec.ts"],
|
|
4
|
+
"sourcesContent": ["import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';\nimport type { AddressInfo } from 'node:net';\nimport { expect, test } from '@playwright/test';\nimport { apiRequest, getAuthToken } from '@open-mercato/core/helpers/integration/api';\nimport { readJsonSafe } from '@open-mercato/core/helpers/integration/generalFixtures';\nimport { drainIntegrationQueue } from '@open-mercato/core/helpers/integration/queue';\nimport { readIntegrationEnvFlag } from '@open-mercato/core/helpers/integration/standaloneEnv';\nimport {\n createWebhookFixture,\n deleteWebhookIfExists,\n getDeliveryDetail,\n} from './helpers/fixtures';\n\n/**\n * TC-WEBHOOK-015: a retried webhook delivery is sent once even when its job runs more than once\n * Source: open-mercato/open-mercato#ISSUE\n *\n * The delivery worker claims a delivery (`pending` \u2192 `sending`) atomically before sending and ignores\n * jobs for deliveries that already finished, so a queue that runs a job twice cannot send the event\n * twice. In the integration env the app process starts an in-process local-queue worker on enqueue\n * while AUTO_SPAWN_WORKERS also runs the webhook worker, so a retried job can run twice.\n * Each delivery here fails its synchronous test send with a non-retryable 400, is retried once, and\n * must reach the receiver exactly once more. Needs OM_WEBHOOKS_ALLOW_PRIVATE_URLS (enabled in the integration env)\n * because the receiver listens on loopback.\n */\nconst PRIVATE_URLS_ALLOWED = readIntegrationEnvFlag('OM_WEBHOOKS_ALLOW_PRIVATE_URLS');\nconst DELIVERY_COUNT = 3;\n\ntype Receiver = {\n server: Server;\n baseUrl: string;\n statuses: (path: string) => number[];\n};\n\nasync function startReceiver(): Promise<Receiver> {\n const sentByPath = new Map<string, number[]>();\n const server = createServer((req: IncomingMessage, res: ServerResponse) => {\n req.resume();\n req.on('end', () => {\n const path = req.url ?? '';\n const sent = sentByPath.get(path) ?? [];\n const status = sent.length === 0 ? 400 : 200;\n sentByPath.set(path, [...sent, status]);\n res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify({ ok: status === 200 }));\n });\n });\n await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));\n const { port } = server.address() as AddressInfo;\n return {\n server,\n baseUrl: `http://127.0.0.1:${port}`,\n statuses: (path: string) => [...(sentByPath.get(path) ?? [])],\n };\n}\n\ntest.describe('TC-WEBHOOK-015: delivery jobs do not send a delivery twice', () => {\n test('each retried delivery reaches the receiver exactly once more', async ({ request }) => {\n test.skip(!PRIVATE_URLS_ALLOWED, 'The loopback receiver needs OM_WEBHOOKS_ALLOW_PRIVATE_URLS');\n test.setTimeout(120_000);\n const token = await getAuthToken(request);\n const receiver = await startReceiver();\n const webhookIds: string[] = [];\n\n try {\n const deliveries: Array<{ path: string; deliveryId: string }> = [];\n for (let index = 0; index < DELIVERY_COUNT; index += 1) {\n const path = `/tc-webhook-015-${index}`;\n const created = await createWebhookFixture(request, token, {\n name: `Webhook delivery claim ${index} ${Date.now()}`,\n url: `${receiver.baseUrl}${path}`,\n subscribedEvents: ['catalog.product.updated'],\n });\n webhookIds.push(created.id);\n const testResponse = await apiRequest(request, 'POST', `/api/webhooks/${created.id}/test`, { token, data: {} });\n expect(testResponse.status()).toBe(200);\n const testBody = await readJsonSafe<{ delivery: { id: string; status: string } }>(testResponse);\n const deliveryId = testBody?.delivery.id;\n expect(testBody?.delivery.status).toBe('expired');\n expect(deliveryId).toEqual(expect.any(String));\n deliveries.push({ path, deliveryId: String(deliveryId) });\n }\n\n const retries = await Promise.all(deliveries.map(({ deliveryId }) =>\n apiRequest(request, 'POST', `/api/webhooks/deliveries/${deliveryId}/retry`, { token })));\n for (const retry of retries) expect(retry.status()).toBe(200);\n\n for (const { deliveryId } of deliveries) {\n await expect.poll(async () => (await getDeliveryDetail(request, token, deliveryId)).status, { timeout: 30_000 }).toBe('delivered');\n }\n await drainIntegrationQueue('webhook-deliveries');\n\n for (const { path, deliveryId } of deliveries) {\n expect(receiver.statuses(path)).toEqual([400, 200]);\n expect((await getDeliveryDetail(request, token, deliveryId)).attemptNumber).toBe(2);\n }\n } finally {\n for (const webhookId of webhookIds) await deleteWebhookIfExists(request, token, webhookId);\n await new Promise<void>((resolve) => receiver.server.close(() => resolve()));\n }\n });\n});\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,oBAA4E;AAErF,SAAS,QAAQ,YAAY;AAC7B,SAAS,YAAY,oBAAoB;AACzC,SAAS,oBAAoB;AAC7B,SAAS,6BAA6B;AACtC,SAAS,8BAA8B;AACvC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAcP,MAAM,uBAAuB,uBAAuB,gCAAgC;AACpF,MAAM,iBAAiB;AAQvB,eAAe,gBAAmC;AAChD,QAAM,aAAa,oBAAI,IAAsB;AAC7C,QAAM,SAAS,aAAa,CAAC,KAAsB,QAAwB;AACzE,QAAI,OAAO;AACX,QAAI,GAAG,OAAO,MAAM;AAClB,YAAM,OAAO,IAAI,OAAO;AACxB,YAAM,OAAO,WAAW,IAAI,IAAI,KAAK,CAAC;AACtC,YAAM,SAAS,KAAK,WAAW,IAAI,MAAM;AACzC,iBAAW,IAAI,MAAM,CAAC,GAAG,MAAM,MAAM,CAAC;AACtC,UAAI,UAAU,QAAQ,EAAE,gBAAgB,mBAAmB,CAAC,EAAE,IAAI,KAAK,UAAU,EAAE,IAAI,WAAW,IAAI,CAAC,CAAC;AAAA,IAC1G,CAAC;AAAA,EACH,CAAC;AACD,QAAM,IAAI,QAAc,CAAC,YAAY,OAAO,OAAO,GAAG,aAAa,OAAO,CAAC;AAC3E,QAAM,EAAE,KAAK,IAAI,OAAO,QAAQ;AAChC,SAAO;AAAA,IACL;AAAA,IACA,SAAS,oBAAoB,IAAI;AAAA,IACjC,UAAU,CAAC,SAAiB,CAAC,GAAI,WAAW,IAAI,IAAI,KAAK,CAAC,CAAE;AAAA,EAC9D;AACF;AAEA,KAAK,SAAS,8DAA8D,MAAM;AAChF,OAAK,gEAAgE,OAAO,EAAE,QAAQ,MAAM;AAC1F,SAAK,KAAK,CAAC,sBAAsB,4DAA4D;AAC7F,SAAK,WAAW,IAAO;AACvB,UAAM,QAAQ,MAAM,aAAa,OAAO;AACxC,UAAM,WAAW,MAAM,cAAc;AACrC,UAAM,aAAuB,CAAC;AAE9B,QAAI;AACF,YAAM,aAA0D,CAAC;AACjE,eAAS,QAAQ,GAAG,QAAQ,gBAAgB,SAAS,GAAG;AACtD,cAAM,OAAO,mBAAmB,KAAK;AACrC,cAAM,UAAU,MAAM,qBAAqB,SAAS,OAAO;AAAA,UACzD,MAAM,0BAA0B,KAAK,IAAI,KAAK,IAAI,CAAC;AAAA,UACnD,KAAK,GAAG,SAAS,OAAO,GAAG,IAAI;AAAA,UAC/B,kBAAkB,CAAC,yBAAyB;AAAA,QAC9C,CAAC;AACD,mBAAW,KAAK,QAAQ,EAAE;AAC1B,cAAM,eAAe,MAAM,WAAW,SAAS,QAAQ,iBAAiB,QAAQ,EAAE,SAAS,EAAE,OAAO,MAAM,CAAC,EAAE,CAAC;AAC9G,eAAO,aAAa,OAAO,CAAC,EAAE,KAAK,GAAG;AACtC,cAAM,WAAW,MAAM,aAA2D,YAAY;AAC9F,cAAM,aAAa,UAAU,SAAS;AACtC,eAAO,UAAU,SAAS,MAAM,EAAE,KAAK,SAAS;AAChD,eAAO,UAAU,EAAE,QAAQ,OAAO,IAAI,MAAM,CAAC;AAC7C,mBAAW,KAAK,EAAE,MAAM,YAAY,OAAO,UAAU,EAAE,CAAC;AAAA,MAC1D;AAEA,YAAM,UAAU,MAAM,QAAQ,IAAI,WAAW,IAAI,CAAC,EAAE,WAAW,MAC7D,WAAW,SAAS,QAAQ,4BAA4B,UAAU,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;AACzF,iBAAW,SAAS,QAAS,QAAO,MAAM,OAAO,CAAC,EAAE,KAAK,GAAG;AAE5D,iBAAW,EAAE,WAAW,KAAK,YAAY;AACvC,cAAM,OAAO,KAAK,aAAa,MAAM,kBAAkB,SAAS,OAAO,UAAU,GAAG,QAAQ,EAAE,SAAS,IAAO,CAAC,EAAE,KAAK,WAAW;AAAA,MACnI;AACA,YAAM,sBAAsB,oBAAoB;AAEhD,iBAAW,EAAE,MAAM,WAAW,KAAK,YAAY;AAC7C,eAAO,SAAS,SAAS,IAAI,CAAC,EAAE,QAAQ,CAAC,KAAK,GAAG,CAAC;AAClD,gBAAQ,MAAM,kBAAkB,SAAS,OAAO,UAAU,GAAG,aAAa,EAAE,KAAK,CAAC;AAAA,MACpF;AAAA,IACF,UAAE;AACA,iBAAW,aAAa,WAAY,OAAM,sBAAsB,SAAS,OAAO,SAAS;AACzF,YAAM,IAAI,QAAc,CAAC,YAAY,SAAS,OAAO,MAAM,MAAM,QAAQ,CAAC,CAAC;AAAA,IAC7E;AAAA,EACF,CAAC;AACH,CAAC;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { buildWebhookHeaders, generateMessageId } from "@open-mercato/shared/lib/webhooks";
|
|
2
2
|
import { findOneWithDecryption } from "@open-mercato/shared/lib/encryption/find";
|
|
3
|
+
import { createLogger } from "@open-mercato/shared/lib/logger";
|
|
4
|
+
import { getTelemetryRuntime } from "@open-mercato/shared/lib/telemetry/runtime";
|
|
3
5
|
import { WebhookDeliveryEntity, WebhookEntity } from "../data/entities.js";
|
|
4
6
|
import { emitWebhooksEvent } from "../events.js";
|
|
5
|
-
import { enqueueWebhookDelivery } from "./queue.js";
|
|
7
|
+
import { enqueueWebhookDelivery, enqueueWebhookDeliveryFromJob } from "./queue.js";
|
|
6
8
|
import { isWebhookIntegrationEnabled, WEBHOOK_INTEGRATION_DISABLED_MESSAGE } from "./integration-state.js";
|
|
7
9
|
import { sanitizeWebhookCustomHeaders } from "./custom-headers.js";
|
|
8
10
|
import {
|
|
@@ -13,6 +15,11 @@ import {
|
|
|
13
15
|
import {
|
|
14
16
|
invalidateWebhookSubscriptionCacheFor
|
|
15
17
|
} from "./subscription-cache.js";
|
|
18
|
+
const logger = createLogger("webhooks");
|
|
19
|
+
const CLAIMABLE_DELIVERY_STATUSES = /* @__PURE__ */ new Set(["pending", "sending"]);
|
|
20
|
+
const SENDING_LEASE_GRACE_MS = 15e3;
|
|
21
|
+
const CLAIM_RECHECK_MARGIN_MS = 1e3;
|
|
22
|
+
const RETRY_DUE_TOLERANCE_MS = 1e3;
|
|
16
23
|
async function createWebhookDelivery(input) {
|
|
17
24
|
const bodyPayload = {
|
|
18
25
|
type: input.eventId,
|
|
@@ -52,6 +59,7 @@ async function processWebhookDeliveryJob(em, job, options = {}) {
|
|
|
52
59
|
organizationId: job.organizationId
|
|
53
60
|
});
|
|
54
61
|
if (!delivery) return null;
|
|
62
|
+
if (!CLAIMABLE_DELIVERY_STATUSES.has(delivery.status)) return null;
|
|
55
63
|
const webhook = await findOneWithDecryption(
|
|
56
64
|
em,
|
|
57
65
|
WebhookEntity,
|
|
@@ -64,6 +72,24 @@ async function processWebhookDeliveryJob(em, job, options = {}) {
|
|
|
64
72
|
{},
|
|
65
73
|
{ tenantId: job.tenantId, organizationId: job.organizationId }
|
|
66
74
|
);
|
|
75
|
+
const claimedAt = /* @__PURE__ */ new Date();
|
|
76
|
+
const leaseMs = (webhook?.timeoutMs ?? 0) + SENDING_LEASE_GRACE_MS;
|
|
77
|
+
const claimed = await claimWebhookDelivery(em, delivery, claimedAt, leaseMs);
|
|
78
|
+
if (!claimed) {
|
|
79
|
+
if (options.scheduleRetries !== false) {
|
|
80
|
+
const retryDueInMs = (delivery.nextRetryAt?.getTime() ?? 0) - claimedAt.getTime();
|
|
81
|
+
await scheduleWebhookDeliveryClaimRecheck(job, Math.max(leaseMs, retryDueInMs));
|
|
82
|
+
}
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
try {
|
|
86
|
+
return await processClaimedWebhookDelivery(em, delivery, webhook, options, claimedAt);
|
|
87
|
+
} catch (error) {
|
|
88
|
+
await releaseWebhookDeliveryClaim(em, delivery, claimedAt);
|
|
89
|
+
throw error;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
async function processClaimedWebhookDelivery(em, delivery, webhook, options, claimedAt) {
|
|
67
93
|
if (!webhook) {
|
|
68
94
|
delivery.status = "failed";
|
|
69
95
|
delivery.errorMessage = "Webhook not found";
|
|
@@ -102,7 +128,7 @@ async function processWebhookDeliveryJob(em, job, options = {}) {
|
|
|
102
128
|
const bodyPayload = normalizeWebhookBody(delivery.eventType, delivery.payload);
|
|
103
129
|
delivery.payload = bodyPayload;
|
|
104
130
|
delivery.status = "sending";
|
|
105
|
-
delivery.lastAttemptAt =
|
|
131
|
+
delivery.lastAttemptAt = claimedAt;
|
|
106
132
|
delivery.errorMessage = null;
|
|
107
133
|
await em.flush();
|
|
108
134
|
const body = JSON.stringify(bodyPayload);
|
|
@@ -217,6 +243,49 @@ async function processWebhookDeliveryJob(em, job, options = {}) {
|
|
|
217
243
|
return { status: delivery.status, deliveryId: delivery.id };
|
|
218
244
|
}
|
|
219
245
|
}
|
|
246
|
+
async function claimWebhookDelivery(em, delivery, claimedAt, leaseMs) {
|
|
247
|
+
const staleSendingBefore = new Date(claimedAt.getTime() - leaseMs);
|
|
248
|
+
const retryDueBy = new Date(claimedAt.getTime() + RETRY_DUE_TOLERANCE_MS);
|
|
249
|
+
const affected = await em.nativeUpdate(
|
|
250
|
+
WebhookDeliveryEntity,
|
|
251
|
+
{
|
|
252
|
+
id: delivery.id,
|
|
253
|
+
tenantId: delivery.tenantId,
|
|
254
|
+
organizationId: delivery.organizationId,
|
|
255
|
+
$or: [
|
|
256
|
+
{ status: "pending", nextRetryAt: null },
|
|
257
|
+
{ status: "pending", nextRetryAt: { $lte: retryDueBy } },
|
|
258
|
+
{ status: "sending", lastAttemptAt: { $lt: staleSendingBefore } }
|
|
259
|
+
]
|
|
260
|
+
},
|
|
261
|
+
{ status: "sending", lastAttemptAt: claimedAt, errorMessage: null, updatedAt: claimedAt }
|
|
262
|
+
);
|
|
263
|
+
return affected > 0;
|
|
264
|
+
}
|
|
265
|
+
async function releaseWebhookDeliveryClaim(em, delivery, claimedAt) {
|
|
266
|
+
try {
|
|
267
|
+
await em.nativeUpdate(
|
|
268
|
+
WebhookDeliveryEntity,
|
|
269
|
+
{
|
|
270
|
+
id: delivery.id,
|
|
271
|
+
tenantId: delivery.tenantId,
|
|
272
|
+
organizationId: delivery.organizationId,
|
|
273
|
+
status: "sending",
|
|
274
|
+
lastAttemptAt: claimedAt
|
|
275
|
+
},
|
|
276
|
+
{ status: "pending", updatedAt: /* @__PURE__ */ new Date() }
|
|
277
|
+
);
|
|
278
|
+
} catch (error) {
|
|
279
|
+
logger.error("Failed to release webhook delivery claim", { deliveryId: delivery.id, err: error });
|
|
280
|
+
getTelemetryRuntime()?.reportError(error, { module: "webhooks", code: "webhooks.delivery_claim_release_failed" });
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
async function scheduleWebhookDeliveryClaimRecheck(job, waitMs) {
|
|
284
|
+
await enqueueWebhookDeliveryFromJob(
|
|
285
|
+
{ deliveryId: job.deliveryId, tenantId: job.tenantId, organizationId: job.organizationId },
|
|
286
|
+
waitMs + CLAIM_RECHECK_MARGIN_MS
|
|
287
|
+
);
|
|
288
|
+
}
|
|
220
289
|
async function recordWebhookSafetyFailure(input) {
|
|
221
290
|
const { em, delivery, webhook, error } = input;
|
|
222
291
|
const message = error instanceof UnsafeWebhookUrlError ? error.message : "Webhook URL rejected by safety check";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/webhooks/lib/delivery.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { buildWebhookHeaders, generateMessageId } from '@open-mercato/shared/lib/webhooks'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\nimport { WebhookDeliveryEntity, WebhookEntity } from '../data/entities'\nimport { emitWebhooksEvent } from '../events'\nimport { enqueueWebhookDelivery } from './queue'\nimport { isWebhookIntegrationEnabled, WEBHOOK_INTEGRATION_DISABLED_MESSAGE } from './integration-state'\nimport { sanitizeWebhookCustomHeaders } from './custom-headers'\nimport {\n assertSafeWebhookDeliveryUrl,\n safeWebhookFetch,\n UnsafeWebhookUrlError,\n} from './url-safety'\nimport {\n invalidateWebhookSubscriptionCacheFor,\n type WebhookCacheDependencyResolver,\n} from './subscription-cache'\n\nexport interface WebhookDeliveryJob {\n deliveryId: string\n tenantId: string\n organizationId: string\n}\n\nexport interface CreateWebhookDeliveryInput {\n em: EntityManager\n webhook: WebhookEntity\n eventId: string\n payload: Record<string, unknown>\n}\n\ntype ProcessWebhookDeliveryOptions = {\n scheduleRetries?: boolean\n resolver?: WebhookCacheDependencyResolver\n}\n\ntype WebhookBody = {\n type: string\n timestamp: string\n data: Record<string, unknown>\n}\n\nexport async function createWebhookDelivery(input: CreateWebhookDeliveryInput): Promise<WebhookDeliveryEntity> {\n const bodyPayload: WebhookBody = {\n type: input.eventId,\n timestamp: new Date().toISOString(),\n data: input.payload,\n }\n const now = new Date()\n\n const delivery = input.em.create(WebhookDeliveryEntity, {\n webhookId: input.webhook.id,\n eventType: input.eventId,\n messageId: generateMessageId(),\n payload: bodyPayload,\n status: 'pending',\n attemptNumber: 0,\n maxAttempts: input.webhook.maxRetries,\n targetUrl: input.webhook.url,\n organizationId: input.webhook.organizationId,\n tenantId: input.webhook.tenantId,\n enqueuedAt: now,\n createdAt: now,\n updatedAt: now,\n })\n\n await input.em.flush()\n await emitWebhooksEvent('webhooks.delivery.enqueued', {\n deliveryId: delivery.id,\n webhookId: input.webhook.id,\n eventType: input.eventId,\n organizationId: input.webhook.organizationId,\n tenantId: input.webhook.tenantId,\n })\n return delivery\n}\n\nexport async function processWebhookDeliveryJob(\n em: EntityManager,\n job: WebhookDeliveryJob,\n options: ProcessWebhookDeliveryOptions = {},\n): Promise<{ status: string; deliveryId: string } | null> {\n const delivery = await em.findOne(WebhookDeliveryEntity, {\n id: job.deliveryId,\n tenantId: job.tenantId,\n organizationId: job.organizationId,\n })\n\n if (!delivery) return null\n\n const webhook = await findOneWithDecryption(\n em,\n WebhookEntity,\n {\n id: delivery.webhookId,\n tenantId: job.tenantId,\n organizationId: job.organizationId,\n deletedAt: null,\n },\n {},\n { tenantId: job.tenantId, organizationId: job.organizationId },\n )\n\n if (!webhook) {\n delivery.status = 'failed'\n delivery.errorMessage = 'Webhook not found'\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n if (!webhook.isActive) {\n delivery.status = 'expired'\n delivery.errorMessage = 'Webhook is inactive'\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n const integrationEnabled = await isWebhookIntegrationEnabled(em, {\n tenantId: webhook.tenantId,\n organizationId: webhook.organizationId,\n })\n\n if (!integrationEnabled) {\n delivery.status = 'expired'\n delivery.errorMessage = WEBHOOK_INTEGRATION_DISABLED_MESSAGE\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n try {\n await assertSafeWebhookDeliveryUrl(webhook.url)\n } catch (error) {\n return await recordWebhookSafetyFailure({\n em,\n delivery,\n webhook,\n error,\n })\n }\n\n const bodyPayload = normalizeWebhookBody(delivery.eventType, delivery.payload)\n delivery.payload = bodyPayload\n delivery.status = 'sending'\n delivery.lastAttemptAt = new Date()\n delivery.errorMessage = null\n await em.flush()\n\n const body = JSON.stringify(bodyPayload)\n const timestamp = Math.floor(Date.now() / 1000)\n const headers = buildWebhookHeaders(\n delivery.messageId,\n timestamp,\n body,\n webhook.secret,\n webhook.previousSecret,\n )\n\n const attemptStartedAt = Date.now()\n\n try {\n const controller = new AbortController()\n const timeoutId = setTimeout(() => controller.abort(), webhook.timeoutMs)\n\n let response: Response\n try {\n response = await safeWebhookFetch(webhook.url, {\n method: webhook.httpMethod,\n redirect: 'manual',\n headers: {\n 'content-type': 'application/json',\n ...sanitizeWebhookCustomHeaders(webhook.customHeaders),\n ...headers,\n },\n body,\n signal: controller.signal,\n })\n } finally {\n clearTimeout(timeoutId)\n }\n\n delivery.attemptNumber += 1\n delivery.responseStatus = response.status\n delivery.responseBody = (await response.text()).slice(0, 4096)\n delivery.responseHeaders = Object.fromEntries(response.headers.entries())\n delivery.durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.lastAttemptAt = new Date()\n\n if (response.ok) {\n delivery.status = 'delivered'\n delivery.deliveredAt = new Date()\n delivery.nextRetryAt = null\n webhook.consecutiveFailures = 0\n webhook.lastSuccessAt = new Date()\n\n await emitWebhooksEvent('webhooks.delivery.succeeded', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n })\n\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n\n await handleFailedDelivery({\n em,\n webhook,\n delivery,\n canRetry: shouldRetryStatus(response.status),\n scheduleRetries: options.scheduleRetries !== false,\n resolver: options.resolver,\n fallbackMessage: `HTTP ${response.status}`,\n })\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n responseStatus: response.status,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: delivery.status === 'pending',\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n } catch (error) {\n if (error instanceof UnsafeWebhookUrlError) {\n // DNS rebinding caught between the upfront safety check and the pinned connect:\n // the same attacker-controlled DNS would defeat retries too, so fail terminally.\n const durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.durationMs = durationMs\n return await recordWebhookSafetyFailure({\n em,\n delivery,\n webhook,\n error,\n durationMs,\n })\n }\n\n delivery.attemptNumber += 1\n delivery.durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.lastAttemptAt = new Date()\n delivery.errorMessage = error instanceof Error ? error.message : 'Unknown delivery error'\n\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n\n await handleFailedDelivery({\n em,\n webhook,\n delivery,\n canRetry: true,\n scheduleRetries: options.scheduleRetries !== false,\n resolver: options.resolver,\n fallbackMessage: delivery.errorMessage,\n })\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n errorMessage: delivery.errorMessage,\n durationMs: Date.now() - attemptStartedAt,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: delivery.status === 'pending',\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n }\n}\n\ntype RecordWebhookSafetyFailureInput = {\n em: EntityManager\n delivery: WebhookDeliveryEntity\n webhook: WebhookEntity\n error: unknown\n durationMs?: number | null\n}\n\nasync function recordWebhookSafetyFailure(\n input: RecordWebhookSafetyFailureInput,\n): Promise<{ status: string; deliveryId: string }> {\n const { em, delivery, webhook, error } = input\n const message = error instanceof UnsafeWebhookUrlError\n ? error.message\n : 'Webhook URL rejected by safety check'\n\n delivery.status = 'failed'\n delivery.errorMessage = message\n delivery.nextRetryAt = null\n delivery.lastAttemptAt = new Date()\n delivery.attemptNumber += 1\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n await em.flush()\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n errorMessage: message,\n durationMs: input.durationMs ?? 0,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: false,\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n}\n\ntype HandleFailedDeliveryInput = {\n em: EntityManager\n webhook: WebhookEntity\n delivery: WebhookDeliveryEntity\n canRetry: boolean\n scheduleRetries: boolean\n fallbackMessage: string\n resolver?: WebhookCacheDependencyResolver\n}\n\nasync function handleFailedDelivery(input: HandleFailedDeliveryInput): Promise<void> {\n const { delivery, webhook } = input\n const retriesRemaining = delivery.attemptNumber < Math.max(delivery.maxAttempts, 1)\n const shouldRetry = input.canRetry && retriesRemaining\n\n if (shouldRetry) {\n const nextRetryAt = calculateNextRetry(delivery.attemptNumber)\n delivery.status = 'pending'\n delivery.nextRetryAt = nextRetryAt\n\n await input.em.flush()\n\n if (input.scheduleRetries) {\n try {\n await enqueueWebhookDelivery(\n {\n deliveryId: delivery.id,\n tenantId: delivery.tenantId,\n organizationId: delivery.organizationId,\n },\n Math.max(nextRetryAt.getTime() - Date.now(), 0),\n )\n } catch (error) {\n delivery.status = 'failed'\n delivery.nextRetryAt = null\n delivery.errorMessage = error instanceof Error\n ? `Retry scheduling failed: ${error.message}`\n : 'Retry scheduling failed'\n }\n }\n } else {\n delivery.status = 'expired'\n delivery.nextRetryAt = null\n\n await emitWebhooksEvent('webhooks.delivery.exhausted', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n errorMessage: delivery.errorMessage ?? input.fallbackMessage,\n })\n }\n\n await input.em.flush()\n\n if (webhook.autoDisableThreshold > 0 && webhook.consecutiveFailures >= webhook.autoDisableThreshold) {\n webhook.isActive = false\n await input.em.flush()\n await emitWebhooksEvent('webhooks.webhook.disabled', {\n webhookId: webhook.id,\n organizationId: webhook.organizationId,\n tenantId: webhook.tenantId,\n consecutiveFailures: webhook.consecutiveFailures,\n })\n await invalidateWebhookSubscriptionCacheFor(input.resolver, webhook.tenantId)\n }\n}\n\nfunction normalizeWebhookBody(eventType: string, payload: Record<string, unknown>): WebhookBody {\n const candidateType = typeof payload.type === 'string' ? payload.type : null\n const candidateTimestamp = typeof payload.timestamp === 'string' ? payload.timestamp : null\n const candidateData = isRecord(payload.data) ? payload.data : null\n\n if (candidateType && candidateTimestamp && candidateData) {\n return {\n type: candidateType,\n timestamp: candidateTimestamp,\n data: candidateData,\n }\n }\n\n return {\n type: eventType,\n timestamp: new Date().toISOString(),\n data: payload,\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction shouldRetryStatus(status: number): boolean {\n if (status >= 200 && status < 300) return false\n if (status === 408 || status === 429) return true\n return status >= 500\n}\n\nfunction calculateNextRetry(attemptNumber: number): Date {\n const baseDelayMs = 1000\n const jitterMs = Math.floor(Math.random() * 1000)\n const delayMs = baseDelayMs * Math.pow(2, Math.max(attemptNumber - 1, 0)) + jitterMs\n return new Date(Date.now() + delayMs)\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,qBAAqB,yBAAyB;AACvD,SAAS,6BAA6B;AACtC,SAAS,uBAAuB,qBAAqB;AACrD,SAAS,yBAAyB;AAClC,SAAS,
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { buildWebhookHeaders, generateMessageId } from '@open-mercato/shared/lib/webhooks'\nimport { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'\nimport { WebhookDeliveryEntity, WebhookEntity } from '../data/entities'\nimport { emitWebhooksEvent } from '../events'\nimport { enqueueWebhookDelivery, enqueueWebhookDeliveryFromJob } from './queue'\nimport { isWebhookIntegrationEnabled, WEBHOOK_INTEGRATION_DISABLED_MESSAGE } from './integration-state'\nimport { sanitizeWebhookCustomHeaders } from './custom-headers'\nimport {\n assertSafeWebhookDeliveryUrl,\n safeWebhookFetch,\n UnsafeWebhookUrlError,\n} from './url-safety'\nimport {\n invalidateWebhookSubscriptionCacheFor,\n type WebhookCacheDependencyResolver,\n} from './subscription-cache'\n\nexport interface WebhookDeliveryJob {\n deliveryId: string\n tenantId: string\n organizationId: string\n}\n\nexport interface CreateWebhookDeliveryInput {\n em: EntityManager\n webhook: WebhookEntity\n eventId: string\n payload: Record<string, unknown>\n}\n\nconst logger = createLogger('webhooks')\n\nconst CLAIMABLE_DELIVERY_STATUSES: ReadonlySet<string> = new Set(['pending', 'sending'])\nconst SENDING_LEASE_GRACE_MS = 15_000\nconst CLAIM_RECHECK_MARGIN_MS = 1_000\nconst RETRY_DUE_TOLERANCE_MS = 1_000\n\ntype ProcessWebhookDeliveryOptions = {\n scheduleRetries?: boolean\n resolver?: WebhookCacheDependencyResolver\n}\n\ntype WebhookBody = {\n type: string\n timestamp: string\n data: Record<string, unknown>\n}\n\nexport async function createWebhookDelivery(input: CreateWebhookDeliveryInput): Promise<WebhookDeliveryEntity> {\n const bodyPayload: WebhookBody = {\n type: input.eventId,\n timestamp: new Date().toISOString(),\n data: input.payload,\n }\n const now = new Date()\n\n const delivery = input.em.create(WebhookDeliveryEntity, {\n webhookId: input.webhook.id,\n eventType: input.eventId,\n messageId: generateMessageId(),\n payload: bodyPayload,\n status: 'pending',\n attemptNumber: 0,\n maxAttempts: input.webhook.maxRetries,\n targetUrl: input.webhook.url,\n organizationId: input.webhook.organizationId,\n tenantId: input.webhook.tenantId,\n enqueuedAt: now,\n createdAt: now,\n updatedAt: now,\n })\n\n await input.em.flush()\n await emitWebhooksEvent('webhooks.delivery.enqueued', {\n deliveryId: delivery.id,\n webhookId: input.webhook.id,\n eventType: input.eventId,\n organizationId: input.webhook.organizationId,\n tenantId: input.webhook.tenantId,\n })\n return delivery\n}\n\nexport async function processWebhookDeliveryJob(\n em: EntityManager,\n job: WebhookDeliveryJob,\n options: ProcessWebhookDeliveryOptions = {},\n): Promise<{ status: string; deliveryId: string } | null> {\n const delivery = await em.findOne(WebhookDeliveryEntity, {\n id: job.deliveryId,\n tenantId: job.tenantId,\n organizationId: job.organizationId,\n })\n\n if (!delivery) return null\n if (!CLAIMABLE_DELIVERY_STATUSES.has(delivery.status)) return null\n\n const webhook = await findOneWithDecryption(\n em,\n WebhookEntity,\n {\n id: delivery.webhookId,\n tenantId: job.tenantId,\n organizationId: job.organizationId,\n deletedAt: null,\n },\n {},\n { tenantId: job.tenantId, organizationId: job.organizationId },\n )\n\n const claimedAt = new Date()\n const leaseMs = (webhook?.timeoutMs ?? 0) + SENDING_LEASE_GRACE_MS\n const claimed = await claimWebhookDelivery(em, delivery, claimedAt, leaseMs)\n if (!claimed) {\n if (options.scheduleRetries !== false) {\n const retryDueInMs = (delivery.nextRetryAt?.getTime() ?? 0) - claimedAt.getTime()\n await scheduleWebhookDeliveryClaimRecheck(job, Math.max(leaseMs, retryDueInMs))\n }\n return null\n }\n\n try {\n return await processClaimedWebhookDelivery(em, delivery, webhook, options, claimedAt)\n } catch (error) {\n await releaseWebhookDeliveryClaim(em, delivery, claimedAt)\n throw error\n }\n}\n\nasync function processClaimedWebhookDelivery(\n em: EntityManager,\n delivery: WebhookDeliveryEntity,\n webhook: WebhookEntity | null,\n options: ProcessWebhookDeliveryOptions,\n claimedAt: Date,\n): Promise<{ status: string; deliveryId: string }> {\n if (!webhook) {\n delivery.status = 'failed'\n delivery.errorMessage = 'Webhook not found'\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n if (!webhook.isActive) {\n delivery.status = 'expired'\n delivery.errorMessage = 'Webhook is inactive'\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n const integrationEnabled = await isWebhookIntegrationEnabled(em, {\n tenantId: webhook.tenantId,\n organizationId: webhook.organizationId,\n })\n\n if (!integrationEnabled) {\n delivery.status = 'expired'\n delivery.errorMessage = WEBHOOK_INTEGRATION_DISABLED_MESSAGE\n delivery.nextRetryAt = null\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n try {\n await assertSafeWebhookDeliveryUrl(webhook.url)\n } catch (error) {\n return await recordWebhookSafetyFailure({\n em,\n delivery,\n webhook,\n error,\n })\n }\n\n const bodyPayload = normalizeWebhookBody(delivery.eventType, delivery.payload)\n delivery.payload = bodyPayload\n delivery.status = 'sending'\n delivery.lastAttemptAt = claimedAt\n delivery.errorMessage = null\n await em.flush()\n\n const body = JSON.stringify(bodyPayload)\n const timestamp = Math.floor(Date.now() / 1000)\n const headers = buildWebhookHeaders(\n delivery.messageId,\n timestamp,\n body,\n webhook.secret,\n webhook.previousSecret,\n )\n\n const attemptStartedAt = Date.now()\n\n try {\n const controller = new AbortController()\n const timeoutId = setTimeout(() => controller.abort(), webhook.timeoutMs)\n\n let response: Response\n try {\n response = await safeWebhookFetch(webhook.url, {\n method: webhook.httpMethod,\n redirect: 'manual',\n headers: {\n 'content-type': 'application/json',\n ...sanitizeWebhookCustomHeaders(webhook.customHeaders),\n ...headers,\n },\n body,\n signal: controller.signal,\n })\n } finally {\n clearTimeout(timeoutId)\n }\n\n delivery.attemptNumber += 1\n delivery.responseStatus = response.status\n delivery.responseBody = (await response.text()).slice(0, 4096)\n delivery.responseHeaders = Object.fromEntries(response.headers.entries())\n delivery.durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.lastAttemptAt = new Date()\n\n if (response.ok) {\n delivery.status = 'delivered'\n delivery.deliveredAt = new Date()\n delivery.nextRetryAt = null\n webhook.consecutiveFailures = 0\n webhook.lastSuccessAt = new Date()\n\n await emitWebhooksEvent('webhooks.delivery.succeeded', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n })\n\n await em.flush()\n return { status: delivery.status, deliveryId: delivery.id }\n }\n\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n\n await handleFailedDelivery({\n em,\n webhook,\n delivery,\n canRetry: shouldRetryStatus(response.status),\n scheduleRetries: options.scheduleRetries !== false,\n resolver: options.resolver,\n fallbackMessage: `HTTP ${response.status}`,\n })\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n responseStatus: response.status,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: delivery.status === 'pending',\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n } catch (error) {\n if (error instanceof UnsafeWebhookUrlError) {\n // DNS rebinding caught between the upfront safety check and the pinned connect:\n // the same attacker-controlled DNS would defeat retries too, so fail terminally.\n const durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.durationMs = durationMs\n return await recordWebhookSafetyFailure({\n em,\n delivery,\n webhook,\n error,\n durationMs,\n })\n }\n\n delivery.attemptNumber += 1\n delivery.durationMs = Date.now() - delivery.enqueuedAt.getTime()\n delivery.lastAttemptAt = new Date()\n delivery.errorMessage = error instanceof Error ? error.message : 'Unknown delivery error'\n\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n\n await handleFailedDelivery({\n em,\n webhook,\n delivery,\n canRetry: true,\n scheduleRetries: options.scheduleRetries !== false,\n resolver: options.resolver,\n fallbackMessage: delivery.errorMessage,\n })\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n errorMessage: delivery.errorMessage,\n durationMs: Date.now() - attemptStartedAt,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: delivery.status === 'pending',\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n }\n}\n\nasync function claimWebhookDelivery(\n em: EntityManager,\n delivery: WebhookDeliveryEntity,\n claimedAt: Date,\n leaseMs: number,\n): Promise<boolean> {\n const staleSendingBefore = new Date(claimedAt.getTime() - leaseMs)\n const retryDueBy = new Date(claimedAt.getTime() + RETRY_DUE_TOLERANCE_MS)\n const affected = await em.nativeUpdate(\n WebhookDeliveryEntity,\n {\n id: delivery.id,\n tenantId: delivery.tenantId,\n organizationId: delivery.organizationId,\n $or: [\n { status: 'pending', nextRetryAt: null },\n { status: 'pending', nextRetryAt: { $lte: retryDueBy } },\n { status: 'sending', lastAttemptAt: { $lt: staleSendingBefore } },\n ],\n },\n { status: 'sending', lastAttemptAt: claimedAt, errorMessage: null, updatedAt: claimedAt },\n )\n return affected > 0\n}\n\nasync function releaseWebhookDeliveryClaim(\n em: EntityManager,\n delivery: WebhookDeliveryEntity,\n claimedAt: Date,\n): Promise<void> {\n try {\n await em.nativeUpdate(\n WebhookDeliveryEntity,\n {\n id: delivery.id,\n tenantId: delivery.tenantId,\n organizationId: delivery.organizationId,\n status: 'sending',\n lastAttemptAt: claimedAt,\n },\n { status: 'pending', updatedAt: new Date() },\n )\n } catch (error) {\n logger.error('Failed to release webhook delivery claim', { deliveryId: delivery.id, err: error })\n getTelemetryRuntime()?.reportError(error, { module: 'webhooks', code: 'webhooks.delivery_claim_release_failed' })\n }\n}\n\nasync function scheduleWebhookDeliveryClaimRecheck(job: WebhookDeliveryJob, waitMs: number): Promise<void> {\n await enqueueWebhookDeliveryFromJob(\n { deliveryId: job.deliveryId, tenantId: job.tenantId, organizationId: job.organizationId },\n waitMs + CLAIM_RECHECK_MARGIN_MS,\n )\n}\n\ntype RecordWebhookSafetyFailureInput = {\n em: EntityManager\n delivery: WebhookDeliveryEntity\n webhook: WebhookEntity\n error: unknown\n durationMs?: number | null\n}\n\nasync function recordWebhookSafetyFailure(\n input: RecordWebhookSafetyFailureInput,\n): Promise<{ status: string; deliveryId: string }> {\n const { em, delivery, webhook, error } = input\n const message = error instanceof UnsafeWebhookUrlError\n ? error.message\n : 'Webhook URL rejected by safety check'\n\n delivery.status = 'failed'\n delivery.errorMessage = message\n delivery.nextRetryAt = null\n delivery.lastAttemptAt = new Date()\n delivery.attemptNumber += 1\n webhook.consecutiveFailures += 1\n webhook.lastFailureAt = new Date()\n await em.flush()\n\n await emitWebhooksEvent('webhooks.delivery.failed', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n errorMessage: message,\n durationMs: input.durationMs ?? 0,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n willRetry: false,\n })\n\n return { status: delivery.status, deliveryId: delivery.id }\n}\n\ntype HandleFailedDeliveryInput = {\n em: EntityManager\n webhook: WebhookEntity\n delivery: WebhookDeliveryEntity\n canRetry: boolean\n scheduleRetries: boolean\n fallbackMessage: string\n resolver?: WebhookCacheDependencyResolver\n}\n\nasync function handleFailedDelivery(input: HandleFailedDeliveryInput): Promise<void> {\n const { delivery, webhook } = input\n const retriesRemaining = delivery.attemptNumber < Math.max(delivery.maxAttempts, 1)\n const shouldRetry = input.canRetry && retriesRemaining\n\n if (shouldRetry) {\n const nextRetryAt = calculateNextRetry(delivery.attemptNumber)\n delivery.status = 'pending'\n delivery.nextRetryAt = nextRetryAt\n\n await input.em.flush()\n\n if (input.scheduleRetries) {\n try {\n await enqueueWebhookDelivery(\n {\n deliveryId: delivery.id,\n tenantId: delivery.tenantId,\n organizationId: delivery.organizationId,\n },\n Math.max(nextRetryAt.getTime() - Date.now(), 0),\n )\n } catch (error) {\n delivery.status = 'failed'\n delivery.nextRetryAt = null\n delivery.errorMessage = error instanceof Error\n ? `Retry scheduling failed: ${error.message}`\n : 'Retry scheduling failed'\n }\n }\n } else {\n delivery.status = 'expired'\n delivery.nextRetryAt = null\n\n await emitWebhooksEvent('webhooks.delivery.exhausted', {\n deliveryId: delivery.id,\n webhookId: webhook.id,\n eventType: delivery.eventType,\n organizationId: delivery.organizationId,\n tenantId: delivery.tenantId,\n errorMessage: delivery.errorMessage ?? input.fallbackMessage,\n })\n }\n\n await input.em.flush()\n\n if (webhook.autoDisableThreshold > 0 && webhook.consecutiveFailures >= webhook.autoDisableThreshold) {\n webhook.isActive = false\n await input.em.flush()\n await emitWebhooksEvent('webhooks.webhook.disabled', {\n webhookId: webhook.id,\n organizationId: webhook.organizationId,\n tenantId: webhook.tenantId,\n consecutiveFailures: webhook.consecutiveFailures,\n })\n await invalidateWebhookSubscriptionCacheFor(input.resolver, webhook.tenantId)\n }\n}\n\nfunction normalizeWebhookBody(eventType: string, payload: Record<string, unknown>): WebhookBody {\n const candidateType = typeof payload.type === 'string' ? payload.type : null\n const candidateTimestamp = typeof payload.timestamp === 'string' ? payload.timestamp : null\n const candidateData = isRecord(payload.data) ? payload.data : null\n\n if (candidateType && candidateTimestamp && candidateData) {\n return {\n type: candidateType,\n timestamp: candidateTimestamp,\n data: candidateData,\n }\n }\n\n return {\n type: eventType,\n timestamp: new Date().toISOString(),\n data: payload,\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value)\n}\n\nfunction shouldRetryStatus(status: number): boolean {\n if (status >= 200 && status < 300) return false\n if (status === 408 || status === 429) return true\n return status >= 500\n}\n\nfunction calculateNextRetry(attemptNumber: number): Date {\n const baseDelayMs = 1000\n const jitterMs = Math.floor(Math.random() * 1000)\n const delayMs = baseDelayMs * Math.pow(2, Math.max(attemptNumber - 1, 0)) + jitterMs\n return new Date(Date.now() + delayMs)\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,qBAAqB,yBAAyB;AACvD,SAAS,6BAA6B;AACtC,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,uBAAuB,qBAAqB;AACrD,SAAS,yBAAyB;AAClC,SAAS,wBAAwB,qCAAqC;AACtE,SAAS,6BAA6B,4CAA4C;AAClF,SAAS,oCAAoC;AAC7C;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,OAEK;AAeP,MAAM,SAAS,aAAa,UAAU;AAEtC,MAAM,8BAAmD,oBAAI,IAAI,CAAC,WAAW,SAAS,CAAC;AACvF,MAAM,yBAAyB;AAC/B,MAAM,0BAA0B;AAChC,MAAM,yBAAyB;AAa/B,eAAsB,sBAAsB,OAAmE;AAC7G,QAAM,cAA2B;AAAA,IAC/B,MAAM,MAAM;AAAA,IACZ,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC,MAAM,MAAM;AAAA,EACd;AACA,QAAM,MAAM,oBAAI,KAAK;AAErB,QAAM,WAAW,MAAM,GAAG,OAAO,uBAAuB;AAAA,IACtD,WAAW,MAAM,QAAQ;AAAA,IACzB,WAAW,MAAM;AAAA,IACjB,WAAW,kBAAkB;AAAA,IAC7B,SAAS;AAAA,IACT,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,aAAa,MAAM,QAAQ;AAAA,IAC3B,WAAW,MAAM,QAAQ;AAAA,IACzB,gBAAgB,MAAM,QAAQ;AAAA,IAC9B,UAAU,MAAM,QAAQ;AAAA,IACxB,YAAY;AAAA,IACZ,WAAW;AAAA,IACX,WAAW;AAAA,EACb,CAAC;AAED,QAAM,MAAM,GAAG,MAAM;AACrB,QAAM,kBAAkB,8BAA8B;AAAA,IACpD,YAAY,SAAS;AAAA,IACrB,WAAW,MAAM,QAAQ;AAAA,IACzB,WAAW,MAAM;AAAA,IACjB,gBAAgB,MAAM,QAAQ;AAAA,IAC9B,UAAU,MAAM,QAAQ;AAAA,EAC1B,CAAC;AACD,SAAO;AACT;AAEA,eAAsB,0BACpB,IACA,KACA,UAAyC,CAAC,GACc;AACxD,QAAM,WAAW,MAAM,GAAG,QAAQ,uBAAuB;AAAA,IACvD,IAAI,IAAI;AAAA,IACR,UAAU,IAAI;AAAA,IACd,gBAAgB,IAAI;AAAA,EACtB,CAAC;AAED,MAAI,CAAC,SAAU,QAAO;AACtB,MAAI,CAAC,4BAA4B,IAAI,SAAS,MAAM,EAAG,QAAO;AAE9D,QAAM,UAAU,MAAM;AAAA,IACpB;AAAA,IACA;AAAA,IACA;AAAA,MACE,IAAI,SAAS;AAAA,MACb,UAAU,IAAI;AAAA,MACd,gBAAgB,IAAI;AAAA,MACpB,WAAW;AAAA,IACb;AAAA,IACA,CAAC;AAAA,IACD,EAAE,UAAU,IAAI,UAAU,gBAAgB,IAAI,eAAe;AAAA,EAC/D;AAEA,QAAM,YAAY,oBAAI,KAAK;AAC3B,QAAM,WAAW,SAAS,aAAa,KAAK;AAC5C,QAAM,UAAU,MAAM,qBAAqB,IAAI,UAAU,WAAW,OAAO;AAC3E,MAAI,CAAC,SAAS;AACZ,QAAI,QAAQ,oBAAoB,OAAO;AACrC,YAAM,gBAAgB,SAAS,aAAa,QAAQ,KAAK,KAAK,UAAU,QAAQ;AAChF,YAAM,oCAAoC,KAAK,KAAK,IAAI,SAAS,YAAY,CAAC;AAAA,IAChF;AACA,WAAO;AAAA,EACT;AAEA,MAAI;AACF,WAAO,MAAM,8BAA8B,IAAI,UAAU,SAAS,SAAS,SAAS;AAAA,EACtF,SAAS,OAAO;AACd,UAAM,4BAA4B,IAAI,UAAU,SAAS;AACzD,UAAM;AAAA,EACR;AACF;AAEA,eAAe,8BACb,IACA,UACA,SACA,SACA,WACiD;AACjD,MAAI,CAAC,SAAS;AACZ,aAAS,SAAS;AAClB,aAAS,eAAe;AACxB,aAAS,cAAc;AACvB,UAAM,GAAG,MAAM;AACf,WAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,EAC5D;AAEA,MAAI,CAAC,QAAQ,UAAU;AACrB,aAAS,SAAS;AAClB,aAAS,eAAe;AACxB,aAAS,cAAc;AACvB,UAAM,GAAG,MAAM;AACf,WAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,EAC5D;AAEA,QAAM,qBAAqB,MAAM,4BAA4B,IAAI;AAAA,IAC/D,UAAU,QAAQ;AAAA,IAClB,gBAAgB,QAAQ;AAAA,EAC1B,CAAC;AAED,MAAI,CAAC,oBAAoB;AACvB,aAAS,SAAS;AAClB,aAAS,eAAe;AACxB,aAAS,cAAc;AACvB,UAAM,GAAG,MAAM;AACf,WAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,EAC5D;AAEA,MAAI;AACF,UAAM,6BAA6B,QAAQ,GAAG;AAAA,EAChD,SAAS,OAAO;AACd,WAAO,MAAM,2BAA2B;AAAA,MACtC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,QAAM,cAAc,qBAAqB,SAAS,WAAW,SAAS,OAAO;AAC7E,WAAS,UAAU;AACnB,WAAS,SAAS;AAClB,WAAS,gBAAgB;AACzB,WAAS,eAAe;AACxB,QAAM,GAAG,MAAM;AAEf,QAAM,OAAO,KAAK,UAAU,WAAW;AACvC,QAAM,YAAY,KAAK,MAAM,KAAK,IAAI,IAAI,GAAI;AAC9C,QAAM,UAAU;AAAA,IACd,SAAS;AAAA,IACT;AAAA,IACA;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ;AAAA,EACV;AAEA,QAAM,mBAAmB,KAAK,IAAI;AAElC,MAAI;AACF,UAAM,aAAa,IAAI,gBAAgB;AACvC,UAAM,YAAY,WAAW,MAAM,WAAW,MAAM,GAAG,QAAQ,SAAS;AAExE,QAAI;AACJ,QAAI;AACF,iBAAW,MAAM,iBAAiB,QAAQ,KAAK;AAAA,QAC7C,QAAQ,QAAQ;AAAA,QAChB,UAAU;AAAA,QACV,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,GAAG,6BAA6B,QAAQ,aAAa;AAAA,UACrD,GAAG;AAAA,QACL;AAAA,QACA;AAAA,QACA,QAAQ,WAAW;AAAA,MACrB,CAAC;AAAA,IACH,UAAE;AACA,mBAAa,SAAS;AAAA,IACxB;AAEA,aAAS,iBAAiB;AAC1B,aAAS,iBAAiB,SAAS;AACnC,aAAS,gBAAgB,MAAM,SAAS,KAAK,GAAG,MAAM,GAAG,IAAI;AAC7D,aAAS,kBAAkB,OAAO,YAAY,SAAS,QAAQ,QAAQ,CAAC;AACxE,aAAS,aAAa,KAAK,IAAI,IAAI,SAAS,WAAW,QAAQ;AAC/D,aAAS,gBAAgB,oBAAI,KAAK;AAElC,QAAI,SAAS,IAAI;AACf,eAAS,SAAS;AAClB,eAAS,cAAc,oBAAI,KAAK;AAChC,eAAS,cAAc;AACvB,cAAQ,sBAAsB;AAC9B,cAAQ,gBAAgB,oBAAI,KAAK;AAEjC,YAAM,kBAAkB,+BAA+B;AAAA,QACrD,YAAY,SAAS;AAAA,QACrB,WAAW,QAAQ;AAAA,QACnB,WAAW,SAAS;AAAA,QACpB,gBAAgB,SAAS;AAAA,QACzB,UAAU,SAAS;AAAA,MACrB,CAAC;AAED,YAAM,GAAG,MAAM;AACf,aAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,IAC5D;AAEA,YAAQ,uBAAuB;AAC/B,YAAQ,gBAAgB,oBAAI,KAAK;AAEjC,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU,kBAAkB,SAAS,MAAM;AAAA,MAC3C,iBAAiB,QAAQ,oBAAoB;AAAA,MAC7C,UAAU,QAAQ;AAAA,MAClB,iBAAiB,QAAQ,SAAS,MAAM;AAAA,IAC1C,CAAC;AAED,UAAM,kBAAkB,4BAA4B;AAAA,MAClD,YAAY,SAAS;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,SAAS;AAAA,MACpB,gBAAgB,SAAS;AAAA,MACzB,gBAAgB,SAAS;AAAA,MACzB,UAAU,SAAS;AAAA,MACnB,WAAW,SAAS,WAAW;AAAA,IACjC,CAAC;AAED,WAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,EAC5D,SAAS,OAAO;AACd,QAAI,iBAAiB,uBAAuB;AAG1C,YAAM,aAAa,KAAK,IAAI,IAAI,SAAS,WAAW,QAAQ;AAC5D,eAAS,aAAa;AACtB,aAAO,MAAM,2BAA2B;AAAA,QACtC;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF,CAAC;AAAA,IACH;AAEA,aAAS,iBAAiB;AAC1B,aAAS,aAAa,KAAK,IAAI,IAAI,SAAS,WAAW,QAAQ;AAC/D,aAAS,gBAAgB,oBAAI,KAAK;AAClC,aAAS,eAAe,iBAAiB,QAAQ,MAAM,UAAU;AAEjE,YAAQ,uBAAuB;AAC/B,YAAQ,gBAAgB,oBAAI,KAAK;AAEjC,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,iBAAiB,QAAQ,oBAAoB;AAAA,MAC7C,UAAU,QAAQ;AAAA,MAClB,iBAAiB,SAAS;AAAA,IAC5B,CAAC;AAED,UAAM,kBAAkB,4BAA4B;AAAA,MAClD,YAAY,SAAS;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,SAAS;AAAA,MACpB,cAAc,SAAS;AAAA,MACvB,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB,gBAAgB,SAAS;AAAA,MACzB,UAAU,SAAS;AAAA,MACnB,WAAW,SAAS,WAAW;AAAA,IACjC,CAAC;AAED,WAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAAA,EAC5D;AACF;AAEA,eAAe,qBACb,IACA,UACA,WACA,SACkB;AAClB,QAAM,qBAAqB,IAAI,KAAK,UAAU,QAAQ,IAAI,OAAO;AACjE,QAAM,aAAa,IAAI,KAAK,UAAU,QAAQ,IAAI,sBAAsB;AACxE,QAAM,WAAW,MAAM,GAAG;AAAA,IACxB;AAAA,IACA;AAAA,MACE,IAAI,SAAS;AAAA,MACb,UAAU,SAAS;AAAA,MACnB,gBAAgB,SAAS;AAAA,MACzB,KAAK;AAAA,QACH,EAAE,QAAQ,WAAW,aAAa,KAAK;AAAA,QACvC,EAAE,QAAQ,WAAW,aAAa,EAAE,MAAM,WAAW,EAAE;AAAA,QACvD,EAAE,QAAQ,WAAW,eAAe,EAAE,KAAK,mBAAmB,EAAE;AAAA,MAClE;AAAA,IACF;AAAA,IACA,EAAE,QAAQ,WAAW,eAAe,WAAW,cAAc,MAAM,WAAW,UAAU;AAAA,EAC1F;AACA,SAAO,WAAW;AACpB;AAEA,eAAe,4BACb,IACA,UACA,WACe;AACf,MAAI;AACF,UAAM,GAAG;AAAA,MACP;AAAA,MACA;AAAA,QACE,IAAI,SAAS;AAAA,QACb,UAAU,SAAS;AAAA,QACnB,gBAAgB,SAAS;AAAA,QACzB,QAAQ;AAAA,QACR,eAAe;AAAA,MACjB;AAAA,MACA,EAAE,QAAQ,WAAW,WAAW,oBAAI,KAAK,EAAE;AAAA,IAC7C;AAAA,EACF,SAAS,OAAO;AACd,WAAO,MAAM,4CAA4C,EAAE,YAAY,SAAS,IAAI,KAAK,MAAM,CAAC;AAChG,wBAAoB,GAAG,YAAY,OAAO,EAAE,QAAQ,YAAY,MAAM,yCAAyC,CAAC;AAAA,EAClH;AACF;AAEA,eAAe,oCAAoC,KAAyB,QAA+B;AACzG,QAAM;AAAA,IACJ,EAAE,YAAY,IAAI,YAAY,UAAU,IAAI,UAAU,gBAAgB,IAAI,eAAe;AAAA,IACzF,SAAS;AAAA,EACX;AACF;AAUA,eAAe,2BACb,OACiD;AACjD,QAAM,EAAE,IAAI,UAAU,SAAS,MAAM,IAAI;AACzC,QAAM,UAAU,iBAAiB,wBAC7B,MAAM,UACN;AAEJ,WAAS,SAAS;AAClB,WAAS,eAAe;AACxB,WAAS,cAAc;AACvB,WAAS,gBAAgB,oBAAI,KAAK;AAClC,WAAS,iBAAiB;AAC1B,UAAQ,uBAAuB;AAC/B,UAAQ,gBAAgB,oBAAI,KAAK;AACjC,QAAM,GAAG,MAAM;AAEf,QAAM,kBAAkB,4BAA4B;AAAA,IAClD,YAAY,SAAS;AAAA,IACrB,WAAW,QAAQ;AAAA,IACnB,WAAW,SAAS;AAAA,IACpB,cAAc;AAAA,IACd,YAAY,MAAM,cAAc;AAAA,IAChC,gBAAgB,SAAS;AAAA,IACzB,UAAU,SAAS;AAAA,IACnB,WAAW;AAAA,EACb,CAAC;AAED,SAAO,EAAE,QAAQ,SAAS,QAAQ,YAAY,SAAS,GAAG;AAC5D;AAYA,eAAe,qBAAqB,OAAiD;AACnF,QAAM,EAAE,UAAU,QAAQ,IAAI;AAC9B,QAAM,mBAAmB,SAAS,gBAAgB,KAAK,IAAI,SAAS,aAAa,CAAC;AAClF,QAAM,cAAc,MAAM,YAAY;AAEtC,MAAI,aAAa;AACf,UAAM,cAAc,mBAAmB,SAAS,aAAa;AAC7D,aAAS,SAAS;AAClB,aAAS,cAAc;AAEvB,UAAM,MAAM,GAAG,MAAM;AAErB,QAAI,MAAM,iBAAiB;AACzB,UAAI;AACF,cAAM;AAAA,UACJ;AAAA,YACE,YAAY,SAAS;AAAA,YACrB,UAAU,SAAS;AAAA,YACnB,gBAAgB,SAAS;AAAA,UAC3B;AAAA,UACA,KAAK,IAAI,YAAY,QAAQ,IAAI,KAAK,IAAI,GAAG,CAAC;AAAA,QAChD;AAAA,MACF,SAAS,OAAO;AACd,iBAAS,SAAS;AAClB,iBAAS,cAAc;AACvB,iBAAS,eAAe,iBAAiB,QACrC,4BAA4B,MAAM,OAAO,KACzC;AAAA,MACN;AAAA,IACF;AAAA,EACF,OAAO;AACL,aAAS,SAAS;AAClB,aAAS,cAAc;AAEvB,UAAM,kBAAkB,+BAA+B;AAAA,MACrD,YAAY,SAAS;AAAA,MACrB,WAAW,QAAQ;AAAA,MACnB,WAAW,SAAS;AAAA,MACpB,gBAAgB,SAAS;AAAA,MACzB,UAAU,SAAS;AAAA,MACnB,cAAc,SAAS,gBAAgB,MAAM;AAAA,IAC/C,CAAC;AAAA,EACH;AAEA,QAAM,MAAM,GAAG,MAAM;AAErB,MAAI,QAAQ,uBAAuB,KAAK,QAAQ,uBAAuB,QAAQ,sBAAsB;AACnG,YAAQ,WAAW;AACnB,UAAM,MAAM,GAAG,MAAM;AACrB,UAAM,kBAAkB,6BAA6B;AAAA,MACnD,WAAW,QAAQ;AAAA,MACnB,gBAAgB,QAAQ;AAAA,MACxB,UAAU,QAAQ;AAAA,MAClB,qBAAqB,QAAQ;AAAA,IAC/B,CAAC;AACD,UAAM,sCAAsC,MAAM,UAAU,QAAQ,QAAQ;AAAA,EAC9E;AACF;AAEA,SAAS,qBAAqB,WAAmB,SAA+C;AAC9F,QAAM,gBAAgB,OAAO,QAAQ,SAAS,WAAW,QAAQ,OAAO;AACxE,QAAM,qBAAqB,OAAO,QAAQ,cAAc,WAAW,QAAQ,YAAY;AACvF,QAAM,gBAAgB,SAAS,QAAQ,IAAI,IAAI,QAAQ,OAAO;AAE9D,MAAI,iBAAiB,sBAAsB,eAAe;AACxD,WAAO;AAAA,MACL,MAAM;AAAA,MACN,WAAW;AAAA,MACX,MAAM;AAAA,IACR;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM;AAAA,IACN,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC,MAAM;AAAA,EACR;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;AAEA,SAAS,kBAAkB,QAAyB;AAClD,MAAI,UAAU,OAAO,SAAS,IAAK,QAAO;AAC1C,MAAI,WAAW,OAAO,WAAW,IAAK,QAAO;AAC7C,SAAO,UAAU;AACnB;AAEA,SAAS,mBAAmB,eAA6B;AACvD,QAAM,cAAc;AACpB,QAAM,WAAW,KAAK,MAAM,KAAK,OAAO,IAAI,GAAI;AAChD,QAAM,UAAU,cAAc,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAG,CAAC,CAAC,IAAI;AAC5E,SAAO,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO;AACtC;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -46,6 +46,9 @@ async function enqueueWebhookDelivery(job, delayMs) {
|
|
|
46
46
|
await ensureLocalWebhookQueueWorkerStarted();
|
|
47
47
|
return jobId;
|
|
48
48
|
}
|
|
49
|
+
async function enqueueWebhookDeliveryFromJob(job, delayMs) {
|
|
50
|
+
return getWebhookQueue().enqueue(job, delayMs > 0 ? { delayMs } : void 0);
|
|
51
|
+
}
|
|
49
52
|
function getInboundDispatchQueue(queueName = WEBHOOK_INBOUND_DISPATCH_QUEUE) {
|
|
50
53
|
const existing = inboundQueues.get(queueName);
|
|
51
54
|
if (existing) return existing;
|
|
@@ -92,6 +95,7 @@ export {
|
|
|
92
95
|
WEBHOOK_INBOUND_DISPATCH_QUEUE,
|
|
93
96
|
enqueueInboundDispatch,
|
|
94
97
|
enqueueWebhookDelivery,
|
|
98
|
+
enqueueWebhookDeliveryFromJob,
|
|
95
99
|
getInboundDispatchQueue,
|
|
96
100
|
getWebhookQueue
|
|
97
101
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/modules/webhooks/lib/queue.ts"],
|
|
4
|
-
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { createModuleQueue, type Queue } from '@open-mercato/queue'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { WebhookDeliveryJob } from './delivery'\nimport type { InboundDispatchJob } from './inbound-dispatch'\n\nconst logger = createLogger('webhooks')\n\nconst queues = new Map<string, Queue<WebhookDeliveryJob>>()\nconst inboundQueues = new Map<string, Queue<InboundDispatchJob>>()\nconst LOCAL_WORKER_PROMISE_KEY = '__openMercatoWebhookLocalWorkerPromise__'\nconst LOCAL_INBOUND_WORKER_PROMISE_KEY = '__openMercatoWebhookLocalInboundWorkerPromise__'\n\nexport const WEBHOOK_DELIVERIES_QUEUE = 'webhook-deliveries'\nexport const WEBHOOK_INBOUND_DISPATCH_QUEUE = 'webhook-inbound-dispatch'\n\nexport function getWebhookQueue(queueName: string = WEBHOOK_DELIVERIES_QUEUE): Queue<WebhookDeliveryJob> {\n const existing = queues.get(queueName)\n if (existing) return existing\n\n const concurrency = Math.max(1, Number.parseInt(process.env.WEBHOOK_QUEUE_CONCURRENCY ?? '10', 10) || 10)\n const created = createModuleQueue<WebhookDeliveryJob>(queueName, { concurrency })\n\n queues.set(queueName, created)\n return created\n}\n\nasync function ensureLocalWebhookQueueWorkerStarted(): Promise<void> {\n if (process.env.QUEUE_STRATEGY === 'async') return\n\n const globalStore = globalThis as typeof globalThis & {\n [LOCAL_WORKER_PROMISE_KEY]?: Promise<void>\n }\n\n if (globalStore[LOCAL_WORKER_PROMISE_KEY]) {\n await globalStore[LOCAL_WORKER_PROMISE_KEY]\n return\n }\n\n globalStore[LOCAL_WORKER_PROMISE_KEY] = (async () => {\n const queue = getWebhookQueue()\n\n await queue.process(async (job) => {\n const [{ createRequestContainer }, { processWebhookDeliveryJob }] = await Promise.all([\n import('@open-mercato/shared/lib/di/container'),\n import('./delivery'),\n ])\n\n const container = await createRequestContainer()\n const em = (container.resolve('em') as EntityManager).fork()\n await processWebhookDeliveryJob(em, job.payload, { resolver: container })\n })\n })().catch((error) => {\n delete globalStore[LOCAL_WORKER_PROMISE_KEY]\n logger.error('Failed to start local delivery worker', { err: error })\n throw error\n })\n\n await globalStore[LOCAL_WORKER_PROMISE_KEY]\n}\n\nexport async function enqueueWebhookDelivery(job: WebhookDeliveryJob, delayMs?: number): Promise<string> {\n const queue = getWebhookQueue()\n const jobId = await queue.enqueue(job, delayMs && delayMs > 0 ? { delayMs } : undefined)\n await ensureLocalWebhookQueueWorkerStarted()\n return jobId\n}\n\nexport function getInboundDispatchQueue(\n queueName: string = WEBHOOK_INBOUND_DISPATCH_QUEUE,\n): Queue<InboundDispatchJob> {\n const existing = inboundQueues.get(queueName)\n if (existing) return existing\n\n const concurrency = Math.max(1, Number.parseInt(process.env.WEBHOOK_INBOUND_QUEUE_CONCURRENCY ?? '5', 10) || 5)\n const created = createModuleQueue<InboundDispatchJob>(queueName, { concurrency })\n\n inboundQueues.set(queueName, created)\n return created\n}\n\nasync function ensureLocalInboundQueueWorkerStarted(): Promise<void> {\n if (process.env.QUEUE_STRATEGY === 'async') return\n\n const globalStore = globalThis as typeof globalThis & {\n [LOCAL_INBOUND_WORKER_PROMISE_KEY]?: Promise<void>\n }\n\n if (globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]) {\n await globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n return\n }\n\n globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY] = (async () => {\n const queue = getInboundDispatchQueue()\n\n await queue.process(async (job) => {\n const [{ createRequestContainer }, { processInboundDispatchJob }] = await Promise.all([\n import('@open-mercato/shared/lib/di/container'),\n import('./inbound-dispatch'),\n ])\n\n const container = await createRequestContainer()\n const em = (container.resolve('em') as EntityManager).fork()\n await processInboundDispatchJob(em, job.payload, {\n resolve: <T,>(name: string) => container.resolve(name) as T,\n })\n })\n })().catch((error) => {\n delete globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n logger.error('Failed to start local inbound dispatch worker', { err: error })\n throw error\n })\n\n await globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n}\n\nexport async function enqueueInboundDispatch(job: InboundDispatchJob): Promise<string> {\n const queue = getInboundDispatchQueue()\n const jobId = await queue.enqueue(job)\n await ensureLocalInboundQueueWorkerStarted()\n return jobId\n}\n"],
|
|
5
|
-
"mappings": "AACA,SAAS,yBAAqC;AAC9C,SAAS,oBAAoB;AAI7B,MAAM,SAAS,aAAa,UAAU;AAEtC,MAAM,SAAS,oBAAI,IAAuC;AAC1D,MAAM,gBAAgB,oBAAI,IAAuC;AACjE,MAAM,2BAA2B;AACjC,MAAM,mCAAmC;AAElC,MAAM,2BAA2B;AACjC,MAAM,iCAAiC;AAEvC,SAAS,gBAAgB,YAAoB,0BAAqD;AACvG,QAAM,WAAW,OAAO,IAAI,SAAS;AACrC,MAAI,SAAU,QAAO;AAErB,QAAM,cAAc,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ,IAAI,6BAA6B,MAAM,EAAE,KAAK,EAAE;AACxG,QAAM,UAAU,kBAAsC,WAAW,EAAE,YAAY,CAAC;AAEhF,SAAO,IAAI,WAAW,OAAO;AAC7B,SAAO;AACT;AAEA,eAAe,uCAAsD;AACnE,MAAI,QAAQ,IAAI,mBAAmB,QAAS;AAE5C,QAAM,cAAc;AAIpB,MAAI,YAAY,wBAAwB,GAAG;AACzC,UAAM,YAAY,wBAAwB;AAC1C;AAAA,EACF;AAEA,cAAY,wBAAwB,KAAK,YAAY;AACnD,UAAM,QAAQ,gBAAgB;AAE9B,UAAM,MAAM,QAAQ,OAAO,QAAQ;AACjC,YAAM,CAAC,EAAE,uBAAuB,GAAG,EAAE,0BAA0B,CAAC,IAAI,MAAM,QAAQ,IAAI;AAAA,QACpF,OAAO,uCAAuC;AAAA,QAC9C,OAAO,YAAY;AAAA,MACrB,CAAC;AAED,YAAM,YAAY,MAAM,uBAAuB;AAC/C,YAAM,KAAM,UAAU,QAAQ,IAAI,EAAoB,KAAK;AAC3D,YAAM,0BAA0B,IAAI,IAAI,SAAS,EAAE,UAAU,UAAU,CAAC;AAAA,IAC1E,CAAC;AAAA,EACH,GAAG,EAAE,MAAM,CAAC,UAAU;AACpB,WAAO,YAAY,wBAAwB;AAC3C,WAAO,MAAM,yCAAyC,EAAE,KAAK,MAAM,CAAC;AACpE,UAAM;AAAA,EACR,CAAC;AAED,QAAM,YAAY,wBAAwB;AAC5C;AAEA,eAAsB,uBAAuB,KAAyB,SAAmC;AACvG,QAAM,QAAQ,gBAAgB;AAC9B,QAAM,QAAQ,MAAM,MAAM,QAAQ,KAAK,WAAW,UAAU,IAAI,EAAE,QAAQ,IAAI,MAAS;AACvF,QAAM,qCAAqC;AAC3C,SAAO;AACT;AAEO,SAAS,wBACd,YAAoB,gCACO;AAC3B,QAAM,WAAW,cAAc,IAAI,SAAS;AAC5C,MAAI,SAAU,QAAO;AAErB,QAAM,cAAc,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ,IAAI,qCAAqC,KAAK,EAAE,KAAK,CAAC;AAC9G,QAAM,UAAU,kBAAsC,WAAW,EAAE,YAAY,CAAC;AAEhF,gBAAc,IAAI,WAAW,OAAO;AACpC,SAAO;AACT;AAEA,eAAe,uCAAsD;AACnE,MAAI,QAAQ,IAAI,mBAAmB,QAAS;AAE5C,QAAM,cAAc;AAIpB,MAAI,YAAY,gCAAgC,GAAG;AACjD,UAAM,YAAY,gCAAgC;AAClD;AAAA,EACF;AAEA,cAAY,gCAAgC,KAAK,YAAY;AAC3D,UAAM,QAAQ,wBAAwB;AAEtC,UAAM,MAAM,QAAQ,OAAO,QAAQ;AACjC,YAAM,CAAC,EAAE,uBAAuB,GAAG,EAAE,0BAA0B,CAAC,IAAI,MAAM,QAAQ,IAAI;AAAA,QACpF,OAAO,uCAAuC;AAAA,QAC9C,OAAO,oBAAoB;AAAA,MAC7B,CAAC;AAED,YAAM,YAAY,MAAM,uBAAuB;AAC/C,YAAM,KAAM,UAAU,QAAQ,IAAI,EAAoB,KAAK;AAC3D,YAAM,0BAA0B,IAAI,IAAI,SAAS;AAAA,QAC/C,SAAS,CAAK,SAAiB,UAAU,QAAQ,IAAI;AAAA,MACvD,CAAC;AAAA,IACH,CAAC;AAAA,EACH,GAAG,EAAE,MAAM,CAAC,UAAU;AACpB,WAAO,YAAY,gCAAgC;AACnD,WAAO,MAAM,iDAAiD,EAAE,KAAK,MAAM,CAAC;AAC5E,UAAM;AAAA,EACR,CAAC;AAED,QAAM,YAAY,gCAAgC;AACpD;AAEA,eAAsB,uBAAuB,KAA0C;AACrF,QAAM,QAAQ,wBAAwB;AACtC,QAAM,QAAQ,MAAM,MAAM,QAAQ,GAAG;AACrC,QAAM,qCAAqC;AAC3C,SAAO;AACT;",
|
|
4
|
+
"sourcesContent": ["import type { EntityManager } from '@mikro-orm/postgresql'\nimport { createModuleQueue, type Queue } from '@open-mercato/queue'\nimport { createLogger } from '@open-mercato/shared/lib/logger'\nimport type { WebhookDeliveryJob } from './delivery'\nimport type { InboundDispatchJob } from './inbound-dispatch'\n\nconst logger = createLogger('webhooks')\n\nconst queues = new Map<string, Queue<WebhookDeliveryJob>>()\nconst inboundQueues = new Map<string, Queue<InboundDispatchJob>>()\nconst LOCAL_WORKER_PROMISE_KEY = '__openMercatoWebhookLocalWorkerPromise__'\nconst LOCAL_INBOUND_WORKER_PROMISE_KEY = '__openMercatoWebhookLocalInboundWorkerPromise__'\n\nexport const WEBHOOK_DELIVERIES_QUEUE = 'webhook-deliveries'\nexport const WEBHOOK_INBOUND_DISPATCH_QUEUE = 'webhook-inbound-dispatch'\n\nexport function getWebhookQueue(queueName: string = WEBHOOK_DELIVERIES_QUEUE): Queue<WebhookDeliveryJob> {\n const existing = queues.get(queueName)\n if (existing) return existing\n\n const concurrency = Math.max(1, Number.parseInt(process.env.WEBHOOK_QUEUE_CONCURRENCY ?? '10', 10) || 10)\n const created = createModuleQueue<WebhookDeliveryJob>(queueName, { concurrency })\n\n queues.set(queueName, created)\n return created\n}\n\nasync function ensureLocalWebhookQueueWorkerStarted(): Promise<void> {\n if (process.env.QUEUE_STRATEGY === 'async') return\n\n const globalStore = globalThis as typeof globalThis & {\n [LOCAL_WORKER_PROMISE_KEY]?: Promise<void>\n }\n\n if (globalStore[LOCAL_WORKER_PROMISE_KEY]) {\n await globalStore[LOCAL_WORKER_PROMISE_KEY]\n return\n }\n\n globalStore[LOCAL_WORKER_PROMISE_KEY] = (async () => {\n const queue = getWebhookQueue()\n\n await queue.process(async (job) => {\n const [{ createRequestContainer }, { processWebhookDeliveryJob }] = await Promise.all([\n import('@open-mercato/shared/lib/di/container'),\n import('./delivery'),\n ])\n\n const container = await createRequestContainer()\n const em = (container.resolve('em') as EntityManager).fork()\n await processWebhookDeliveryJob(em, job.payload, { resolver: container })\n })\n })().catch((error) => {\n delete globalStore[LOCAL_WORKER_PROMISE_KEY]\n logger.error('Failed to start local delivery worker', { err: error })\n throw error\n })\n\n await globalStore[LOCAL_WORKER_PROMISE_KEY]\n}\n\nexport async function enqueueWebhookDelivery(job: WebhookDeliveryJob, delayMs?: number): Promise<string> {\n const queue = getWebhookQueue()\n const jobId = await queue.enqueue(job, delayMs && delayMs > 0 ? { delayMs } : undefined)\n await ensureLocalWebhookQueueWorkerStarted()\n return jobId\n}\n\nexport async function enqueueWebhookDeliveryFromJob(job: WebhookDeliveryJob, delayMs: number): Promise<string> {\n return getWebhookQueue().enqueue(job, delayMs > 0 ? { delayMs } : undefined)\n}\n\nexport function getInboundDispatchQueue(\n queueName: string = WEBHOOK_INBOUND_DISPATCH_QUEUE,\n): Queue<InboundDispatchJob> {\n const existing = inboundQueues.get(queueName)\n if (existing) return existing\n\n const concurrency = Math.max(1, Number.parseInt(process.env.WEBHOOK_INBOUND_QUEUE_CONCURRENCY ?? '5', 10) || 5)\n const created = createModuleQueue<InboundDispatchJob>(queueName, { concurrency })\n\n inboundQueues.set(queueName, created)\n return created\n}\n\nasync function ensureLocalInboundQueueWorkerStarted(): Promise<void> {\n if (process.env.QUEUE_STRATEGY === 'async') return\n\n const globalStore = globalThis as typeof globalThis & {\n [LOCAL_INBOUND_WORKER_PROMISE_KEY]?: Promise<void>\n }\n\n if (globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]) {\n await globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n return\n }\n\n globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY] = (async () => {\n const queue = getInboundDispatchQueue()\n\n await queue.process(async (job) => {\n const [{ createRequestContainer }, { processInboundDispatchJob }] = await Promise.all([\n import('@open-mercato/shared/lib/di/container'),\n import('./inbound-dispatch'),\n ])\n\n const container = await createRequestContainer()\n const em = (container.resolve('em') as EntityManager).fork()\n await processInboundDispatchJob(em, job.payload, {\n resolve: <T,>(name: string) => container.resolve(name) as T,\n })\n })\n })().catch((error) => {\n delete globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n logger.error('Failed to start local inbound dispatch worker', { err: error })\n throw error\n })\n\n await globalStore[LOCAL_INBOUND_WORKER_PROMISE_KEY]\n}\n\nexport async function enqueueInboundDispatch(job: InboundDispatchJob): Promise<string> {\n const queue = getInboundDispatchQueue()\n const jobId = await queue.enqueue(job)\n await ensureLocalInboundQueueWorkerStarted()\n return jobId\n}\n"],
|
|
5
|
+
"mappings": "AACA,SAAS,yBAAqC;AAC9C,SAAS,oBAAoB;AAI7B,MAAM,SAAS,aAAa,UAAU;AAEtC,MAAM,SAAS,oBAAI,IAAuC;AAC1D,MAAM,gBAAgB,oBAAI,IAAuC;AACjE,MAAM,2BAA2B;AACjC,MAAM,mCAAmC;AAElC,MAAM,2BAA2B;AACjC,MAAM,iCAAiC;AAEvC,SAAS,gBAAgB,YAAoB,0BAAqD;AACvG,QAAM,WAAW,OAAO,IAAI,SAAS;AACrC,MAAI,SAAU,QAAO;AAErB,QAAM,cAAc,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ,IAAI,6BAA6B,MAAM,EAAE,KAAK,EAAE;AACxG,QAAM,UAAU,kBAAsC,WAAW,EAAE,YAAY,CAAC;AAEhF,SAAO,IAAI,WAAW,OAAO;AAC7B,SAAO;AACT;AAEA,eAAe,uCAAsD;AACnE,MAAI,QAAQ,IAAI,mBAAmB,QAAS;AAE5C,QAAM,cAAc;AAIpB,MAAI,YAAY,wBAAwB,GAAG;AACzC,UAAM,YAAY,wBAAwB;AAC1C;AAAA,EACF;AAEA,cAAY,wBAAwB,KAAK,YAAY;AACnD,UAAM,QAAQ,gBAAgB;AAE9B,UAAM,MAAM,QAAQ,OAAO,QAAQ;AACjC,YAAM,CAAC,EAAE,uBAAuB,GAAG,EAAE,0BAA0B,CAAC,IAAI,MAAM,QAAQ,IAAI;AAAA,QACpF,OAAO,uCAAuC;AAAA,QAC9C,OAAO,YAAY;AAAA,MACrB,CAAC;AAED,YAAM,YAAY,MAAM,uBAAuB;AAC/C,YAAM,KAAM,UAAU,QAAQ,IAAI,EAAoB,KAAK;AAC3D,YAAM,0BAA0B,IAAI,IAAI,SAAS,EAAE,UAAU,UAAU,CAAC;AAAA,IAC1E,CAAC;AAAA,EACH,GAAG,EAAE,MAAM,CAAC,UAAU;AACpB,WAAO,YAAY,wBAAwB;AAC3C,WAAO,MAAM,yCAAyC,EAAE,KAAK,MAAM,CAAC;AACpE,UAAM;AAAA,EACR,CAAC;AAED,QAAM,YAAY,wBAAwB;AAC5C;AAEA,eAAsB,uBAAuB,KAAyB,SAAmC;AACvG,QAAM,QAAQ,gBAAgB;AAC9B,QAAM,QAAQ,MAAM,MAAM,QAAQ,KAAK,WAAW,UAAU,IAAI,EAAE,QAAQ,IAAI,MAAS;AACvF,QAAM,qCAAqC;AAC3C,SAAO;AACT;AAEA,eAAsB,8BAA8B,KAAyB,SAAkC;AAC7G,SAAO,gBAAgB,EAAE,QAAQ,KAAK,UAAU,IAAI,EAAE,QAAQ,IAAI,MAAS;AAC7E;AAEO,SAAS,wBACd,YAAoB,gCACO;AAC3B,QAAM,WAAW,cAAc,IAAI,SAAS;AAC5C,MAAI,SAAU,QAAO;AAErB,QAAM,cAAc,KAAK,IAAI,GAAG,OAAO,SAAS,QAAQ,IAAI,qCAAqC,KAAK,EAAE,KAAK,CAAC;AAC9G,QAAM,UAAU,kBAAsC,WAAW,EAAE,YAAY,CAAC;AAEhF,gBAAc,IAAI,WAAW,OAAO;AACpC,SAAO;AACT;AAEA,eAAe,uCAAsD;AACnE,MAAI,QAAQ,IAAI,mBAAmB,QAAS;AAE5C,QAAM,cAAc;AAIpB,MAAI,YAAY,gCAAgC,GAAG;AACjD,UAAM,YAAY,gCAAgC;AAClD;AAAA,EACF;AAEA,cAAY,gCAAgC,KAAK,YAAY;AAC3D,UAAM,QAAQ,wBAAwB;AAEtC,UAAM,MAAM,QAAQ,OAAO,QAAQ;AACjC,YAAM,CAAC,EAAE,uBAAuB,GAAG,EAAE,0BAA0B,CAAC,IAAI,MAAM,QAAQ,IAAI;AAAA,QACpF,OAAO,uCAAuC;AAAA,QAC9C,OAAO,oBAAoB;AAAA,MAC7B,CAAC;AAED,YAAM,YAAY,MAAM,uBAAuB;AAC/C,YAAM,KAAM,UAAU,QAAQ,IAAI,EAAoB,KAAK;AAC3D,YAAM,0BAA0B,IAAI,IAAI,SAAS;AAAA,QAC/C,SAAS,CAAK,SAAiB,UAAU,QAAQ,IAAI;AAAA,MACvD,CAAC;AAAA,IACH,CAAC;AAAA,EACH,GAAG,EAAE,MAAM,CAAC,UAAU;AACpB,WAAO,YAAY,gCAAgC;AACnD,WAAO,MAAM,iDAAiD,EAAE,KAAK,MAAM,CAAC;AAC5E,UAAM;AAAA,EACR,CAAC;AAED,QAAM,YAAY,gCAAgC;AACpD;AAEA,eAAsB,uBAAuB,KAA0C;AACrF,QAAM,QAAQ,wBAAwB;AACtC,QAAM,QAAQ,MAAM,MAAM,QAAQ,GAAG;AACrC,QAAM,qCAAqC;AAC3C,SAAO;AACT;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@open-mercato/webhooks",
|
|
3
|
-
"version": "0.10.1-develop.
|
|
3
|
+
"version": "0.10.1-develop.8098.1.f4decc8a57",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Webhooks module for Open Mercato — Standard Webhooks compliant outbound/inbound delivery",
|
|
6
6
|
"type": "module",
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
"scripts": {
|
|
9
9
|
"build": "node build.mjs",
|
|
10
10
|
"test": "jest --config jest.config.cjs",
|
|
11
|
+
"test:pg-integration": "cross-env OM_PG_INTEGRATION=1 jest --config jest.config.cjs --runInBand 'src/modules/webhooks/lib/__tests__/delivery-claim\\.integration\\.test\\.ts$'",
|
|
11
12
|
"typecheck": "tsc --noEmit"
|
|
12
13
|
},
|
|
13
14
|
"exports": {
|
|
@@ -70,32 +71,34 @@
|
|
|
70
71
|
}
|
|
71
72
|
},
|
|
72
73
|
"dependencies": {
|
|
73
|
-
"@open-mercato/cache": "0.10.1-develop.
|
|
74
|
-
"@open-mercato/core": "0.10.1-develop.
|
|
75
|
-
"@open-mercato/queue": "0.10.1-develop.
|
|
76
|
-
"@open-mercato/ui": "0.10.1-develop.
|
|
74
|
+
"@open-mercato/cache": "0.10.1-develop.8098.1.f4decc8a57",
|
|
75
|
+
"@open-mercato/core": "0.10.1-develop.8098.1.f4decc8a57",
|
|
76
|
+
"@open-mercato/queue": "0.10.1-develop.8098.1.f4decc8a57",
|
|
77
|
+
"@open-mercato/ui": "0.10.1-develop.8098.1.f4decc8a57",
|
|
77
78
|
"svix": "^1.99.1"
|
|
78
79
|
},
|
|
79
80
|
"peerDependencies": {
|
|
80
81
|
"@mikro-orm/postgresql": "^7.0.14",
|
|
81
|
-
"@open-mercato/ai-assistant": "0.10.1-develop.
|
|
82
|
-
"@open-mercato/shared": "0.10.1-develop.
|
|
82
|
+
"@open-mercato/ai-assistant": "0.10.1-develop.8098.1.f4decc8a57",
|
|
83
|
+
"@open-mercato/shared": "0.10.1-develop.8098.1.f4decc8a57",
|
|
83
84
|
"react": "^19.0.0",
|
|
84
85
|
"react-dom": "^19.0.0",
|
|
85
86
|
"react-is": ">=18.0.0"
|
|
86
87
|
},
|
|
87
88
|
"devDependencies": {
|
|
88
|
-
"@open-mercato/ai-assistant": "0.10.1-develop.
|
|
89
|
-
"@open-mercato/shared": "0.10.1-develop.
|
|
89
|
+
"@open-mercato/ai-assistant": "0.10.1-develop.8098.1.f4decc8a57",
|
|
90
|
+
"@open-mercato/shared": "0.10.1-develop.8098.1.f4decc8a57",
|
|
90
91
|
"@types/jest": "^30.0.0",
|
|
91
92
|
"@types/react": "^19.2.18",
|
|
92
93
|
"@types/react-dom": "^19.2.5",
|
|
94
|
+
"cross-env": "^10.1.0",
|
|
93
95
|
"esbuild": "^0.28.2",
|
|
94
96
|
"glob": "^14.0.3",
|
|
95
97
|
"jest": "^30.4.2",
|
|
96
98
|
"react": "19.2.8",
|
|
97
99
|
"react-dom": "19.2.8",
|
|
98
100
|
"react-is": "^19.2.8",
|
|
101
|
+
"testcontainers": "^12.0.3",
|
|
99
102
|
"ts-jest": "^29.4.12",
|
|
100
103
|
"typescript": "7.0.2"
|
|
101
104
|
},
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
|
|
2
|
+
import type { AddressInfo } from 'node:net';
|
|
3
|
+
import { expect, test } from '@playwright/test';
|
|
4
|
+
import { apiRequest, getAuthToken } from '@open-mercato/core/helpers/integration/api';
|
|
5
|
+
import { readJsonSafe } from '@open-mercato/core/helpers/integration/generalFixtures';
|
|
6
|
+
import { drainIntegrationQueue } from '@open-mercato/core/helpers/integration/queue';
|
|
7
|
+
import { readIntegrationEnvFlag } from '@open-mercato/core/helpers/integration/standaloneEnv';
|
|
8
|
+
import {
|
|
9
|
+
createWebhookFixture,
|
|
10
|
+
deleteWebhookIfExists,
|
|
11
|
+
getDeliveryDetail,
|
|
12
|
+
} from './helpers/fixtures';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* TC-WEBHOOK-015: a retried webhook delivery is sent once even when its job runs more than once
|
|
16
|
+
* Source: open-mercato/open-mercato#ISSUE
|
|
17
|
+
*
|
|
18
|
+
* The delivery worker claims a delivery (`pending` → `sending`) atomically before sending and ignores
|
|
19
|
+
* jobs for deliveries that already finished, so a queue that runs a job twice cannot send the event
|
|
20
|
+
* twice. In the integration env the app process starts an in-process local-queue worker on enqueue
|
|
21
|
+
* while AUTO_SPAWN_WORKERS also runs the webhook worker, so a retried job can run twice.
|
|
22
|
+
* Each delivery here fails its synchronous test send with a non-retryable 400, is retried once, and
|
|
23
|
+
* must reach the receiver exactly once more. Needs OM_WEBHOOKS_ALLOW_PRIVATE_URLS (enabled in the integration env)
|
|
24
|
+
* because the receiver listens on loopback.
|
|
25
|
+
*/
|
|
26
|
+
const PRIVATE_URLS_ALLOWED = readIntegrationEnvFlag('OM_WEBHOOKS_ALLOW_PRIVATE_URLS');
|
|
27
|
+
const DELIVERY_COUNT = 3;
|
|
28
|
+
|
|
29
|
+
type Receiver = {
|
|
30
|
+
server: Server;
|
|
31
|
+
baseUrl: string;
|
|
32
|
+
statuses: (path: string) => number[];
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
async function startReceiver(): Promise<Receiver> {
|
|
36
|
+
const sentByPath = new Map<string, number[]>();
|
|
37
|
+
const server = createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
38
|
+
req.resume();
|
|
39
|
+
req.on('end', () => {
|
|
40
|
+
const path = req.url ?? '';
|
|
41
|
+
const sent = sentByPath.get(path) ?? [];
|
|
42
|
+
const status = sent.length === 0 ? 400 : 200;
|
|
43
|
+
sentByPath.set(path, [...sent, status]);
|
|
44
|
+
res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify({ ok: status === 200 }));
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
|
48
|
+
const { port } = server.address() as AddressInfo;
|
|
49
|
+
return {
|
|
50
|
+
server,
|
|
51
|
+
baseUrl: `http://127.0.0.1:${port}`,
|
|
52
|
+
statuses: (path: string) => [...(sentByPath.get(path) ?? [])],
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
test.describe('TC-WEBHOOK-015: delivery jobs do not send a delivery twice', () => {
|
|
57
|
+
test('each retried delivery reaches the receiver exactly once more', async ({ request }) => {
|
|
58
|
+
test.skip(!PRIVATE_URLS_ALLOWED, 'The loopback receiver needs OM_WEBHOOKS_ALLOW_PRIVATE_URLS');
|
|
59
|
+
test.setTimeout(120_000);
|
|
60
|
+
const token = await getAuthToken(request);
|
|
61
|
+
const receiver = await startReceiver();
|
|
62
|
+
const webhookIds: string[] = [];
|
|
63
|
+
|
|
64
|
+
try {
|
|
65
|
+
const deliveries: Array<{ path: string; deliveryId: string }> = [];
|
|
66
|
+
for (let index = 0; index < DELIVERY_COUNT; index += 1) {
|
|
67
|
+
const path = `/tc-webhook-015-${index}`;
|
|
68
|
+
const created = await createWebhookFixture(request, token, {
|
|
69
|
+
name: `Webhook delivery claim ${index} ${Date.now()}`,
|
|
70
|
+
url: `${receiver.baseUrl}${path}`,
|
|
71
|
+
subscribedEvents: ['catalog.product.updated'],
|
|
72
|
+
});
|
|
73
|
+
webhookIds.push(created.id);
|
|
74
|
+
const testResponse = await apiRequest(request, 'POST', `/api/webhooks/${created.id}/test`, { token, data: {} });
|
|
75
|
+
expect(testResponse.status()).toBe(200);
|
|
76
|
+
const testBody = await readJsonSafe<{ delivery: { id: string; status: string } }>(testResponse);
|
|
77
|
+
const deliveryId = testBody?.delivery.id;
|
|
78
|
+
expect(testBody?.delivery.status).toBe('expired');
|
|
79
|
+
expect(deliveryId).toEqual(expect.any(String));
|
|
80
|
+
deliveries.push({ path, deliveryId: String(deliveryId) });
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const retries = await Promise.all(deliveries.map(({ deliveryId }) =>
|
|
84
|
+
apiRequest(request, 'POST', `/api/webhooks/deliveries/${deliveryId}/retry`, { token })));
|
|
85
|
+
for (const retry of retries) expect(retry.status()).toBe(200);
|
|
86
|
+
|
|
87
|
+
for (const { deliveryId } of deliveries) {
|
|
88
|
+
await expect.poll(async () => (await getDeliveryDetail(request, token, deliveryId)).status, { timeout: 30_000 }).toBe('delivered');
|
|
89
|
+
}
|
|
90
|
+
await drainIntegrationQueue('webhook-deliveries');
|
|
91
|
+
|
|
92
|
+
for (const { path, deliveryId } of deliveries) {
|
|
93
|
+
expect(receiver.statuses(path)).toEqual([400, 200]);
|
|
94
|
+
expect((await getDeliveryDetail(request, token, deliveryId)).attemptNumber).toBe(2);
|
|
95
|
+
}
|
|
96
|
+
} finally {
|
|
97
|
+
for (const webhookId of webhookIds) await deleteWebhookIfExists(request, token, webhookId);
|
|
98
|
+
await new Promise<void>((resolve) => receiver.server.close(() => resolve()));
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
});
|
|
@@ -0,0 +1,291 @@
|
|
|
1
|
+
/** @jest-environment node */
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
3
|
+
import { MikroORM } from '@mikro-orm/core'
|
|
4
|
+
import { PostgreSqlDriver, type EntityManager } from '@mikro-orm/postgresql'
|
|
5
|
+
import type { StartedTestContainer } from 'testcontainers'
|
|
6
|
+
import { WebhookDeliveryEntity, WebhookEntity } from '../../data/entities'
|
|
7
|
+
import { processWebhookDeliveryJob } from '../delivery'
|
|
8
|
+
|
|
9
|
+
jest.mock('../../events', () => ({
|
|
10
|
+
emitWebhooksEvent: jest.fn(async () => undefined),
|
|
11
|
+
}))
|
|
12
|
+
|
|
13
|
+
jest.mock('@open-mercato/shared/lib/encryption/find', () => ({
|
|
14
|
+
findOneWithDecryption: jest.fn(),
|
|
15
|
+
}))
|
|
16
|
+
|
|
17
|
+
jest.mock('../queue', () => ({
|
|
18
|
+
enqueueWebhookDelivery: jest.fn(async () => 'job-1'),
|
|
19
|
+
enqueueWebhookDeliveryFromJob: jest.fn(async () => 'job-2'),
|
|
20
|
+
}))
|
|
21
|
+
|
|
22
|
+
jest.mock('../integration-state', () => ({
|
|
23
|
+
isWebhookIntegrationEnabled: jest.fn(async () => true),
|
|
24
|
+
WEBHOOK_INTEGRATION_DISABLED_MESSAGE: 'disabled',
|
|
25
|
+
}))
|
|
26
|
+
|
|
27
|
+
jest.mock('../url-safety', () => {
|
|
28
|
+
class UnsafeWebhookUrlError extends Error {}
|
|
29
|
+
return {
|
|
30
|
+
assertSafeWebhookDeliveryUrl: jest.fn(async () => undefined),
|
|
31
|
+
safeWebhookFetch: jest.fn(),
|
|
32
|
+
UnsafeWebhookUrlError,
|
|
33
|
+
}
|
|
34
|
+
})
|
|
35
|
+
|
|
36
|
+
import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
|
|
37
|
+
import { enqueueWebhookDeliveryFromJob } from '../queue'
|
|
38
|
+
import { safeWebhookFetch } from '../url-safety'
|
|
39
|
+
|
|
40
|
+
const findOneWithDecryptionMock = findOneWithDecryption as jest.MockedFunction<typeof findOneWithDecryption>
|
|
41
|
+
const enqueueFromJobMock = enqueueWebhookDeliveryFromJob as jest.MockedFunction<typeof enqueueWebhookDeliveryFromJob>
|
|
42
|
+
const safeWebhookFetchMock = safeWebhookFetch as jest.MockedFunction<typeof safeWebhookFetch>
|
|
43
|
+
|
|
44
|
+
const describeWithPostgres = process.env.OM_PG_INTEGRATION === '1' ? describe : describe.skip
|
|
45
|
+
|
|
46
|
+
const DATABASE_NAME = 'webhooks_delivery_claim_test'
|
|
47
|
+
const DATABASE_USER = 'webhooks_delivery_claim'
|
|
48
|
+
const DATABASE_PASSWORD = 'webhooks_delivery_claim'
|
|
49
|
+
const CONCURRENT_WORKERS = 8
|
|
50
|
+
const SEND_DURATION_MS = 300
|
|
51
|
+
const WEBHOOK_TIMEOUT_MS = 5_000
|
|
52
|
+
|
|
53
|
+
type TestOrm = MikroORM<PostgreSqlDriver, EntityManager<PostgreSqlDriver>>
|
|
54
|
+
|
|
55
|
+
type Seeded = { webhookId: string; deliveryId: string; tenantId: string; organizationId: string }
|
|
56
|
+
|
|
57
|
+
function sleep(delayMs: number): Promise<void> {
|
|
58
|
+
return new Promise((resolve) => setTimeout(resolve, delayMs))
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function createBarrier(parties: number): () => Promise<void> {
|
|
62
|
+
let arrived = 0
|
|
63
|
+
let open: () => void = () => undefined
|
|
64
|
+
const opened = new Promise<void>((resolve) => {
|
|
65
|
+
open = resolve
|
|
66
|
+
})
|
|
67
|
+
return async () => {
|
|
68
|
+
arrived += 1
|
|
69
|
+
if (arrived >= parties) open()
|
|
70
|
+
await opened
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
type FindOneArgs = [EntityManager, typeof WebhookEntity, Record<string, unknown>]
|
|
75
|
+
|
|
76
|
+
describeWithPostgres('processWebhookDeliveryJob claim on real Postgres', () => {
|
|
77
|
+
let postgres: StartedTestContainer | null = null
|
|
78
|
+
let orm: TestOrm | null = null
|
|
79
|
+
|
|
80
|
+
beforeAll(async () => {
|
|
81
|
+
const { GenericContainer, Wait } = await import('testcontainers')
|
|
82
|
+
postgres = await new GenericContainer('postgres:16')
|
|
83
|
+
.withEnvironment({
|
|
84
|
+
POSTGRES_DB: DATABASE_NAME,
|
|
85
|
+
POSTGRES_USER: DATABASE_USER,
|
|
86
|
+
POSTGRES_PASSWORD: DATABASE_PASSWORD,
|
|
87
|
+
})
|
|
88
|
+
.withExposedPorts(5432)
|
|
89
|
+
.withWaitStrategy(Wait.forLogMessage(/database system is ready to accept connections/, 2))
|
|
90
|
+
.start()
|
|
91
|
+
orm = await MikroORM.init<PostgreSqlDriver, EntityManager<PostgreSqlDriver>>({
|
|
92
|
+
driver: PostgreSqlDriver,
|
|
93
|
+
host: postgres.getHost(),
|
|
94
|
+
port: postgres.getMappedPort(5432),
|
|
95
|
+
dbName: DATABASE_NAME,
|
|
96
|
+
user: DATABASE_USER,
|
|
97
|
+
password: DATABASE_PASSWORD,
|
|
98
|
+
entities: [WebhookEntity, WebhookDeliveryEntity],
|
|
99
|
+
debug: false,
|
|
100
|
+
pool: { min: 0, max: CONCURRENT_WORKERS + 2 },
|
|
101
|
+
})
|
|
102
|
+
await orm.schema.create()
|
|
103
|
+
}, 120_000)
|
|
104
|
+
|
|
105
|
+
afterAll(async () => {
|
|
106
|
+
await Promise.allSettled([orm?.close(true)])
|
|
107
|
+
await Promise.allSettled([postgres?.stop()])
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
beforeEach(() => {
|
|
111
|
+
jest.clearAllMocks()
|
|
112
|
+
findOneWithDecryptionMock.mockImplementation(((...args: FindOneArgs) => args[0].findOne(args[1], args[2])) as never)
|
|
113
|
+
safeWebhookFetchMock.mockImplementation(async () => {
|
|
114
|
+
await sleep(SEND_DURATION_MS)
|
|
115
|
+
return new Response('ok', { status: 200 })
|
|
116
|
+
})
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
function requireOrm(): TestOrm {
|
|
120
|
+
if (!orm) throw new Error('[internal] ORM not started')
|
|
121
|
+
return orm
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
async function seed(delivery: Partial<WebhookDeliveryEntity> = {}, webhook: Partial<WebhookEntity> = {}): Promise<Seeded> {
|
|
125
|
+
const em = requireOrm().em.fork()
|
|
126
|
+
const tenantId = randomUUID()
|
|
127
|
+
const organizationId = randomUUID()
|
|
128
|
+
const createdWebhook = em.create(WebhookEntity, {
|
|
129
|
+
name: 'Claim test',
|
|
130
|
+
url: 'https://example.com/hook',
|
|
131
|
+
secret: 'whsec_dGVzdC1zZWNyZXQtZm9yLWNsYWltLXRlc3Q=',
|
|
132
|
+
subscribedEvents: ['customers.person.created'],
|
|
133
|
+
httpMethod: 'POST',
|
|
134
|
+
maxRetries: 3,
|
|
135
|
+
timeoutMs: WEBHOOK_TIMEOUT_MS,
|
|
136
|
+
isActive: true,
|
|
137
|
+
organizationId,
|
|
138
|
+
tenantId,
|
|
139
|
+
...webhook,
|
|
140
|
+
})
|
|
141
|
+
await em.flush()
|
|
142
|
+
const now = new Date()
|
|
143
|
+
const createdDelivery = em.create(WebhookDeliveryEntity, {
|
|
144
|
+
webhookId: createdWebhook.id,
|
|
145
|
+
eventType: 'customers.person.created',
|
|
146
|
+
messageId: `msg_${randomUUID()}`,
|
|
147
|
+
payload: { type: 'customers.person.created', timestamp: now.toISOString(), data: { id: 'person-1' } },
|
|
148
|
+
status: 'pending',
|
|
149
|
+
attemptNumber: 0,
|
|
150
|
+
maxAttempts: 3,
|
|
151
|
+
targetUrl: createdWebhook.url,
|
|
152
|
+
organizationId,
|
|
153
|
+
tenantId,
|
|
154
|
+
enqueuedAt: now,
|
|
155
|
+
createdAt: now,
|
|
156
|
+
updatedAt: now,
|
|
157
|
+
...delivery,
|
|
158
|
+
})
|
|
159
|
+
await em.flush()
|
|
160
|
+
return { webhookId: createdWebhook.id, deliveryId: createdDelivery.id, tenantId, organizationId }
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
async function readDelivery(seeded: Seeded): Promise<WebhookDeliveryEntity> {
|
|
164
|
+
const row = await requireOrm().em.fork().findOne(WebhookDeliveryEntity, { id: seeded.deliveryId })
|
|
165
|
+
if (!row) throw new Error('[internal] delivery row missing')
|
|
166
|
+
return row
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function runJob(seeded: Seeded, scheduleRetries = false) {
|
|
170
|
+
return processWebhookDeliveryJob(
|
|
171
|
+
requireOrm().em.fork(),
|
|
172
|
+
{ deliveryId: seeded.deliveryId, tenantId: seeded.tenantId, organizationId: seeded.organizationId },
|
|
173
|
+
{ scheduleRetries },
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function holdWorkersBeforeClaim(parties: number): void {
|
|
178
|
+
const arrive = createBarrier(parties)
|
|
179
|
+
findOneWithDecryptionMock.mockImplementation((async (...args: FindOneArgs) => {
|
|
180
|
+
await arrive()
|
|
181
|
+
return args[0].findOne(args[1], args[2])
|
|
182
|
+
}) as never)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
it('sends a delivery once when several workers process the same job concurrently', async () => {
|
|
186
|
+
const seeded = await seed()
|
|
187
|
+
holdWorkersBeforeClaim(CONCURRENT_WORKERS)
|
|
188
|
+
|
|
189
|
+
await Promise.all(Array.from({ length: CONCURRENT_WORKERS }, () => runJob(seeded)))
|
|
190
|
+
|
|
191
|
+
expect(safeWebhookFetchMock).toHaveBeenCalledTimes(1)
|
|
192
|
+
const row = await readDelivery(seeded)
|
|
193
|
+
expect(row.status).toBe('delivered')
|
|
194
|
+
expect(row.attemptNumber).toBe(1)
|
|
195
|
+
}, 30_000)
|
|
196
|
+
|
|
197
|
+
it('does not re-send or rewrite a delivery that already finished', async () => {
|
|
198
|
+
const deliveredAt = new Date()
|
|
199
|
+
const delivered = await seed({ status: 'delivered', attemptNumber: 1, deliveredAt })
|
|
200
|
+
const deactivated = await seed({ status: 'delivered', attemptNumber: 1, deliveredAt }, { isActive: false })
|
|
201
|
+
|
|
202
|
+
await expect(runJob(delivered)).resolves.toBeNull()
|
|
203
|
+
await expect(runJob(deactivated)).resolves.toBeNull()
|
|
204
|
+
|
|
205
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
206
|
+
expect((await readDelivery(delivered)).status).toBe('delivered')
|
|
207
|
+
const deactivatedRow = await readDelivery(deactivated)
|
|
208
|
+
expect(deactivatedRow.status).toBe('delivered')
|
|
209
|
+
expect(deactivatedRow.errorMessage ?? null).toBeNull()
|
|
210
|
+
}, 30_000)
|
|
211
|
+
|
|
212
|
+
it('leaves a delivery another worker is still sending alone and rechecks it after the lease', async () => {
|
|
213
|
+
const seeded = await seed({ status: 'sending', lastAttemptAt: new Date() }, { isActive: false })
|
|
214
|
+
|
|
215
|
+
await expect(runJob(seeded, true)).resolves.toBeNull()
|
|
216
|
+
|
|
217
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
218
|
+
const row = await readDelivery(seeded)
|
|
219
|
+
expect(row.status).toBe('sending')
|
|
220
|
+
expect(row.attemptNumber).toBe(0)
|
|
221
|
+
expect(row.errorMessage ?? null).toBeNull()
|
|
222
|
+
expect(enqueueFromJobMock).toHaveBeenCalledWith(
|
|
223
|
+
{ deliveryId: seeded.deliveryId, tenantId: seeded.tenantId, organizationId: seeded.organizationId },
|
|
224
|
+
WEBHOOK_TIMEOUT_MS + 16_000,
|
|
225
|
+
)
|
|
226
|
+
}, 30_000)
|
|
227
|
+
|
|
228
|
+
it('returns a claimed delivery to pending when processing throws, so a queue retry can send it', async () => {
|
|
229
|
+
const seeded = await seed()
|
|
230
|
+
const em = requireOrm().em.fork()
|
|
231
|
+
jest.spyOn(em, 'flush').mockRejectedValueOnce(new Error('connection reset'))
|
|
232
|
+
const job = { deliveryId: seeded.deliveryId, tenantId: seeded.tenantId, organizationId: seeded.organizationId }
|
|
233
|
+
|
|
234
|
+
await expect(processWebhookDeliveryJob(em, job, { scheduleRetries: false })).rejects.toThrow('connection reset')
|
|
235
|
+
expect((await readDelivery(seeded)).status).toBe('pending')
|
|
236
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
237
|
+
|
|
238
|
+
await runJob(seeded)
|
|
239
|
+
|
|
240
|
+
expect(safeWebhookFetchMock).toHaveBeenCalledTimes(1)
|
|
241
|
+
const row = await readDelivery(seeded)
|
|
242
|
+
expect(row.status).toBe('delivered')
|
|
243
|
+
expect(row.attemptNumber).toBe(1)
|
|
244
|
+
}, 30_000)
|
|
245
|
+
|
|
246
|
+
it('stores a failed attempt as pending for its retry', async () => {
|
|
247
|
+
const seeded = await seed()
|
|
248
|
+
safeWebhookFetchMock.mockResolvedValueOnce(new Response('down', { status: 503 }))
|
|
249
|
+
|
|
250
|
+
await runJob(seeded)
|
|
251
|
+
|
|
252
|
+
const row = await readDelivery(seeded)
|
|
253
|
+
expect(row.status).toBe('pending')
|
|
254
|
+
expect(row.attemptNumber).toBe(1)
|
|
255
|
+
expect(row.nextRetryAt).toBeInstanceOf(Date)
|
|
256
|
+
}, 30_000)
|
|
257
|
+
|
|
258
|
+
it('does not send a scheduled retry before it is due and rechecks it then', async () => {
|
|
259
|
+
const nextRetryAt = new Date(Date.now() + 120_000)
|
|
260
|
+
const scheduled = await seed({ attemptNumber: 1, nextRetryAt })
|
|
261
|
+
const due = await seed({ attemptNumber: 1, nextRetryAt: new Date(Date.now() - 1_000) })
|
|
262
|
+
|
|
263
|
+
await expect(runJob(scheduled, true)).resolves.toBeNull()
|
|
264
|
+
|
|
265
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
266
|
+
const scheduledRow = await readDelivery(scheduled)
|
|
267
|
+
expect(scheduledRow.status).toBe('pending')
|
|
268
|
+
expect(scheduledRow.attemptNumber).toBe(1)
|
|
269
|
+
expect(scheduledRow.nextRetryAt?.getTime()).toBe(nextRetryAt.getTime())
|
|
270
|
+
expect(enqueueFromJobMock).toHaveBeenCalledTimes(1)
|
|
271
|
+
const [, recheckDelayMs] = enqueueFromJobMock.mock.calls[0]
|
|
272
|
+
expect(recheckDelayMs).toBeGreaterThan(110_000)
|
|
273
|
+
expect(recheckDelayMs).toBeLessThanOrEqual(121_000)
|
|
274
|
+
|
|
275
|
+
await expect(runJob(due)).resolves.toEqual({ status: 'delivered', deliveryId: due.deliveryId })
|
|
276
|
+
expect(safeWebhookFetchMock).toHaveBeenCalledTimes(1)
|
|
277
|
+
expect((await readDelivery(due)).attemptNumber).toBe(2)
|
|
278
|
+
}, 30_000)
|
|
279
|
+
|
|
280
|
+
it('reclaims a delivery whose sending worker stopped past its timeout', async () => {
|
|
281
|
+
const seeded = await seed({ status: 'sending', lastAttemptAt: new Date(Date.now() - WEBHOOK_TIMEOUT_MS - 60_000) })
|
|
282
|
+
holdWorkersBeforeClaim(CONCURRENT_WORKERS)
|
|
283
|
+
|
|
284
|
+
await Promise.all(Array.from({ length: CONCURRENT_WORKERS }, () => runJob(seeded)))
|
|
285
|
+
|
|
286
|
+
expect(safeWebhookFetchMock).toHaveBeenCalledTimes(1)
|
|
287
|
+
const row = await readDelivery(seeded)
|
|
288
|
+
expect(row.status).toBe('delivered')
|
|
289
|
+
expect(row.attemptNumber).toBe(1)
|
|
290
|
+
}, 30_000)
|
|
291
|
+
})
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
2
|
+
import { processWebhookDeliveryJob } from '../delivery'
|
|
3
|
+
|
|
4
|
+
jest.mock('../../events', () => ({
|
|
5
|
+
emitWebhooksEvent: jest.fn(async () => undefined),
|
|
6
|
+
}))
|
|
7
|
+
|
|
8
|
+
jest.mock('@open-mercato/shared/lib/encryption/find', () => ({
|
|
9
|
+
findOneWithDecryption: jest.fn(),
|
|
10
|
+
}))
|
|
11
|
+
|
|
12
|
+
jest.mock('../integration-state', () => ({
|
|
13
|
+
isWebhookIntegrationEnabled: jest.fn(async () => true),
|
|
14
|
+
WEBHOOK_INTEGRATION_DISABLED_MESSAGE: 'disabled',
|
|
15
|
+
}))
|
|
16
|
+
|
|
17
|
+
jest.mock('../queue', () => ({
|
|
18
|
+
enqueueWebhookDelivery: jest.fn(async () => 'job-1'),
|
|
19
|
+
enqueueWebhookDeliveryFromJob: jest.fn(async () => 'job-2'),
|
|
20
|
+
}))
|
|
21
|
+
|
|
22
|
+
jest.mock('../url-safety', () => {
|
|
23
|
+
class UnsafeWebhookUrlError extends Error {}
|
|
24
|
+
return {
|
|
25
|
+
assertSafeWebhookDeliveryUrl: jest.fn(async () => undefined),
|
|
26
|
+
safeWebhookFetch: jest.fn(),
|
|
27
|
+
UnsafeWebhookUrlError,
|
|
28
|
+
}
|
|
29
|
+
})
|
|
30
|
+
|
|
31
|
+
import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
|
|
32
|
+
import { enqueueWebhookDelivery, enqueueWebhookDeliveryFromJob } from '../queue'
|
|
33
|
+
import { safeWebhookFetch } from '../url-safety'
|
|
34
|
+
|
|
35
|
+
const findOneWithDecryptionMock = findOneWithDecryption as jest.MockedFunction<typeof findOneWithDecryption>
|
|
36
|
+
const enqueueWebhookDeliveryMock = enqueueWebhookDelivery as jest.MockedFunction<typeof enqueueWebhookDelivery>
|
|
37
|
+
const enqueueFromJobMock = enqueueWebhookDeliveryFromJob as jest.MockedFunction<typeof enqueueWebhookDeliveryFromJob>
|
|
38
|
+
const safeWebhookFetchMock = safeWebhookFetch as jest.MockedFunction<typeof safeWebhookFetch>
|
|
39
|
+
|
|
40
|
+
const JOB = { deliveryId: 'delivery-1', tenantId: 'tenant-1', organizationId: 'org-1' }
|
|
41
|
+
|
|
42
|
+
function buildDelivery(overrides: Record<string, unknown> = {}) {
|
|
43
|
+
return {
|
|
44
|
+
id: 'delivery-1',
|
|
45
|
+
webhookId: 'webhook-1',
|
|
46
|
+
eventType: 'customers.person.created',
|
|
47
|
+
messageId: 'msg-1',
|
|
48
|
+
payload: { type: 'customers.person.created', timestamp: new Date().toISOString(), data: {} },
|
|
49
|
+
status: 'pending',
|
|
50
|
+
attemptNumber: 0,
|
|
51
|
+
maxAttempts: 3,
|
|
52
|
+
targetUrl: 'https://example.com/hook',
|
|
53
|
+
organizationId: 'org-1',
|
|
54
|
+
tenantId: 'tenant-1',
|
|
55
|
+
enqueuedAt: new Date(),
|
|
56
|
+
...overrides,
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function buildEm(delivery: Record<string, unknown>, claimedRows: number) {
|
|
61
|
+
return {
|
|
62
|
+
findOne: jest.fn(async () => delivery),
|
|
63
|
+
nativeUpdate: jest.fn(async () => claimedRows),
|
|
64
|
+
flush: jest.fn(async () => undefined),
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
describe('processWebhookDeliveryJob — delivery claim', () => {
|
|
69
|
+
beforeEach(() => {
|
|
70
|
+
jest.clearAllMocks()
|
|
71
|
+
findOneWithDecryptionMock.mockResolvedValue({
|
|
72
|
+
id: 'webhook-1',
|
|
73
|
+
url: 'https://example.com/hook',
|
|
74
|
+
secret: 'whsec_dGVzdC1zZWNyZXQ=',
|
|
75
|
+
previousSecret: null,
|
|
76
|
+
httpMethod: 'POST',
|
|
77
|
+
timeoutMs: 5000,
|
|
78
|
+
customHeaders: null,
|
|
79
|
+
isActive: true,
|
|
80
|
+
consecutiveFailures: 0,
|
|
81
|
+
autoDisableThreshold: 0,
|
|
82
|
+
organizationId: 'org-1',
|
|
83
|
+
tenantId: 'tenant-1',
|
|
84
|
+
} as never)
|
|
85
|
+
safeWebhookFetchMock.mockResolvedValue(new Response('ok', { status: 200 }))
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it.each(['delivered', 'failed', 'expired'])('ignores a job for a %s delivery', async (status) => {
|
|
89
|
+
const delivery = buildDelivery({ status })
|
|
90
|
+
const em = buildEm(delivery, 1)
|
|
91
|
+
|
|
92
|
+
const result = await processWebhookDeliveryJob(em as unknown as EntityManager, JOB, { scheduleRetries: false })
|
|
93
|
+
|
|
94
|
+
expect(result).toBeNull()
|
|
95
|
+
expect(findOneWithDecryptionMock).not.toHaveBeenCalled()
|
|
96
|
+
expect(em.nativeUpdate).not.toHaveBeenCalled()
|
|
97
|
+
expect(em.flush).not.toHaveBeenCalled()
|
|
98
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
99
|
+
expect(delivery.status).toBe(status)
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
it('does not send when another worker holds the claim and rechecks once the lease ends', async () => {
|
|
103
|
+
const delivery = buildDelivery()
|
|
104
|
+
const em = buildEm(delivery, 0)
|
|
105
|
+
|
|
106
|
+
const result = await processWebhookDeliveryJob(em as unknown as EntityManager, JOB)
|
|
107
|
+
|
|
108
|
+
expect(result).toBeNull()
|
|
109
|
+
expect(em.nativeUpdate).toHaveBeenCalledTimes(1)
|
|
110
|
+
expect(em.flush).not.toHaveBeenCalled()
|
|
111
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
112
|
+
expect(enqueueFromJobMock).toHaveBeenCalledWith(JOB, 5000 + 15_000 + 1000)
|
|
113
|
+
expect(enqueueWebhookDeliveryMock).not.toHaveBeenCalled()
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
it('rechecks a delivery with a scheduled retry once the retry is due', async () => {
|
|
117
|
+
const em = buildEm(buildDelivery({ nextRetryAt: new Date(Date.now() + 120_000) }), 0)
|
|
118
|
+
|
|
119
|
+
await processWebhookDeliveryJob(em as unknown as EntityManager, JOB)
|
|
120
|
+
|
|
121
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
122
|
+
const [, recheckDelayMs] = enqueueFromJobMock.mock.calls[0]
|
|
123
|
+
expect(recheckDelayMs).toBeGreaterThan(110_000)
|
|
124
|
+
expect(recheckDelayMs).toBeLessThanOrEqual(121_000)
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
it('fails the job when the recheck cannot be scheduled so the queue retries it', async () => {
|
|
128
|
+
const em = buildEm(buildDelivery(), 0)
|
|
129
|
+
enqueueFromJobMock.mockRejectedValueOnce(new Error('queue unavailable'))
|
|
130
|
+
|
|
131
|
+
await expect(processWebhookDeliveryJob(em as unknown as EntityManager, JOB)).rejects.toThrow('queue unavailable')
|
|
132
|
+
|
|
133
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
134
|
+
expect(em.nativeUpdate).toHaveBeenCalledTimes(1)
|
|
135
|
+
})
|
|
136
|
+
|
|
137
|
+
it('does not schedule a recheck when retries are not scheduled', async () => {
|
|
138
|
+
const em = buildEm(buildDelivery(), 0)
|
|
139
|
+
|
|
140
|
+
await processWebhookDeliveryJob(em as unknown as EntityManager, JOB, { scheduleRetries: false })
|
|
141
|
+
|
|
142
|
+
expect(enqueueFromJobMock).not.toHaveBeenCalled()
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
it('releases its claim when processing throws after claiming', async () => {
|
|
146
|
+
const em = buildEm(buildDelivery(), 1)
|
|
147
|
+
em.flush.mockRejectedValueOnce(new Error('connection reset'))
|
|
148
|
+
|
|
149
|
+
await expect(
|
|
150
|
+
processWebhookDeliveryJob(em as unknown as EntityManager, JOB, { scheduleRetries: false }),
|
|
151
|
+
).rejects.toThrow('connection reset')
|
|
152
|
+
|
|
153
|
+
expect(safeWebhookFetchMock).not.toHaveBeenCalled()
|
|
154
|
+
expect(em.nativeUpdate).toHaveBeenCalledTimes(2)
|
|
155
|
+
const [, claimFilter, claimData] = em.nativeUpdate.mock.calls[0] as unknown as [unknown, unknown, { lastAttemptAt: Date }]
|
|
156
|
+
expect(claimFilter).toEqual(expect.objectContaining({ id: 'delivery-1' }))
|
|
157
|
+
const [, releaseFilter, releaseData] = em.nativeUpdate.mock.calls[1] as unknown as [unknown, Record<string, unknown>, Record<string, unknown>]
|
|
158
|
+
expect(releaseFilter).toEqual(expect.objectContaining({
|
|
159
|
+
id: 'delivery-1',
|
|
160
|
+
tenantId: 'tenant-1',
|
|
161
|
+
organizationId: 'org-1',
|
|
162
|
+
status: 'sending',
|
|
163
|
+
lastAttemptAt: claimData.lastAttemptAt,
|
|
164
|
+
}))
|
|
165
|
+
expect(releaseData).toEqual(expect.objectContaining({ status: 'pending' }))
|
|
166
|
+
})
|
|
167
|
+
|
|
168
|
+
it('claims a pending delivery before sending it', async () => {
|
|
169
|
+
const delivery = buildDelivery()
|
|
170
|
+
const em = buildEm(delivery, 1)
|
|
171
|
+
|
|
172
|
+
const result = await processWebhookDeliveryJob(em as unknown as EntityManager, JOB, { scheduleRetries: false })
|
|
173
|
+
|
|
174
|
+
expect(result).toEqual({ status: 'delivered', deliveryId: 'delivery-1' })
|
|
175
|
+
expect(safeWebhookFetchMock).toHaveBeenCalledTimes(1)
|
|
176
|
+
expect(em.nativeUpdate.mock.invocationCallOrder[0]).toBeLessThan(safeWebhookFetchMock.mock.invocationCallOrder[0])
|
|
177
|
+
const [, filter, data] = em.nativeUpdate.mock.calls[0] as unknown as [
|
|
178
|
+
unknown,
|
|
179
|
+
{ $or: [{ status: string; nextRetryAt: null }, { status: string; nextRetryAt: { $lte: Date } }, { status: string; lastAttemptAt: { $lt: Date } }] },
|
|
180
|
+
{ status: string; lastAttemptAt: Date },
|
|
181
|
+
]
|
|
182
|
+
expect(filter).toEqual(expect.objectContaining({ id: 'delivery-1', tenantId: 'tenant-1', organizationId: 'org-1' }))
|
|
183
|
+
expect(filter.$or[0]).toEqual({ status: 'pending', nextRetryAt: null })
|
|
184
|
+
expect(filter.$or[1].status).toBe('pending')
|
|
185
|
+
expect(filter.$or[1].nextRetryAt.$lte.getTime() - data.lastAttemptAt.getTime()).toBe(1000)
|
|
186
|
+
expect(filter.$or[2].status).toBe('sending')
|
|
187
|
+
expect(data.lastAttemptAt.getTime() - filter.$or[2].lastAttemptAt.$lt.getTime()).toBe(5000 + 15_000)
|
|
188
|
+
expect(data.status).toBe('sending')
|
|
189
|
+
expect(delivery.attemptNumber).toBe(1)
|
|
190
|
+
})
|
|
191
|
+
})
|
|
@@ -117,6 +117,7 @@ describe('processWebhookDeliveryJob — custom headers cannot override signed he
|
|
|
117
117
|
function buildEm(delivery: Record<string, unknown>) {
|
|
118
118
|
return {
|
|
119
119
|
findOne: jest.fn(async () => delivery),
|
|
120
|
+
nativeUpdate: jest.fn(async () => 1),
|
|
120
121
|
flush: jest.fn(async () => undefined),
|
|
121
122
|
} as unknown as EntityManager
|
|
122
123
|
}
|
|
@@ -41,6 +41,7 @@ describe('processWebhookDeliveryJob — URL safety guard', () => {
|
|
|
41
41
|
function buildEm(delivery: Record<string, unknown>) {
|
|
42
42
|
return {
|
|
43
43
|
findOne: jest.fn(async () => delivery),
|
|
44
|
+
nativeUpdate: jest.fn(async () => 1),
|
|
44
45
|
flush: jest.fn(async () => undefined),
|
|
45
46
|
} as unknown as EntityManager
|
|
46
47
|
}
|
|
@@ -138,6 +138,7 @@ describe('processWebhookDeliveryJob auto-disable invalidates the subscription ca
|
|
|
138
138
|
function buildEm(delivery: Record<string, unknown>) {
|
|
139
139
|
return {
|
|
140
140
|
findOne: jest.fn(async () => delivery),
|
|
141
|
+
nativeUpdate: jest.fn(async () => 1),
|
|
141
142
|
flush: jest.fn(async () => undefined),
|
|
142
143
|
} as unknown as EntityManager
|
|
143
144
|
}
|
|
@@ -171,6 +172,7 @@ describe('processWebhookDeliveryJob auto-disable invalidates the subscription ca
|
|
|
171
172
|
}
|
|
172
173
|
const em = {
|
|
173
174
|
findOne: jest.fn(async () => delivery),
|
|
175
|
+
nativeUpdate: jest.fn(async () => 1),
|
|
174
176
|
flush: jest.fn(async () => { calls.push(`flush:isActive=${webhook.isActive}`) }),
|
|
175
177
|
} as unknown as EntityManager
|
|
176
178
|
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import type { EntityManager } from '@mikro-orm/postgresql'
|
|
2
2
|
import { buildWebhookHeaders, generateMessageId } from '@open-mercato/shared/lib/webhooks'
|
|
3
3
|
import { findOneWithDecryption } from '@open-mercato/shared/lib/encryption/find'
|
|
4
|
+
import { createLogger } from '@open-mercato/shared/lib/logger'
|
|
5
|
+
import { getTelemetryRuntime } from '@open-mercato/shared/lib/telemetry/runtime'
|
|
4
6
|
import { WebhookDeliveryEntity, WebhookEntity } from '../data/entities'
|
|
5
7
|
import { emitWebhooksEvent } from '../events'
|
|
6
|
-
import { enqueueWebhookDelivery } from './queue'
|
|
8
|
+
import { enqueueWebhookDelivery, enqueueWebhookDeliveryFromJob } from './queue'
|
|
7
9
|
import { isWebhookIntegrationEnabled, WEBHOOK_INTEGRATION_DISABLED_MESSAGE } from './integration-state'
|
|
8
10
|
import { sanitizeWebhookCustomHeaders } from './custom-headers'
|
|
9
11
|
import {
|
|
@@ -29,6 +31,13 @@ export interface CreateWebhookDeliveryInput {
|
|
|
29
31
|
payload: Record<string, unknown>
|
|
30
32
|
}
|
|
31
33
|
|
|
34
|
+
const logger = createLogger('webhooks')
|
|
35
|
+
|
|
36
|
+
const CLAIMABLE_DELIVERY_STATUSES: ReadonlySet<string> = new Set(['pending', 'sending'])
|
|
37
|
+
const SENDING_LEASE_GRACE_MS = 15_000
|
|
38
|
+
const CLAIM_RECHECK_MARGIN_MS = 1_000
|
|
39
|
+
const RETRY_DUE_TOLERANCE_MS = 1_000
|
|
40
|
+
|
|
32
41
|
type ProcessWebhookDeliveryOptions = {
|
|
33
42
|
scheduleRetries?: boolean
|
|
34
43
|
resolver?: WebhookCacheDependencyResolver
|
|
@@ -87,6 +96,7 @@ export async function processWebhookDeliveryJob(
|
|
|
87
96
|
})
|
|
88
97
|
|
|
89
98
|
if (!delivery) return null
|
|
99
|
+
if (!CLAIMABLE_DELIVERY_STATUSES.has(delivery.status)) return null
|
|
90
100
|
|
|
91
101
|
const webhook = await findOneWithDecryption(
|
|
92
102
|
em,
|
|
@@ -101,6 +111,32 @@ export async function processWebhookDeliveryJob(
|
|
|
101
111
|
{ tenantId: job.tenantId, organizationId: job.organizationId },
|
|
102
112
|
)
|
|
103
113
|
|
|
114
|
+
const claimedAt = new Date()
|
|
115
|
+
const leaseMs = (webhook?.timeoutMs ?? 0) + SENDING_LEASE_GRACE_MS
|
|
116
|
+
const claimed = await claimWebhookDelivery(em, delivery, claimedAt, leaseMs)
|
|
117
|
+
if (!claimed) {
|
|
118
|
+
if (options.scheduleRetries !== false) {
|
|
119
|
+
const retryDueInMs = (delivery.nextRetryAt?.getTime() ?? 0) - claimedAt.getTime()
|
|
120
|
+
await scheduleWebhookDeliveryClaimRecheck(job, Math.max(leaseMs, retryDueInMs))
|
|
121
|
+
}
|
|
122
|
+
return null
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
try {
|
|
126
|
+
return await processClaimedWebhookDelivery(em, delivery, webhook, options, claimedAt)
|
|
127
|
+
} catch (error) {
|
|
128
|
+
await releaseWebhookDeliveryClaim(em, delivery, claimedAt)
|
|
129
|
+
throw error
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function processClaimedWebhookDelivery(
|
|
134
|
+
em: EntityManager,
|
|
135
|
+
delivery: WebhookDeliveryEntity,
|
|
136
|
+
webhook: WebhookEntity | null,
|
|
137
|
+
options: ProcessWebhookDeliveryOptions,
|
|
138
|
+
claimedAt: Date,
|
|
139
|
+
): Promise<{ status: string; deliveryId: string }> {
|
|
104
140
|
if (!webhook) {
|
|
105
141
|
delivery.status = 'failed'
|
|
106
142
|
delivery.errorMessage = 'Webhook not found'
|
|
@@ -144,7 +180,7 @@ export async function processWebhookDeliveryJob(
|
|
|
144
180
|
const bodyPayload = normalizeWebhookBody(delivery.eventType, delivery.payload)
|
|
145
181
|
delivery.payload = bodyPayload
|
|
146
182
|
delivery.status = 'sending'
|
|
147
|
-
delivery.lastAttemptAt =
|
|
183
|
+
delivery.lastAttemptAt = claimedAt
|
|
148
184
|
delivery.errorMessage = null
|
|
149
185
|
await em.flush()
|
|
150
186
|
|
|
@@ -279,6 +315,61 @@ export async function processWebhookDeliveryJob(
|
|
|
279
315
|
}
|
|
280
316
|
}
|
|
281
317
|
|
|
318
|
+
async function claimWebhookDelivery(
|
|
319
|
+
em: EntityManager,
|
|
320
|
+
delivery: WebhookDeliveryEntity,
|
|
321
|
+
claimedAt: Date,
|
|
322
|
+
leaseMs: number,
|
|
323
|
+
): Promise<boolean> {
|
|
324
|
+
const staleSendingBefore = new Date(claimedAt.getTime() - leaseMs)
|
|
325
|
+
const retryDueBy = new Date(claimedAt.getTime() + RETRY_DUE_TOLERANCE_MS)
|
|
326
|
+
const affected = await em.nativeUpdate(
|
|
327
|
+
WebhookDeliveryEntity,
|
|
328
|
+
{
|
|
329
|
+
id: delivery.id,
|
|
330
|
+
tenantId: delivery.tenantId,
|
|
331
|
+
organizationId: delivery.organizationId,
|
|
332
|
+
$or: [
|
|
333
|
+
{ status: 'pending', nextRetryAt: null },
|
|
334
|
+
{ status: 'pending', nextRetryAt: { $lte: retryDueBy } },
|
|
335
|
+
{ status: 'sending', lastAttemptAt: { $lt: staleSendingBefore } },
|
|
336
|
+
],
|
|
337
|
+
},
|
|
338
|
+
{ status: 'sending', lastAttemptAt: claimedAt, errorMessage: null, updatedAt: claimedAt },
|
|
339
|
+
)
|
|
340
|
+
return affected > 0
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
async function releaseWebhookDeliveryClaim(
|
|
344
|
+
em: EntityManager,
|
|
345
|
+
delivery: WebhookDeliveryEntity,
|
|
346
|
+
claimedAt: Date,
|
|
347
|
+
): Promise<void> {
|
|
348
|
+
try {
|
|
349
|
+
await em.nativeUpdate(
|
|
350
|
+
WebhookDeliveryEntity,
|
|
351
|
+
{
|
|
352
|
+
id: delivery.id,
|
|
353
|
+
tenantId: delivery.tenantId,
|
|
354
|
+
organizationId: delivery.organizationId,
|
|
355
|
+
status: 'sending',
|
|
356
|
+
lastAttemptAt: claimedAt,
|
|
357
|
+
},
|
|
358
|
+
{ status: 'pending', updatedAt: new Date() },
|
|
359
|
+
)
|
|
360
|
+
} catch (error) {
|
|
361
|
+
logger.error('Failed to release webhook delivery claim', { deliveryId: delivery.id, err: error })
|
|
362
|
+
getTelemetryRuntime()?.reportError(error, { module: 'webhooks', code: 'webhooks.delivery_claim_release_failed' })
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
async function scheduleWebhookDeliveryClaimRecheck(job: WebhookDeliveryJob, waitMs: number): Promise<void> {
|
|
367
|
+
await enqueueWebhookDeliveryFromJob(
|
|
368
|
+
{ deliveryId: job.deliveryId, tenantId: job.tenantId, organizationId: job.organizationId },
|
|
369
|
+
waitMs + CLAIM_RECHECK_MARGIN_MS,
|
|
370
|
+
)
|
|
371
|
+
}
|
|
372
|
+
|
|
282
373
|
type RecordWebhookSafetyFailureInput = {
|
|
283
374
|
em: EntityManager
|
|
284
375
|
delivery: WebhookDeliveryEntity
|
|
@@ -66,6 +66,10 @@ export async function enqueueWebhookDelivery(job: WebhookDeliveryJob, delayMs?:
|
|
|
66
66
|
return jobId
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
+
export async function enqueueWebhookDeliveryFromJob(job: WebhookDeliveryJob, delayMs: number): Promise<string> {
|
|
70
|
+
return getWebhookQueue().enqueue(job, delayMs > 0 ? { delayMs } : undefined)
|
|
71
|
+
}
|
|
72
|
+
|
|
69
73
|
export function getInboundDispatchQueue(
|
|
70
74
|
queueName: string = WEBHOOK_INBOUND_DISPATCH_QUEUE,
|
|
71
75
|
): Queue<InboundDispatchJob> {
|