@rytass/bpm-core-nestjs-module 0.13.3 → 0.13.5

package/src/lib/workflow-webhook/workflow-webhook-delivery.service.js

@@ -0,0 +1,705 @@
+"use strict";
+var WorkflowWebhookDeliveryService_1;
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkflowWebhookDeliveryService = exports.BPM_WORKFLOW_WEBHOOK_FETCH = void 0;
+const tslib_1 = require("tslib");
+const node_crypto_1 = require("node:crypto");
+const common_1 = require("@nestjs/common");
+const typeorm_1 = require("@nestjs/typeorm");
+const typeorm_2 = require("typeorm");
+const outbox_1 = require("../common/outbox");
+const activity_log_entity_1 = require("../workflow-engine/activity-log.entity");
+const workflow_engine_enums_1 = require("../workflow-engine/workflow-engine.enums");
+const workflow_graph_1 = require("@rytass/bpm-core-shared/workflow-graph");
+const workflow_webhook_allowlist_1 = require("./workflow-webhook-allowlist");
+const workflow_webhook_delivery_enums_1 = require("./workflow-webhook-delivery.enums");
+const workflow_webhook_delivery_entity_1 = require("./workflow-webhook-delivery.entity");
+const workflow_webhook_enqueue_1 = require("./workflow-webhook-enqueue");
+const workflow_webhook_options_1 = require("./workflow-webhook-options");
+const workflow_webhook_service_1 = require("./workflow-webhook.service");
+/**
+ * The HTTP client deliveries go through. Defaults to the global `fetch`;
+ * bound mainly so tests can observe requests without a network.
+ */
+exports.BPM_WORKFLOW_WEBHOOK_FETCH = Symbol('BPM_WORKFLOW_WEBHOOK_FETCH');
+/** How much of a failing response body an administrator gets to see. */
+const MAX_ERROR_DETAIL_LENGTH = 500;
+/** Bytes read from a failing response before the rest is discarded. */
+const MAX_ERROR_BODY_BYTES = 4_096;
+const ALLOWED_METHODS = ['PATCH', 'POST', 'PUT'];
+/** Deliveries from one claimed batch that are attempted at the same time. */
+const MAX_CONCURRENT_ATTEMPTS = 5;
+/**
+ * A claimed row whose attempt never wrote back is reclaimable after this. Rows
+ * are claimed no more than {@link MAX_CONCURRENT_ATTEMPTS} at a time and all of
+ * them start at once, so no claimed row ever waits in a queue: the window only
+ * has to cover one `buildRequest()` plus one request (30 s each at most) and
+ * the write.
+ */
+const STALE_CLAIM_MS = workflow_webhook_options_1.WORKFLOW_WEBHOOK_MAX_TIMEOUT_MS * 3;
+let WorkflowWebhookDeliveryService = WorkflowWebhookDeliveryService_1 = class WorkflowWebhookDeliveryService {
+    constructor(deliveryRepository, activityLogRepository, webhookService, options = workflow_webhook_options_1.DEFAULT_BPM_WORKFLOW_WEBHOOK_OPTIONS, fetchImpl) {
+        this.deliveryRepository = deliveryRepository;
+        this.activityLogRepository = activityLogRepository;
+        this.webhookService = webhookService;
+        this.options = options;
+        this.logger = new common_1.Logger(WorkflowWebhookDeliveryService_1.name);
+        this.fetchImpl =
+            fetchImpl ??
+                ((input, init) => fetch(input, init));
+    }
+    /** The clock every attempt reads; separate so tests can pin it. */
+    readCurrentTime() {
+        return new Date();
+    }
+    /**
+     * Queues a NOTIFY node's webhooks inside the engine transaction `manager`.
+     * Sends nothing: delivery starts only once that transaction commits
+     * (ADR 18 §3.5). Rows that can already be judged undeliverable are written
+     * as `FAILED` together with their terminal activity log.
+     */
+    async enqueueNotifyWebhooks(manager, context, targets) {
+        const listIssues = (0, workflow_graph_1.readNotifyWebhookStructureIssues)(targets).filter((issue) => issue.targetIndex === null);
+        // Publish refuses these, so reaching one means the snapshot was written
+        // around the lint. Nothing can be queued, but it must not pass silently.
+        if (listIssues.length) {
+            this.logger.warn(`Instance ${context.instance.id} node ${context.node.id}: webhooks not queued (${listIssues
+                .map((issue) => issue.code)
+                .join(', ')})`);
+        }
+        const drafts = await (0, workflow_webhook_enqueue_1.buildWorkflowWebhookDeliveryDrafts)({
+            context,
+            resolveEndpoint: (key, version) => this.webhookService.getEndpoint(key, version, manager),
+            targets,
+        });
+        if (!drafts.length) {
+            return [];
+        }
+        const repository = manager.getRepository(workflow_webhook_delivery_entity_1.WorkflowWebhookDeliveryEntity);
+        const saved = await repository.save(drafts.map((draft) => repository.create({ ...draft, attemptCount: 0 })));
+        const failed = saved.filter((row) => row.status === workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.FAILED);
+        if (failed.length) {
+            const activityRepository = manager.getRepository(activity_log_entity_1.ActivityLogEntity);
+            const activities = await Promise.all(failed.map(async (row) => activityRepository.create(this.createTerminalActivity(row, await this.readEndpointLabel(row, manager)))));
+            await activityRepository.save(activities);
+        }
+        return saved.map((row) => row.id);
+    }
+    /** Every delivery queued for an instance, oldest first (ADR 18 §3.10). */
+    async listInstanceDeliveries(instanceId) {
+        return this.deliveryRepository.find({
+            order: { createdAt: 'ASC' },
+            where: { instanceId },
+        });
+    }
+    /**
+     * Puts a `FAILED` delivery back in the queue with a fresh attempt budget
+     * and starts it at once. The delivery id does not change, so a receiver
+     * that already processed an earlier attempt still recognizes it.
+     *
+     * Only `FAILED` rows that were actually attempted qualify. A row in flight
+     * or already sent would be sent again; a row that failed while being queued
+     * (endpoint missing, lookup failed, parameter invalid) froze an event that
+     * never passed the parameter checks, and sending it would break the
+     * endpoint's contract. The state change and the audit entry commit together.
+     */
+    async retryFailedDelivery(id, actorMemberId) {
+        const existing = await this.deliveryRepository.findOne({ where: { id } });
+        if (!existing) {
+            throw new common_1.NotFoundException(`Webhook delivery ${id} was not found`);
+        }
+        // Read before the transaction: a host lookup has no business holding the
+        // row lock or a second pooled connection.
+        const endpointLabel = await this.readEndpointLabel(existing);
+        const retried = await this.deliveryRepository.manager.transaction(async (manager) => {
+            const repository = manager.getRepository(workflow_webhook_delivery_entity_1.WorkflowWebhookDeliveryEntity);
+            const result = await repository.update({
+                attemptCount: (0, typeorm_2.MoreThan)(0),
+                id,
+                status: workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.FAILED,
+            }, {
+                attemptCount: 0,
+                nextRetryAt: null,
+                status: workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.PENDING,
+            });
+            const row = await repository.findOne({ where: { id } });
+            if (!row) {
+                throw new common_1.NotFoundException(`Webhook delivery ${id} was not found`);
+            }
+            if (!result.affected) {
+                throw new common_1.BadRequestException(row.status === workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.FAILED
+                    ? `Webhook delivery ${id} failed before it was ever sent (${row.lastErrorCode ?? 'unknown'}) and cannot be retried`
+                    : `Webhook delivery ${id} is ${row.status}; only FAILED deliveries can be retried`);
+            }
+            const activityRepository = manager.getRepository(activity_log_entity_1.ActivityLogEntity);
+            await activityRepository.save(activityRepository.create({
+                actorMemberId,
+                eventType: workflow_engine_enums_1.ActivityLogEventTypeEnum.WEBHOOK_DELIVERY_RETRIED,
+                instanceId: row.instanceId,
+                nodeId: row.nodeId,
+                payload: {
+                    action: 'NOTIFY_WEBHOOK',
+                    deliveryId: row.id,
+                    endpointKey: row.endpointKey,
+                    endpointLabel,
+                    endpointVersion: row.endpointVersion,
+                    previousErrorCode: row.lastErrorCode,
+                    targetId: row.targetId,
+                },
+                taskId: null,
+            }));
+            return row;
+        });
+        // After the commit, like the subscriber's kick: the scheduler would get
+        // there too, but an administrator who pressed retry expects an answer now.
+        setImmediate(() => {
+            this.deliverByIds([retried.id]).catch((error) => {
+                this.logger.warn(`Retried webhook delivery ${retried.id} could not start immediately (${readErrorName(error)})`);
+            });
+        });
+        return retried;
+    }
+    /**
+     * One scheduler pass: claim what is due, a few rows at a time, until the
+     * batch size is reached or nothing more is due.
+     *
+     * Claiming the whole batch up front and attempting it a few at a time left
+     * the rows at the back holding a claim while they waited; past the stale
+     * window another worker took them over. Claiming only what starts at once
+     * means a claimed row is always a row in flight.
+     */
+    async deliverDue(now) {
+        const { batchSize } = this.options.delivery;
+        let total = 0;
+        while (total < batchSize) {
+            const claimed = await this.claim({
+                ids: null,
+                limit: Math.min(MAX_CONCURRENT_ATTEMPTS, batchSize - total),
+                now: total === 0 && now ? now : this.readCurrentTime(),
+            });
+            if (!claimed.length) {
+                break;
+            }
+            await this.attemptTogether(claimed);
+            total += claimed.length;
+        }
+        return total;
+    }
+    /**
+     * Attempts specific rows right after the transaction that queued them
+     * committed, in groups that start together. Rows another worker already
+     * claimed, or that are not due, are skipped by the claim itself.
+     */
+    async deliverByIds(ids, now) {
+        const unique = [...new Set(ids)];
+        let total = 0;
+        for (let start = 0; start < unique.length; start += MAX_CONCURRENT_ATTEMPTS) {
+            const group = unique.slice(start, start + MAX_CONCURRENT_ATTEMPTS);
+            const claimed = await this.claim({
+                ids: group,
+                limit: group.length,
+                now: start === 0 && now ? now : this.readCurrentTime(),
+            });
+            await this.attemptTogether(claimed);
+            total += claimed.length;
+        }
+        return total;
+    }
+    /**
+     * Starts every claimed row at once (never more than
+     * {@link MAX_CONCURRENT_ATTEMPTS}). Strictly one by one, a single receiver
+     * that runs into its timeout held every other delivery back by that long
+     * (seen on the wrapper host: a 10 s timeout delayed an unrelated endpoint by
+     * 10 s). Each row still re-stamps and writes back under its own claim.
+     */
+    async attemptTogether(rows) {
+        await Promise.all(rows.map((row) => this.attemptSafely(row)));
+    }
+    async attemptSafely(row) {
+        try {
+            await this.attempt(row);
+        }
+        catch (error) {
+            // Only a failed write can land here. The row stays claimed and the
+            // stale window hands it to a later scan. The error's name only: a
+            // driver message can quote the parameters it was given.
+            this.logger.error(`Webhook delivery ${row.id} could not record its outcome (${readErrorName(error)})`);
+        }
+    }
+    async claim({ ids, limit, now, }) {
+        const staleBefore = new Date(now.getTime() - STALE_CLAIM_MS);
+        return this.deliveryRepository.manager.transaction(async (manager) => {
+            const claimedRows = (await manager.query(`
+            UPDATE workflow_webhook_deliveries
+               SET status = $1,
+                   last_attempt_at = $2,
+                   updated_at = $2
+             WHERE id IN (
+               SELECT id
+                 FROM workflow_webhook_deliveries
+                WHERE (
+                    (status = $3 AND (next_retry_at IS NULL OR next_retry_at <= $2))
+                    OR (status = $1 AND last_attempt_at <= $4)
+                  )
+                  AND ($6::uuid[] IS NULL OR id = ANY($6::uuid[]))
+                ORDER BY created_at ASC
+                FOR UPDATE SKIP LOCKED
+                LIMIT $5
+             )
+             RETURNING id
+          `, [
+                workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.DELIVERY_IN_PROGRESS,
+                now,
+                workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.PENDING,
+                staleBefore,
+                limit,
+                ids ? [...ids] : null,
+            ]));
+            const claimedIds = (0, outbox_1.readClaimedIds)(claimedRows);
+            return claimedIds.length
+                ? manager
+                    .getRepository(workflow_webhook_delivery_entity_1.WorkflowWebhookDeliveryEntity)
+                    .find({ where: { id: (0, typeorm_2.In)([...claimedIds]) } })
+                : [];
+        });
+    }
+    async attempt(row) {
+        const startedAt = this.readCurrentTime();
+        // Re-stamping at the start of the attempt confirms this worker still owns
+        // the row: if another one reclaimed it between the claim and this line,
+        // the stamp no longer matches and this attempt stands down instead of
+        // sending a second time.
+        if (!(await this.moveClaim(row, row.lastAttemptAt, startedAt))) {
+            return;
+        }
+        const claimed = {
+            ...row,
+            lastAttemptAt: startedAt,
+        };
+        const attempt = row.attemptCount + 1;
+        const outcome = await this.resolveOutcome(claimed, attempt).catch((error) => ({
+            code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.INTERNAL_ERROR,
+            detail: readErrorName(error),
+            kind: 'RETRY',
+            status: null,
+        }));
+        await this.record(claimed, attempt, outcome, this.readCurrentTime());
+    }
+    async resolveOutcome(row, attempt) {
+        const lookup = await this.webhookService
+            .getEndpoint(row.endpointKey, row.endpointVersion)
+            .then((entry) => ({ entry }), (error) => ({ error }));
+        if ('error' in lookup) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.ENDPOINT_LOOKUP_FAILED,
+                detail: readErrorName(lookup.error),
+                kind: 'RETRY',
+                status: null,
+            };
+        }
+        if (!lookup.entry) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.ENDPOINT_MISSING,
+                detail: `${row.endpointKey}@${row.endpointVersion} is not registered`,
+                kind: 'FAIL',
+                status: null,
+            };
+        }
+        // Checked per attempt: disabling an endpoint stops what is already queued.
+        if (lookup.entry.endpoint.descriptor.disabled) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.ENDPOINT_DISABLED,
+                detail: `${row.endpointKey}@${row.endpointVersion} is disabled`,
+                kind: 'FAIL',
+                status: null,
+            };
+        }
+        return this.sendEvent(lookup.entry, {
+            ...row.event,
+            attempt,
+            deliveryId: row.id,
+            eventType: 'workflow.notify',
+        });
+    }
+    /**
+     * Sends one event to an endpoint exactly as a delivery attempt would —
+     * same refusals, allowlist, timeout, redirect handling and signature —
+     * without a delivery row, for an administrator's test send (ADR 18 §3.13).
+     * The endpoint may be disabled: testing before enabling is the point.
+     */
+    async sendTestEvent(entry, event) {
+        const outcome = await this.sendEvent(entry, event).catch((error) => ({
+            code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.INTERNAL_ERROR,
+            detail: readErrorName(error),
+            kind: 'FAIL',
+            status: null,
+        }));
+        return outcome.kind === 'SENT'
+            ? { errorCode: null, errorDetail: null, ok: true, status: outcome.status }
+            : {
+                errorCode: outcome.code,
+                errorDetail: outcome.detail ? truncate(outcome.detail) : null,
+                ok: false,
+                status: outcome.status,
+            };
+    }
+    async sendEvent(entry, event) {
+        // Wrapped so a host that throws synchronously, or returns a non-promise,
+        // is judged like any other failure instead of escaping the attempt.
+        const built = await (0, outbox_1.withDispatchTimeout)(Promise.resolve().then(() => entry.endpoint.buildRequest(event)), workflow_webhook_options_1.WORKFLOW_WEBHOOK_MAX_TIMEOUT_MS).then((value) => ({ value }), (error) => ({ error }));
+        if ('error' in built) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.BUILD_REQUEST_FAILED,
+                // The host's message may name the secret it failed to read, so only
+                // the error's kind is kept.
+                detail: readErrorName(built.error),
+                kind: 'RETRY',
+                status: null,
+            };
+        }
+        if (!isWebhookRequest(built.value)) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.INVALID_REQUEST,
+                detail: 'buildRequest did not return a request with a string url',
+                kind: 'FAIL',
+                status: null,
+            };
+        }
+        const request = built.value;
+        const refusal = this.readRequestRefusal(request, entry);
+        if (refusal) {
+            return refusal;
+        }
+        const body = request.body ?? JSON.stringify(event);
+        const timeoutMs = Math.min(typeof request.timeoutMs === 'number' && request.timeoutMs > 0
+            ? request.timeoutMs
+            : this.options.delivery.defaultTimeoutMs, workflow_webhook_options_1.WORKFLOW_WEBHOOK_MAX_TIMEOUT_MS);
+        return this.post({
+            body,
+            // Signed at the moment of sending, not when the row was claimed, so a
+            // receiver's replay window measures the request it actually got.
+            headers: buildHeaders(request, body, event.deliveryId, this.readCurrentTime()),
+            method: request.method ?? 'POST',
+            timeoutMs,
+            url: request.url,
+        });
+    }
+    /**
+     * A request BPM refuses to send at all. These never become retryable: the
+     * same endpoint would return the same request next time. None of the
+     * details quote the URL.
+     */
+    readRequestRefusal(request, entry) {
+        const invalid = (detail) => ({
+            code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.INVALID_REQUEST,
+            detail,
+            kind: 'FAIL',
+            status: null,
+        });
+        const parsed = (() => {
+            try {
+                return new URL(request.url);
+            }
+            catch {
+                return null;
+            }
+        })();
+        if (!parsed ||
+            (parsed.protocol !== 'https:' && parsed.protocol !== 'http:')) {
+            return invalid('buildRequest returned a URL that is not http(s)');
+        }
+        if (parsed.username || parsed.password) {
+            return invalid('buildRequest returned a URL that carries credentials');
+        }
+        if (request.method !== undefined &&
+            !ALLOWED_METHODS.includes(String(request.method))) {
+            return invalid('buildRequest returned a method other than POST, PUT or PATCH');
+        }
+        if (request.body !== undefined && typeof request.body !== 'string') {
+            return invalid('buildRequest returned a body that is not a string');
+        }
+        const enforceAllowlist = entry.source === 'DATABASE' || this.options.enforceAllowlistForRegistry;
+        // Checked on every attempt, not only when the endpoint was saved, so
+        // tightening the allowlist stops deliveries that are already queued.
+        if (enforceAllowlist &&
+            !(0, workflow_webhook_allowlist_1.isWorkflowWebhookUrlAllowed)(request.url, this.options.allowedUrlPatterns)) {
+            return {
+                code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.URL_NOT_ALLOWED,
+                detail: null,
+                kind: 'FAIL',
+                status: null,
+            };
+        }
+        return null;
+    }
+    async post({ body, headers, method, timeoutMs, url, }) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), timeoutMs);
+        timer.unref();
+        try {
+            const response = await this.fetchImpl(url, {
+                body,
+                headers,
+                method,
+                // A 3xx is a failure, never followed: an allowed host must not be
+                // able to bounce the request somewhere the allowlist would refuse.
+                redirect: 'manual',
+                signal: controller.signal,
+            });
+            const status = response.status;
+            if (status >= 200 && status < 300) {
+                await discardBody(response);
+                return { kind: 'SENT', status };
+            }
+            // `redirect: 'manual'` surfaces as an opaque redirect with status 0 in
+            // spec-compliant fetch implementations and as the raw 3xx in Node's.
+            if (response.type === 'opaqueredirect' ||
+                (status >= 300 && status < 400)) {
+                await discardBody(response);
+                return {
+                    code: workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.REDIRECT,
+                    detail: null,
+                    kind: 'FAIL',
+                    status: status || null,
+                };
+            }
+            const detail = truncate(await readBodyPrefix(response));
+            return {
+                code: `WEBHOOK_HTTP_${status}`,
+                detail: detail || null,
+                kind: isRetryableStatus(status) ? 'RETRY' : 'FAIL',
+                status,
+            };
+        }
+        catch (error) {
+            const aborted = controller.signal.aborted ||
+                (error instanceof Error && error.name === 'AbortError');
+            return {
+                code: aborted
+                    ? workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.TIMEOUT
+                    : workflow_webhook_delivery_enums_1.WORKFLOW_WEBHOOK_DELIVERY_ERROR_CODES.NETWORK,
+                // A fetch error message can quote the URL; the system error code
+                // (ECONNREFUSED, ENOTFOUND, ...) says what happened without it.
+                detail: aborted ? null : readNetworkErrorCode(error),
+                kind: 'RETRY',
+                status: null,
+            };
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    async record(row, attempt, outcome, finishedAt) {
+        const exhausted = outcome.kind === 'RETRY' && attempt >= this.options.delivery.maxAttempts;
+        const terminal = outcome.kind !== 'RETRY' || exhausted;
+        const changes = outcome.kind === 'SENT'
+            ? {
+                attemptCount: attempt,
+                lastErrorCode: null,
+                lastErrorDetail: null,
+                lastResponseStatus: outcome.status,
+                nextRetryAt: null,
+                sentAt: finishedAt,
+                status: workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.SENT,
+            }
+            : {
+                attemptCount: attempt,
+                lastErrorCode: outcome.code,
+                // Every detail source passes through here, so every one is made
+                // storable — including a host error's `name`.
+                lastErrorDetail: outcome.detail === null ? null : truncate(String(outcome.detail)),
+                lastResponseStatus: outcome.status,
+                nextRetryAt: terminal
+                    ? null
+                    : new Date(finishedAt.getTime() + this.readRetryDelay(attempt)),
+                status: terminal
+                    ? workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.FAILED
+                    : workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.PENDING,
+            };
+        // Written only if this worker still holds the claim it started with, so a
+        // late write-back can never overwrite what another worker already
+        // recorded — a SENT row least of all.
+        const result = await this.deliveryRepository.update({
+            id: row.id,
+            lastAttemptAt: row.lastAttemptAt ?? (0, typeorm_2.IsNull)(),
+            status: workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.DELIVERY_IN_PROGRESS,
+        }, changes);
+        if (!result.affected) {
+            this.logger.warn(`Webhook delivery ${row.id} was reclaimed before attempt ${attempt} finished; its outcome was discarded`);
+            return;
+        }
+        // Intermediate retries stay off the instance timeline; only the final
+        // word about a delivery is worth a reader's attention (ADR 18 §3.10).
+        if (terminal) {
+            // The outcome is already recorded; a failed timeline write must not be
+            // reported as if the delivery itself were unrecorded.
+            const endpointLabel = await this.readEndpointLabel(row);
+            await this.activityLogRepository
+                .save(this.activityLogRepository.create(this.createTerminalActivity({ ...row, ...changes }, endpointLabel)))
+                .catch((error) => {
+                this.logger.error(`Webhook delivery ${row.id} is ${changes.status} but its activity log could not be written (${readErrorName(error)})`);
+            });
+        }
+    }
+    /**
+     * The endpoint's display name for the timeline, which every reader of the
+     * instance sees; `null` when the endpoint is gone and the key must stand in.
+     */
+    async readEndpointLabel(row, manager) {
+        // Every step is guarded: a host entry missing its descriptor must cost
+        // the label, not the activity log it is written into.
+        return Promise.resolve()
+            .then(() => this.webhookService.getEndpoint(row.endpointKey, row.endpointVersion, manager))
+            .then((entry) => {
+            const label = entry?.endpoint?.descriptor?.label;
+            return typeof label === 'string' ? label : null;
+        })
+            .catch(() => null);
+    }
+    /** Takes the claim over from `from` to `to`; `false` if it moved on. */
+    async moveClaim(row, from, to) {
+        const result = await this.deliveryRepository.update({
+            id: row.id,
+            lastAttemptAt: from ?? (0, typeorm_2.IsNull)(),
+            status: workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.DELIVERY_IN_PROGRESS,
+        }, { lastAttemptAt: to });
+        return Boolean(result.affected);
+    }
+    /**
+     * `base * 2^(attempt - 1)` with ±20 % jitter so a receiver that came back up
+     * is not hit by every queued delivery in the same second, then capped: the
+     * cap is a promise to the operator, so jitter may not exceed it.
+     */
+    readRetryDelay(attempt) {
+        const { maxRetryDelayMs, retryBaseDelayMs } = this.options.delivery;
+        const exponential = retryBaseDelayMs * 2 ** Math.min(Math.max(attempt - 1, 0), 30);
+        return Math.min(Math.round(exponential * (0.8 + Math.random() * 0.4)), maxRetryDelayMs);
+    }
+    /**
+     * Visible to everyone who can read the instance, so it carries the outcome
+     * and nothing else: no URL, no response body, no parameters.
+     */
+    createTerminalActivity(row, endpointLabel) {
+        return {
+            actorMemberId: null,
+            eventType: row.status === workflow_webhook_delivery_enums_1.WorkflowWebhookDeliveryStatusEnum.SENT
+                ? workflow_engine_enums_1.ActivityLogEventTypeEnum.SERVICE_TASK_EXECUTED
+                : workflow_engine_enums_1.ActivityLogEventTypeEnum.SERVICE_TASK_FAILED,
+            instanceId: row.instanceId,
+            nodeId: row.nodeId,
+            payload: {
+                action: 'NOTIFY_WEBHOOK',
+                attempts: row.attemptCount,
+                deliveryId: row.id,
+                endpointKey: row.endpointKey,
+                endpointLabel,
+                endpointVersion: row.endpointVersion,
+                errorCode: row.lastErrorCode,
+                status: row.lastResponseStatus,
+                targetId: row.targetId,
+            },
+            taskId: null,
+        };
+    }
+};
+exports.WorkflowWebhookDeliveryService = WorkflowWebhookDeliveryService;
+exports.WorkflowWebhookDeliveryService = WorkflowWebhookDeliveryService = WorkflowWebhookDeliveryService_1 = tslib_1.__decorate([
+    (0, common_1.Injectable)(),
+    tslib_1.__param(0, (0, typeorm_1.InjectRepository)(workflow_webhook_delivery_entity_1.WorkflowWebhookDeliveryEntity)),
+    tslib_1.__param(1, (0, typeorm_1.InjectRepository)(activity_log_entity_1.ActivityLogEntity)),
+    tslib_1.__param(3, (0, common_1.Optional)()),
+    tslib_1.__param(3, (0, common_1.Inject)(workflow_webhook_options_1.BPM_WORKFLOW_WEBHOOK_OPTIONS)),
+    tslib_1.__param(4, (0, common_1.Optional)()),
+    tslib_1.__param(4, (0, common_1.Inject)(exports.BPM_WORKFLOW_WEBHOOK_FETCH)),
+    tslib_1.__metadata("design:paramtypes", [typeorm_2.Repository,
+        typeorm_2.Repository,
+        workflow_webhook_service_1.WorkflowWebhookService, Object, Function])
+], WorkflowWebhookDeliveryService);
+function isWebhookRequest(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.url === 'string');
+}
+function buildHeaders(request, body, deliveryId, sentAt) {
+    const hostHeaders = Object.fromEntries(Object.entries(request.headers ?? {}).filter(([name, value]) => typeof value === 'string' && !name.toLowerCase().startsWith('x-bpm-')));
+    const hasContentType = Object.keys(hostHeaders).some((name) => name.toLowerCase() === 'content-type');
+    const timestamp = String(Math.floor(sentAt.getTime() / 1000));
+    // BPM's own headers are applied last and a host cannot supply an
+    // `x-bpm-*` header, so a receiver can trust them.
+    return {
+        ...(hasContentType ? {} : { 'content-type': 'application/json' }),
+        ...hostHeaders,
+        'x-bpm-delivery-id': deliveryId,
+        'x-bpm-event': 'workflow.notify',
+        ...(request.signingSecret
+            ? {
+                'x-bpm-signature-sha256': (0, node_crypto_1.createHmac)('sha256', request.signingSecret)
+                    .update(`${timestamp}.${body}`)
+                    .digest('hex'),
+                'x-bpm-timestamp': timestamp,
+            }
+            : {}),
+    };
+}
+/** Releases the connection without reading a body nobody will look at. */
+async function discardBody(response) {
+    await response.body?.cancel().catch(() => undefined);
+}
+/**
+ * Reads at most {@link MAX_ERROR_BODY_BYTES} of a failing response, so a
+ * receiver answering with megabytes of HTML cannot make BPM buffer them.
+ */
+async function readBodyPrefix(response) {
+    const reader = response.body?.getReader();
+    if (!reader) {
+        return response.text().catch(() => '');
+    }
+    const chunks = [];
+    let received = 0;
+    try {
+        while (received < MAX_ERROR_BODY_BYTES) {
+            const { done, value } = await reader.read();
+            if (done || !value) {
+                break;
+            }
+            chunks.push(value);
+            received += value.byteLength;
+        }
+    }
+    catch {
+        // A body that fails mid-read still says something; keep what arrived.
+    }
+    finally {
+        await reader.cancel().catch(() => undefined);
+    }
+    return new TextDecoder()
+        .decode(Buffer.concat(chunks).subarray(0, MAX_ERROR_BODY_BYTES))
+        .replace(/�+$/u, '');
+}
+function readErrorName(error) {
+    return error instanceof Error ? error.name : typeof error;
+}
+function readNetworkErrorCode(error) {
+    const cause = typeof error === 'object' && error !== null && 'cause' in error
+        ? error.cause
+        : undefined;
+    const code = typeof cause === 'object' && cause !== null && 'code' in cause
+        ? cause.code
+        : undefined;
+    return typeof code === 'string' ? code : readErrorName(error);
+}
+function isRetryableStatus(status) {
+    return status === 408 || status === 429 || status >= 500;
+}
+/**
+ * Makes a receiver-supplied string safe to store: PostgreSQL `text` rejects
+ * NUL, and a write that always fails would leave the row claimed and resent
+ * forever instead of counting towards `maxAttempts`.
+ */
+function truncate(value) {
+    const storable = value.split('\u0000').join('');
+    return storable.length > MAX_ERROR_DETAIL_LENGTH
+        ? storable.slice(0, MAX_ERROR_DETAIL_LENGTH)
+        : storable;
+}
+//# sourceMappingURL=workflow-webhook-delivery.service.js.map