@friggframework/core 2.0.0--canary.656.7302a64.0 → 2.0.0--canary.657.b14221f.0

package/CLAUDE.md

@@ -546,6 +546,61 @@ The single source of the max receive count is
 devtools integration builder uses it for the queue's `RedrivePolicy` and sets
 it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
 
+**Rate-limit deferral** (ADR-049): when a queue handler throws an error with
+`isRateLimited` and a `retryAt` (a `RateLimitError` from the Requester, or one a
+module throws), `Worker.run` does not give the message back to SQS for a 30
+minute redelivery that spends one of its three receives. It puts the message
+back so it runs at `retryAt`:
+
+| Wait | What the worker does | Record |
+|---|---|---|
+| Up to 900 s | Sends the same body again with `DelaySeconds`, then acknowledges the old message | handled |
+| Over 900 s, and `SCHEDULER_ROLE_ARN` is set | Creates a one-time EventBridge schedule at `retryAt` that sends the body to the queue, then acknowledges | handled |
+| Over 900 s, no scheduler | Extends the message's visibility timeout to `retryAt` (12 h at most). This uses one receive | reported failed |
+
+- The new message starts at receive count 1. A deferred body carries
+  `_frigg: { deferrals, firstDeferredAt }`; `_frigg` is reserved and the handler
+  never sees it.
+- Two caps bound it: `FRIGG_QUEUE_MAX_DEFERRALS` (default 10) and
+  `FRIGG_QUEUE_MAX_DEFERRED_MS` (default 24 h, from the first deferral to the
+  retry time). Past a cap, with no `retryAt`, with no event source ARN or on a
+  FIFO queue, the record fails like any retryable error and `delivery`
+  applies.
+- The mock scheduler is never used for a deferral: a schedule kept in memory
+  would be lost with the acknowledged message.
+- With a `processId` in the message, the worker writes
+  `Process.context.rateLimit = { status: 'WAITING' | 'EXHAUSTED', mechanism,
+  retryAt, reason, module, deferrals, updatedAt }`. `EXHAUSTED` means a cap ended
+  the deferrals, or a visibility change came on the last delivery. The value is
+  one per process, so it holds the latest known wait: a new wait replaces it only
+  when its `retryAt` is later, a `WAITING` value never hides an `EXHAUSTED` one,
+  and after a deferred or redelivered message ran the worker sets it to `null`
+  only when it is `WAITING` and its `retryAt` is past or within 5 s.
+- Delivery stays at least once. The new message is sent before the old one is
+  acknowledged, so a crash between the two can run the work twice.
+- Handlers check `delivery.isLastAttempt` first, then rethrow. The delay and
+  schedule tiers send a new message, so they never reach the last attempt. The
+  visibility tier and a cap use a receive: on the last one the worker writes an
+  ERROR (`frigg.worker.record_lost_rate_limited`), and the handler has this one
+  chance to end the run.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
+        throw error; // a RateLimitError too: core puts the message back at retryAt
+    }
+}
+```
+
+Records: `frigg.worker.record_deferred` (`mechanism` `delay` or `schedule`),
+`record_visibility_extended`, `record_deferral_capped`,
+`record_lost_rate_limited` and `rate_limit_state_failed`. A queue in a stack
+that does not own it (`ownership.queue: 'external'`) needs its own
+`sqs:ChangeMessageVisibility` and scheduler access.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.

package/core/CLAUDE.md

@@ -79,6 +79,25 @@ await worker.send({
 }, delaySeconds);
 ```
 
+**Rate-limit deferral** (ADR-049): `run()` handles an error with `isRateLimited`
+and a `retryAt` through `defer(record, body, error, delivery)`, which returns
+`{ outcome }`:
+- `acked`: the message was sent again with `DelaySeconds` (wait up to 900 s) or
+  scheduled with the one-time scheduler (longer wait). The record is handled.
+- `failed`: the visibility timeout was extended to `retryAt`, or a cap
+  (`FRIGG_QUEUE_MAX_DEFERRALS`, `FRIGG_QUEUE_MAX_DEFERRED_MS`) ended the
+  deferrals. The record is reported failed.
+- `skipped`: no `retryAt`, no `eventSourceARN`, a FIFO queue, or a failed send.
+  The record fails as any error does, and `record_failed` carries
+  `deferral: { skipped }`.
+
+Subclasses override two no-op hooks to keep run state: `recordRateLimitWait(body,
+error, state)` runs after the message was put back, and `clearRateLimitWait(body)`
+runs after a deferred or redelivered message succeeded. A failing hook logs
+`frigg.worker.rate_limit_state_failed` and never changes the outcome. The queue
+worker of an integration (`createQueueWorker`) writes `Process.context.rateLimit`
+in them. See `packages/core/CLAUDE.md` section 7.
+
 ### Delegate Pattern System (`Delegate.js:3-27`)
 
 **Purpose**: Observer/delegation pattern for decoupled component communication

package/core/Worker.js

@@ -1,14 +1,39 @@
-const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/client-sqs');
+const { randomUUID } = require('node:crypto');
+const {
+    SQSClient,
+    GetQueueUrlCommand,
+    SendMessageCommand,
+    ChangeMessageVisibilityCommand,
+} = require('@aws-sdk/client-sqs');
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
 const { readQueueDelivery } = require('../queues/queue-delivery');
+const { awsConfigOptions } = require('../queues/queuer-util');
+const {
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+} = require('../queues/queue-deferral');
 const { runMessageScope } = require('./invocation-scope');
-const { getLogger } = require('../logs');
+const { getLogger, serializeError } = require('../logs');
 
-const sqs = new SQSClient({ region: process.env.AWS_REGION });
+const sqs = new SQSClient({
+    region: process.env.AWS_REGION,
+    ...awsConfigOptions(),
+});
 const log = getLogger('frigg.worker');
 
+const validDate = (value) =>
+    value instanceof Date && !Number.isNaN(value.getTime()) ? value : null;
+const queueNameOf = (arn) =>
+    typeof arn === 'string' && arn ? arn.split(':').pop() : undefined;
+const causeOf = (error) => serializeError(error).message;
+
 class Worker {
     async getQueueURL(params) {
         // Passing params in because there will be multiple QueueNames
@@ -33,12 +58,31 @@ class Worker {
                 // messageId, receiveCount and the event come from the scope.
                 log.debug('Record started', { eventName: 'frigg.worker.record_started' });
 
+                const delivery = readQueueDelivery(record);
+                let runParams;
                 try {
-                    const runParams = JSON.parse(record.body);
+                    runParams = JSON.parse(record.body);
                     this._validateParams(runParams);
-                    await this._run(runParams, context, readQueueDelivery(record));
+                    await this._run(runParams, context, delivery);
+                    await this._clearDeferredState(runParams, delivery);
                     log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
                 } catch (error) {
+                    let deferral;
+                    if (error.isRateLimited && runParams) {
+                        deferral = await this.defer(
+                            record,
+                            runParams,
+                            error,
+                            delivery
+                        );
+                        if (deferral.outcome === 'acked') return;
+                        if (deferral.outcome === 'failed') {
+                            batchItemFailures.push({
+                                itemIdentifier: record.messageId,
+                            });
+                            return;
+                        }
+                    }
                     if (error.isHaltError) {
                         // HaltError means "discard this message, don't retry".
                         // Treat as success so SQS deletes it from the queue.
@@ -55,6 +99,14 @@ class Worker {
                     log.warn('Record failed, returned for retry', {
                         eventName: 'frigg.worker.record_failed',
                         error,
+                        ...(deferral && {
+                            deferral: {
+                                skipped: deferral.skipped,
+                                ...(deferral.cause && {
+                                    cause: causeOf(deferral.cause),
+                                }),
+                            },
+                        }),
                     });
                     batchItemFailures.push({ itemIdentifier: record.messageId });
                 }
@@ -75,6 +127,285 @@ class Worker {
         // parameters
     }
 
+    /**
+     * Puts a rate-limited message back so it runs at `error.retryAt`, without
+     * spending a receive. The message is sent again with a delay of up to 900 s,
+     * or scheduled with the one-time scheduler, or has its visibility timeout
+     * extended. The new message goes out before the old one is acknowledged, so
+     * delivery stays at least once.
+     *
+     * @returns {Promise<{outcome: 'acked'|'failed'|'skipped', skipped?: string, cause?: Error}>}
+     *   `acked`: the message is handled. `failed`: report the record failed and
+     *   let SQS redeliver it. `skipped`: nothing was done; fail the record as
+     *   any other error.
+     */
+    async defer(record, body, error, delivery) {
+        const retryAt = validDate(error.retryAt);
+        if (!retryAt) return { outcome: 'skipped', skipped: 'no_retry_at' };
+        const queueName = queueNameOf(record.eventSourceARN);
+        if (!queueName || queueName.endsWith('.fifo')) {
+            return { outcome: 'skipped', skipped: 'no_event_source' };
+        }
+
+        const now = Date.now();
+        const waitMs = Math.max(0, retryAt.getTime() - now);
+        const deferral = nextDeferral(body, now);
+        const details = {
+            waitMs,
+            retryAt: retryAt.toISOString(),
+            deferrals: deferral.deferrals,
+            firstDeferredAt: deferral.firstDeferredAt,
+            reason: error.reason,
+            module: error.module,
+            statusCode: error.statusCode,
+        };
+        const state = (status, mechanism) => ({
+            status,
+            mechanism,
+            deferrals: deferral.deferrals,
+            retryAt,
+        });
+
+        const limits = readDeferralLimits();
+        if (isDeferralCapped({ ...deferral, retryAt }, limits)) {
+            log.warn('Rate-limit deferral capped', {
+                eventName: 'frigg.worker.record_deferral_capped',
+                ...details,
+                maxDeferrals: limits.maxDeferrals,
+                maxDeferredMs: limits.maxDeferredMs,
+                deferredMs:
+                    retryAt.getTime() - Date.parse(deferral.firstDeferredAt),
+                error,
+            });
+            await this._recordWait(body, error, state('EXHAUSTED', 'none'));
+            this._logIfLost(delivery, error, details);
+            return { outcome: 'failed' };
+        }
+
+        const tier = {
+            record,
+            queueName,
+            body,
+            next: withDeferral(body, deferral),
+            error,
+            delivery,
+            retryAt,
+            waitMs,
+            details,
+            state,
+        };
+        if (waitMs <= MAX_DELAY_SECONDS * 1000)
+            return this._deferWithDelay(tier);
+        const scheduled = await this._deferWithSchedule(tier);
+        if (scheduled?.outcome) return scheduled;
+        return this._deferWithVisibility({
+            ...tier,
+            scheduleError: scheduled?.scheduleError,
+        });
+    }
+
+    async _deferWithDelay({
+        queueName,
+        body,
+        next,
+        error,
+        waitMs,
+        details,
+        state,
+    }) {
+        const delaySeconds = Math.ceil(waitMs / 1000);
+        try {
+            const queueUrl = await this._queueUrl(queueName);
+            await this.send({ ...next, QueueUrl: queueUrl }, delaySeconds);
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'send_failed', cause };
+        }
+        await this._recordWait(body, error, state('WAITING', 'delay'));
+        log.warn('Record deferred', {
+            eventName: 'frigg.worker.record_deferred',
+            mechanism: 'delay',
+            delaySeconds,
+            ...details,
+            error,
+        });
+        return { outcome: 'acked' };
+    }
+
+    /** `acked`, `{ scheduleError }` when the schedule failed, null with no scheduler. */
+    async _deferWithSchedule({
+        record,
+        body,
+        next,
+        error,
+        retryAt,
+        details,
+        state,
+    }) {
+        const scheduler = this.getSchedulerService();
+        if (!scheduler) return null;
+        const scheduleName = `frigg-defer-${record.messageId || randomUUID()}`;
+        try {
+            await scheduler.scheduleOneTime({
+                scheduleName,
+                scheduleAt: retryAt,
+                queueResourceId: record.eventSourceARN,
+                payload: next,
+            });
+        } catch (cause) {
+            if (cause?.name !== 'ConflictException')
+                return { scheduleError: cause };
+        }
+        await this._recordWait(body, error, state('WAITING', 'schedule'));
+        log.warn('Record deferred', {
+            eventName: 'frigg.worker.record_deferred',
+            mechanism: 'schedule',
+            scheduleName,
+            ...details,
+            error,
+        });
+        return { outcome: 'acked' };
+    }
+
+    async _deferWithVisibility({
+        record,
+        queueName,
+        body,
+        error,
+        delivery,
+        waitMs,
+        details,
+        state,
+        scheduleError,
+    }) {
+        const visibilityTimeout = Math.min(
+            Math.ceil(waitMs / 1000),
+            MAX_VISIBILITY_TIMEOUT_SECONDS
+        );
+        try {
+            const queueUrl = await this._queueUrl(queueName);
+            await sqs.send(
+                new ChangeMessageVisibilityCommand({
+                    QueueUrl: queueUrl,
+                    ReceiptHandle: record.receiptHandle,
+                    VisibilityTimeout: visibilityTimeout,
+                })
+            );
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'visibility_failed', cause };
+        }
+        await this._recordWait(
+            body,
+            error,
+            state(
+                delivery.isLastAttempt ? 'EXHAUSTED' : 'WAITING',
+                'visibility'
+            )
+        );
+        log.warn('Record visibility extended', {
+            eventName: 'frigg.worker.record_visibility_extended',
+            mechanism: 'visibility',
+            visibilityTimeout,
+            ...details,
+            ...(scheduleError && { scheduleError: causeOf(scheduleError) }),
+            error,
+        });
+        this._logIfLost(delivery, error, details);
+        return { outcome: 'failed' };
+    }
+
+    /**
+     * The one-time scheduler, or null when the stack has none. The mock
+     * scheduler is never used: a schedule kept in memory would be lost with
+     * the message.
+     */
+    getSchedulerService() {
+        if (this._schedulerService !== undefined) return this._schedulerService;
+        this._schedulerService = null;
+        if (
+            process.env.SCHEDULER_ROLE_ARN &&
+            process.env.SCHEDULER_PROVIDER !== 'mock'
+        ) {
+            const {
+                createSchedulerService,
+                SCHEDULER_PROVIDERS,
+            } = require('../infrastructure/scheduler/scheduler-service-factory');
+            this._schedulerService = createSchedulerService({
+                provider: SCHEDULER_PROVIDERS.EVENTBRIDGE,
+            });
+        }
+        return this._schedulerService;
+    }
+
+    /**
+     * Hook: the message was put back for a rate limit. Runs after it was sent.
+     * @param {Object} body The parsed message body.
+     * @param {Error} error The RateLimitError.
+     * @param {{status: 'WAITING'|'EXHAUSTED', mechanism: string, deferrals: number, retryAt: Date}} state
+     */
+    async recordRateLimitWait() {
+        // A subclass keeps the run state; nothing to do by default.
+    }
+
+    /**
+     * Hook: a message that was put back for a rate limit ran without error.
+     * @param {Object} body The parsed message body.
+     */
+    async clearRateLimitWait() {
+        // A subclass keeps the run state; nothing to do by default.
+    }
+
+    _queueUrl(queueName) {
+        this._queueUrls ??= new Map();
+        if (!this._queueUrls.has(queueName)) {
+            const lookup = this.getQueueURL({ QueueName: queueName }).catch(
+                (error) => {
+                    this._queueUrls.delete(queueName);
+                    throw error;
+                }
+            );
+            this._queueUrls.set(queueName, lookup);
+        }
+        return this._queueUrls.get(queueName);
+    }
+
+    async _recordWait(body, error, state) {
+        await this._runStateHook('record', () =>
+            this.recordRateLimitWait(body, error, state)
+        );
+    }
+
+    async _clearDeferredState(body, delivery) {
+        const redelivered = delivery.receiveCount > 1;
+        if (readDeferral(body).deferrals === 0 && !redelivered) return;
+        await this._runStateHook('clear', () => this.clearRateLimitWait(body));
+    }
+
+    async _runStateHook(operation, hook) {
+        try {
+            await hook();
+        } catch (error) {
+            log.warn('Rate-limit run state not written', {
+                eventName: 'frigg.worker.rate_limit_state_failed',
+                operation,
+                error,
+            });
+        }
+    }
+
+    _logIfLost(delivery, error, details) {
+        if (!delivery.isLastAttempt) return;
+        log.error(
+            'Rate-limited record lost: the message goes to the dead-letter queue next',
+            {
+                eventName: 'frigg.worker.record_lost_rate_limited',
+                retryAt: details.retryAt,
+                reason: error.reason,
+                module: error.module,
+                error,
+            }
+        );
+    }
+
     // returns the message id
     async send(params, delay = 0) {
         this._validateParams(params);

package/handlers/backend-utils.js

@@ -22,6 +22,15 @@ const {
 } = require('../integrations/utils/map-integration-dto');
 const { processNotFound } = require('../integrations/use-cases/process-errors');
 
+const CLEAR_RATE_LIMIT_TOLERANCE_MS = 5_000;
+
+const keepsCurrentRateLimit = (current, next) => {
+    if (!current) return false;
+    if (next.status === 'EXHAUSTED') return false;
+    if (current.status === 'EXHAUSTED') return true;
+    return Date.parse(current.retryAt) > next.retryAt.getTime();
+};
+
 const loadRouterFromObject = (IntegrationClass, routerObject) => {
     const router = Router();
     const { path, method, event } = routerObject;
@@ -165,6 +174,42 @@ const createQueueWorker = (integrationClass) => {
     const integrationName = integrationClass.Definition.name;
 
     class QueueWorker extends Worker {
+        async recordRateLimitWait(body, error, state) {
+            const processId = body.data?.processId;
+            if (!processId) return;
+            const processRepository = createProcessRepository();
+            const process = await processRepository.findById(processId);
+            if (keepsCurrentRateLimit(process?.context?.rateLimit, state))
+                return;
+            await processRepository.applyProcessUpdate(processId, {
+                set: {
+                    'context.rateLimit': {
+                        status: state.status,
+                        mechanism: state.mechanism,
+                        retryAt: state.retryAt.toISOString(),
+                        reason: error.reason,
+                        module: error.module,
+                        deferrals: state.deferrals,
+                        updatedAt: new Date().toISOString(),
+                    },
+                },
+            });
+        }
+
+        async clearRateLimitWait(body) {
+            const processId = body?.data?.processId;
+            if (!processId) return;
+            const processRepository = createProcessRepository();
+            const process = await processRepository.findById(processId);
+            const current = process?.context?.rateLimit;
+            if (current?.status !== 'WAITING') return;
+            const remainingMs = Date.parse(current.retryAt) - Date.now();
+            if (remainingMs > CLEAR_RATE_LIMIT_TOLERANCE_MS) return;
+            await processRepository.applyProcessUpdate(processId, {
+                set: { 'context.rateLimit': null },
+            });
+        }
+
         async _run(params, context, delivery) {
             const logCtx = {
                 integration: integrationName,

package/modules/requester/oauth-2.js

@@ -398,6 +398,12 @@ class OAuth2Requester extends Requester {
         );
         transportError.statusCode = status;
         transportError.isTokenRefreshTransportFailure = true;
+        if (error?.isRateLimited) {
+            transportError.isRateLimited = true;
+            transportError.retryAt = error.retryAt;
+            transportError.waitMs = error.waitMs;
+            transportError.reason = error.reason;
+        }
         return transportError;
     }
 

package/package.json

@@ -1,12 +1,13 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.656.7302a64.0",
+    "version": "2.0.0--canary.657.b14221f.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
         "@aws-sdk/client-lambda": "^3.714.0",
         "@aws-sdk/client-s3": "^3.588.0",
+        "@aws-sdk/client-scheduler": "^3.588.0",
         "@aws-sdk/client-sqs": "^3.588.0",
         "@aws-sdk/client-ssm": "^3.588.0",
         "@aws-sdk/s3-request-presigner": "^3.588.0",
@@ -47,9 +48,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.656.7302a64.0",
-        "@friggframework/prettier-config": "2.0.0--canary.656.7302a64.0",
-        "@friggframework/test": "2.0.0--canary.656.7302a64.0",
+        "@friggframework/eslint-config": "2.0.0--canary.657.b14221f.0",
+        "@friggframework/prettier-config": "2.0.0--canary.657.b14221f.0",
+        "@friggframework/test": "2.0.0--canary.657.b14221f.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -89,5 +90,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "7302a64c61c7eb20e0b42a972c0406f75b1f347b"
+    "gitHead": "b14221fca51737284a61407b42e003637930465e"
 }

package/queues/queue-deferral.js

@@ -0,0 +1,79 @@
+const MAX_DEFERRALS = 10;
+const MAX_DEFERRED_MS = 24 * 60 * 60 * 1000;
+const MAX_DELAY_SECONDS = 900;
+const MAX_VISIBILITY_TIMEOUT_SECONDS = 43_200;
+const MAX_DEFERRALS_ENV = 'FRIGG_QUEUE_MAX_DEFERRALS';
+const MAX_DEFERRED_MS_ENV = 'FRIGG_QUEUE_MAX_DEFERRED_MS';
+
+const toPositiveInteger = (value) => {
+    const number = Number(value);
+    return Number.isInteger(number) && number > 0 ? number : undefined;
+};
+
+/**
+ * @typedef {Object} DeferralLimits
+ * @property {number} maxDeferrals Times one message may be put back for a rate limit.
+ * @property {number} maxDeferredMs Longest total time from the first deferral to the last retry time.
+ */
+
+/**
+ * @param {Object} [env=process.env]
+ * @returns {DeferralLimits}
+ */
+const readDeferralLimits = (env = process.env) => ({
+    maxDeferrals: toPositiveInteger(env[MAX_DEFERRALS_ENV]) ?? MAX_DEFERRALS,
+    maxDeferredMs:
+        toPositiveInteger(env[MAX_DEFERRED_MS_ENV]) ?? MAX_DEFERRED_MS,
+});
+
+/**
+ * The deferral counters a message body carries in `_frigg`.
+ * @returns {{deferrals: number, firstDeferredAt: string|undefined}}
+ */
+const readDeferral = (body) => {
+    const frigg = body?._frigg;
+    const deferrals =
+        Number.isInteger(frigg?.deferrals) && frigg.deferrals > 0
+            ? frigg.deferrals
+            : 0;
+    const firstDeferredAt =
+        typeof frigg?.firstDeferredAt === 'string' &&
+        !Number.isNaN(Date.parse(frigg.firstDeferredAt))
+            ? frigg.firstDeferredAt
+            : undefined;
+    return { deferrals, firstDeferredAt };
+};
+
+const nextDeferral = (body, now = Date.now()) => {
+    const current = readDeferral(body);
+    return {
+        deferrals: current.deferrals + 1,
+        firstDeferredAt: current.firstDeferredAt ?? new Date(now).toISOString(),
+    };
+};
+
+const withDeferral = (body, deferral) => ({
+    ...body,
+    _frigg: { ...body._frigg, ...deferral },
+});
+
+const isDeferralCapped = (
+    { deferrals, firstDeferredAt, retryAt },
+    { maxDeferrals, maxDeferredMs }
+) =>
+    deferrals > maxDeferrals ||
+    retryAt.getTime() - Date.parse(firstDeferredAt) > maxDeferredMs;
+
+module.exports = {
+    MAX_DEFERRALS,
+    MAX_DEFERRALS_ENV,
+    MAX_DEFERRED_MS,
+    MAX_DEFERRED_MS_ENV,
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+};

package/queues/queuer-util.js

@@ -126,4 +126,4 @@ const QueuerUtil = {
     },
 };
 
-module.exports = { QueuerUtil };
+module.exports = { QueuerUtil, awsConfigOptions };

package/types/core/index.d.ts

@@ -47,8 +47,28 @@ declare module "@friggframework/core" {
     send(params: object & { QueueUrl: any }, delay?: number): Promise<string>;
 
     sendAsyncSQSMessage(params: SendSQSMessageParams): Promise<string>;
+
+    /**
+     * Hook: a message was put back for a rate limit. Runs after the message
+     * was sent. A failure is logged and never changes the outcome.
+     */
+    recordRateLimitWait(
+      body: object,
+      error: Error,
+      state: RateLimitWaitState
+    ): Promise<void>;
+
+    /** Hook: a message that was put back for a rate limit ran without error. */
+    clearRateLimitWait(body: object): Promise<void>;
   }
 
+  export type RateLimitWaitState = {
+    status: "WAITING" | "EXHAUSTED";
+    mechanism: "delay" | "schedule" | "visibility" | "none";
+    deferrals: number;
+    retryAt: Date;
+  };
+
   interface IWorker {
     getQueueURL(params: GetQueueURLParams): Promise<string | undefined>;
     run(params: { Records: any }, context?: object): Promise<BatchItemFailuresResponse>;