@friggframework/core 2.0.0--canary.656.10f1676.0 → 2.0.0--canary.657.8f335eb.0

package/CLAUDE.md

@@ -381,7 +381,10 @@ in this order:
 | 4 | The fixed `backOff` ladder (1, 3, 10, 30, 60, 180 s) |
 
 - A 429 with no hint from steps 1 to 3 keeps today's ladder: the same calls,
-  the same delays, then a plain `FetchError`. Nothing budgets it.
+  the same delays, then a plain `FetchError`. Nothing budgets it. A response
+  that `classify` names as a limit but that has no time follows the same
+  ladder, and the last error is a `FetchError` flagged `isRateLimited` with
+  the `reason`, so the queue worker does not halt it.
 - With a hint, the wait is `max(hint, minRetryAfterMs, 1 s)` plus at most 10 %
   jitter. The total sleep of one request is capped at `maxInProcessWaitMs`
   (default 5 minutes) and at the time left in the invocation less one request
@@ -543,6 +546,58 @@ The single source of the max receive count is
 devtools integration builder uses it for the queue's `RedrivePolicy` and sets
 it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
 
+**Rate-limit deferral** (ADR-049): when a queue handler throws an error with
+`isRateLimited` and a `retryAt` (a `RateLimitError` from the Requester, or one a
+module throws), `Worker.run` does not give the message back to SQS for a 30
+minute redelivery that spends one of its three receives. It puts the message
+back so it runs at `retryAt`:
+
+| Wait | What the worker does | Record |
+|---|---|---|
+| Up to 900 s | Sends the same body again with `DelaySeconds`, then acknowledges the old message | handled |
+| Over 900 s, and `SCHEDULER_ROLE_ARN` is set | Creates a one-time EventBridge schedule at `retryAt` that sends the body to the queue, then acknowledges | handled |
+| Over 900 s, no scheduler | Extends the message's visibility timeout to `retryAt` (12 h at most). This uses one receive | reported failed |
+
+- The new message starts at receive count 1. A deferred body carries
+  `_frigg: { deferrals, firstDeferredAt }`; `_frigg` is reserved and the handler
+  never sees it.
+- Two caps bound it: `FRIGG_QUEUE_MAX_DEFERRALS` (default 10) and
+  `FRIGG_QUEUE_MAX_DEFERRED_MS` (default 24 h, from the first deferral to the
+  retry time). Past a cap, with no `retryAt`, with no event source ARN or on a
+  FIFO queue, the record fails like any retryable error and `delivery`
+  applies.
+- The mock scheduler is never used for a deferral: a schedule kept in memory
+  would be lost with the acknowledged message.
+- With a `processId` in the message, the worker writes
+  `Process.context.rateLimit = { status: 'WAITING' | 'EXHAUSTED', mechanism,
+  retryAt, reason, module, deferrals, updatedAt }`, and `null` after the message
+  ran. `EXHAUSTED` means a cap ended the deferrals, or a visibility change came
+  on the last delivery.
+- Delivery stays at least once. The new message is sent before the old one is
+  acknowledged, so a crash between the two can run the work twice.
+- Handlers check `delivery.isLastAttempt` first, then rethrow. The delay and
+  schedule tiers send a new message, so they never reach the last attempt. The
+  visibility tier and a cap use a receive: on the last one the worker writes an
+  ERROR (`frigg.worker.record_lost_rate_limited`), and the handler has this one
+  chance to end the run.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
+        throw error; // a RateLimitError too: core puts the message back at retryAt
+    }
+}
+```
+
+Records: `frigg.worker.record_deferred` (`mechanism` `delay` or `schedule`),
+`record_visibility_extended`, `record_deferral_capped`,
+`record_lost_rate_limited` and `rate_limit_state_failed`. A queue in a stack
+that does not own it (`ownership.queue: 'external'`) needs its own
+`sqs:ChangeMessageVisibility` and scheduler access.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.

package/core/CLAUDE.md

@@ -79,6 +79,25 @@ await worker.send({
 }, delaySeconds);
 ```
 
+**Rate-limit deferral** (ADR-049): `run()` handles an error with `isRateLimited`
+and a `retryAt` through `defer(record, body, error, delivery)`, which returns
+`{ outcome }`:
+- `acked`: the message was sent again with `DelaySeconds` (wait up to 900 s) or
+  scheduled with the one-time scheduler (longer wait). The record is handled.
+- `failed`: the visibility timeout was extended to `retryAt`, or a cap
+  (`FRIGG_QUEUE_MAX_DEFERRALS`, `FRIGG_QUEUE_MAX_DEFERRED_MS`) ended the
+  deferrals. The record is reported failed.
+- `skipped`: no `retryAt`, no `eventSourceARN`, a FIFO queue, or a failed send.
+  The record fails as any error does, and `record_failed` carries
+  `deferral: { skipped }`.
+
+Subclasses override two no-op hooks to keep run state: `recordRateLimitWait(body,
+error, state)` runs after the message was put back, and `clearRateLimitWait(body)`
+runs after a deferred or redelivered message succeeded. A failing hook logs
+`frigg.worker.rate_limit_state_failed` and never changes the outcome. The queue
+worker of an integration (`createQueueWorker`) writes `Process.context.rateLimit`
+in them. See `packages/core/CLAUDE.md` section 7.
+
 ### Delegate Pattern System (`Delegate.js:3-27`)
 
 **Purpose**: Observer/delegation pattern for decoupled component communication

package/core/Worker.js

@@ -1,14 +1,39 @@
-const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/client-sqs');
+const { randomUUID } = require('node:crypto');
+const {
+    SQSClient,
+    GetQueueUrlCommand,
+    SendMessageCommand,
+    ChangeMessageVisibilityCommand,
+} = require('@aws-sdk/client-sqs');
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
 const { readQueueDelivery } = require('../queues/queue-delivery');
+const { awsConfigOptions } = require('../queues/queuer-util');
+const {
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+} = require('../queues/queue-deferral');
 const { runMessageScope } = require('./invocation-scope');
-const { getLogger } = require('../logs');
+const { getLogger, serializeError } = require('../logs');
 
-const sqs = new SQSClient({ region: process.env.AWS_REGION });
+const sqs = new SQSClient({
+    region: process.env.AWS_REGION,
+    ...awsConfigOptions(),
+});
 const log = getLogger('frigg.worker');
 
+const validDate = (value) =>
+    value instanceof Date && !Number.isNaN(value.getTime()) ? value : null;
+const queueNameOf = (arn) =>
+    typeof arn === 'string' && arn ? arn.split(':').pop() : undefined;
+const causeOf = (error) => serializeError(error).message;
+
 class Worker {
     async getQueueURL(params) {
         // Passing params in because there will be multiple QueueNames
@@ -33,12 +58,31 @@ class Worker {
                 // messageId, receiveCount and the event come from the scope.
                 log.debug('Record started', { eventName: 'frigg.worker.record_started' });
 
+                const delivery = readQueueDelivery(record);
+                let runParams;
                 try {
-                    const runParams = JSON.parse(record.body);
+                    runParams = JSON.parse(record.body);
                     this._validateParams(runParams);
-                    await this._run(runParams, context, readQueueDelivery(record));
+                    await this._run(runParams, context, delivery);
+                    await this._clearDeferredState(runParams, delivery);
                     log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
                 } catch (error) {
+                    let deferral;
+                    if (error.isRateLimited && runParams) {
+                        deferral = await this.defer(
+                            record,
+                            runParams,
+                            error,
+                            delivery
+                        );
+                        if (deferral.outcome === 'acked') return;
+                        if (deferral.outcome === 'failed') {
+                            batchItemFailures.push({
+                                itemIdentifier: record.messageId,
+                            });
+                            return;
+                        }
+                    }
                     if (error.isHaltError) {
                         // HaltError means "discard this message, don't retry".
                         // Treat as success so SQS deletes it from the queue.
@@ -55,6 +99,14 @@ class Worker {
                     log.warn('Record failed, returned for retry', {
                         eventName: 'frigg.worker.record_failed',
                         error,
+                        ...(deferral && {
+                            deferral: {
+                                skipped: deferral.skipped,
+                                ...(deferral.cause && {
+                                    cause: causeOf(deferral.cause),
+                                }),
+                            },
+                        }),
                     });
                     batchItemFailures.push({ itemIdentifier: record.messageId });
                 }
@@ -75,6 +127,240 @@ class Worker {
         // parameters
     }
 
+    /**
+     * Puts a rate-limited message back so it runs at `error.retryAt`, without
+     * spending a receive. The message is sent again with a delay of up to 900 s,
+     * or scheduled with the one-time scheduler, or has its visibility timeout
+     * extended. The new message goes out before the old one is acknowledged, so
+     * delivery stays at least once.
+     *
+     * @returns {Promise<{outcome: 'acked'|'failed'|'skipped', skipped?: string, cause?: Error}>}
+     *   `acked`: the message is handled. `failed`: report the record failed and
+     *   let SQS redeliver it. `skipped`: nothing was done; fail the record as
+     *   any other error.
+     */
+    async defer(record, body, error, delivery) {
+        const retryAt = validDate(error.retryAt);
+        if (!retryAt) return { outcome: 'skipped', skipped: 'no_retry_at' };
+        const queueName = queueNameOf(record.eventSourceARN);
+        if (!queueName || queueName.endsWith('.fifo')) {
+            return { outcome: 'skipped', skipped: 'no_event_source' };
+        }
+
+        const now = Date.now();
+        const waitMs = Math.max(0, retryAt.getTime() - now);
+        const deferral = nextDeferral(body, now);
+        const details = {
+            waitMs,
+            retryAt: retryAt.toISOString(),
+            deferrals: deferral.deferrals,
+            firstDeferredAt: deferral.firstDeferredAt,
+            reason: error.reason,
+            module: error.module,
+            statusCode: error.statusCode,
+        };
+        const state = (status, mechanism) => ({
+            status,
+            mechanism,
+            deferrals: deferral.deferrals,
+            retryAt,
+        });
+
+        const limits = readDeferralLimits();
+        if (isDeferralCapped({ ...deferral, retryAt }, limits)) {
+            log.warn('Rate-limit deferral capped', {
+                eventName: 'frigg.worker.record_deferral_capped',
+                ...details,
+                maxDeferrals: limits.maxDeferrals,
+                maxDeferredMs: limits.maxDeferredMs,
+                deferredMs:
+                    retryAt.getTime() - Date.parse(deferral.firstDeferredAt),
+                error,
+            });
+            await this._recordWait(body, error, state('EXHAUSTED', 'none'));
+            this._logIfLost(delivery, error, details);
+            return { outcome: 'failed' };
+        }
+
+        const next = withDeferral(body, deferral);
+
+        if (waitMs <= MAX_DELAY_SECONDS * 1000) {
+            const delaySeconds = Math.ceil(waitMs / 1000);
+            try {
+                const queueUrl = await this._queueUrl(queueName);
+                await this.send({ ...next, QueueUrl: queueUrl }, delaySeconds);
+            } catch (cause) {
+                return { outcome: 'skipped', skipped: 'send_failed', cause };
+            }
+            await this._recordWait(body, error, state('WAITING', 'delay'));
+            log.warn('Record deferred', {
+                eventName: 'frigg.worker.record_deferred',
+                mechanism: 'delay',
+                delaySeconds,
+                ...details,
+                error,
+            });
+            return { outcome: 'acked' };
+        }
+
+        let scheduleError;
+        const scheduler = this.getSchedulerService();
+        if (scheduler) {
+            const scheduleName = `frigg-defer-${
+                record.messageId || randomUUID()
+            }`;
+            try {
+                await scheduler.scheduleOneTime({
+                    scheduleName,
+                    scheduleAt: retryAt,
+                    queueResourceId: record.eventSourceARN,
+                    payload: next,
+                });
+            } catch (cause) {
+                if (cause?.name !== 'ConflictException') scheduleError = cause;
+            }
+            if (!scheduleError) {
+                await this._recordWait(
+                    body,
+                    error,
+                    state('WAITING', 'schedule')
+                );
+                log.warn('Record deferred', {
+                    eventName: 'frigg.worker.record_deferred',
+                    mechanism: 'schedule',
+                    scheduleName,
+                    ...details,
+                    error,
+                });
+                return { outcome: 'acked' };
+            }
+        }
+
+        const visibilityTimeout = Math.min(
+            Math.ceil(waitMs / 1000),
+            MAX_VISIBILITY_TIMEOUT_SECONDS
+        );
+        try {
+            const queueUrl = await this._queueUrl(queueName);
+            await sqs.send(
+                new ChangeMessageVisibilityCommand({
+                    QueueUrl: queueUrl,
+                    ReceiptHandle: record.receiptHandle,
+                    VisibilityTimeout: visibilityTimeout,
+                })
+            );
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'visibility_failed', cause };
+        }
+        await this._recordWait(
+            body,
+            error,
+            state(
+                delivery.isLastAttempt ? 'EXHAUSTED' : 'WAITING',
+                'visibility'
+            )
+        );
+        log.warn('Record visibility extended', {
+            eventName: 'frigg.worker.record_visibility_extended',
+            mechanism: 'visibility',
+            visibilityTimeout,
+            ...details,
+            ...(scheduleError && { scheduleError: causeOf(scheduleError) }),
+            error,
+        });
+        this._logIfLost(delivery, error, details);
+        return { outcome: 'failed' };
+    }
+
+    /**
+     * The one-time scheduler, or null when the stack has none. The mock
+     * scheduler is never used: a schedule kept in memory would be lost with
+     * the message.
+     */
+    getSchedulerService() {
+        if (this._schedulerService !== undefined) return this._schedulerService;
+        this._schedulerService = null;
+        if (
+            process.env.SCHEDULER_ROLE_ARN &&
+            process.env.SCHEDULER_PROVIDER !== 'mock'
+        ) {
+            const {
+                createSchedulerService,
+                SCHEDULER_PROVIDERS,
+            } = require('../infrastructure/scheduler/scheduler-service-factory');
+            this._schedulerService = createSchedulerService({
+                provider: SCHEDULER_PROVIDERS.EVENTBRIDGE,
+            });
+        }
+        return this._schedulerService;
+    }
+
+    /**
+     * Hook: the message was put back for a rate limit. Runs after it was sent.
+     * @param {Object} body The parsed message body.
+     * @param {Error} error The RateLimitError.
+     * @param {{status: 'WAITING'|'EXHAUSTED', mechanism: string, deferrals: number, retryAt: Date}} state
+     */
+    async recordRateLimitWait() {}
+
+    /**
+     * Hook: a message that was put back for a rate limit ran without error.
+     * @param {Object} body The parsed message body.
+     */
+    async clearRateLimitWait() {}
+
+    _queueUrl(queueName) {
+        this._queueUrls ??= new Map();
+        if (!this._queueUrls.has(queueName)) {
+            const lookup = this.getQueueURL({ QueueName: queueName }).catch(
+                (error) => {
+                    this._queueUrls.delete(queueName);
+                    throw error;
+                }
+            );
+            this._queueUrls.set(queueName, lookup);
+        }
+        return this._queueUrls.get(queueName);
+    }
+
+    async _recordWait(body, error, state) {
+        await this._runStateHook('record', () =>
+            this.recordRateLimitWait(body, error, state)
+        );
+    }
+
+    async _clearDeferredState(body, delivery) {
+        if (readDeferral(body).deferrals === 0 && !(delivery.receiveCount > 1))
+            return;
+        await this._runStateHook('clear', () => this.clearRateLimitWait(body));
+    }
+
+    async _runStateHook(operation, hook) {
+        try {
+            await hook();
+        } catch (error) {
+            log.warn('Rate-limit run state not written', {
+                eventName: 'frigg.worker.rate_limit_state_failed',
+                operation,
+                error,
+            });
+        }
+    }
+
+    _logIfLost(delivery, error, details) {
+        if (!delivery.isLastAttempt) return;
+        log.error(
+            'Rate-limited record lost: the message goes to the dead-letter queue next',
+            {
+                eventName: 'frigg.worker.record_lost_rate_limited',
+                retryAt: details.retryAt,
+                reason: error.reason,
+                module: error.module,
+                error,
+            }
+        );
+    }
+
     // returns the message id
     async send(params, delay = 0) {
         this._validateParams(params);

package/errors/fetch-error.js

@@ -58,7 +58,7 @@ class FetchError extends BaseError {
         const provided = options.responseBody ?? options.body;
         let responseBody = provided;
         if (
-            !responseBody &&
+            responseBody === undefined &&
             response &&
             !response.bodyUsed &&
             typeof response.text === 'function'

package/handlers/backend-utils.js

@@ -165,6 +165,32 @@ const createQueueWorker = (integrationClass) => {
     const integrationName = integrationClass.Definition.name;
 
     class QueueWorker extends Worker {
+        async recordRateLimitWait(body, error, state) {
+            const processId = body.data?.processId;
+            if (!processId) return;
+            await createProcessRepository().applyProcessUpdate(processId, {
+                set: {
+                    'context.rateLimit': {
+                        status: state.status,
+                        mechanism: state.mechanism,
+                        retryAt: state.retryAt.toISOString(),
+                        reason: error.reason,
+                        module: error.module,
+                        deferrals: state.deferrals,
+                        updatedAt: new Date().toISOString(),
+                    },
+                },
+            });
+        }
+
+        async clearRateLimitWait(body) {
+            const processId = body?.data?.processId;
+            if (!processId) return;
+            await createProcessRepository().applyProcessUpdate(processId, {
+                set: { 'context.rateLimit': null },
+            });
+        }
+
         async _run(params, context, delivery) {
             const logCtx = {
                 integration: integrationName,

package/modules/requester/oauth-2.js

@@ -398,6 +398,12 @@ class OAuth2Requester extends Requester {
         );
         transportError.statusCode = status;
         transportError.isTokenRefreshTransportFailure = true;
+        if (error?.isRateLimited) {
+            transportError.isRateLimited = true;
+            transportError.retryAt = error.retryAt;
+            transportError.waitMs = error.waitMs;
+            transportError.reason = error.reason;
+        }
         return transportError;
     }
 

package/modules/requester/rate-limit/parsers.js

@@ -90,7 +90,7 @@ function hintFromWait(waitMs, now, { source = 'header', ...extra } = {}) {
  * Builds a hint from an absolute time. A time in the past waits 0 ms.
  */
 function hintFromRetryAt(retryAt, now, { source = 'header', ...extra } = {}) {
-    const at = retryAt instanceof Date ? retryAt.getTime() : NaN;
+    const at = retryAt instanceof Date ? retryAt.getTime() : Number.NaN;
     if (Number.isNaN(at)) return null;
     const waitMs = Math.max(0, at - now);
     if (waitMs > MAX_HINT_WAIT_MS) return null;
@@ -216,7 +216,7 @@ function pickStructured(items) {
 
 function readKeyValue(text, key) {
     const match = new RegExp(
-        `(?:^|[,;\\s])${key}\\s*=\\s*([^,;\\s]+)`,
+        String.raw`(?:^|[,;\s])${key}\s*=\s*([^,;\s]+)`,
         'i'
     ).exec(text);
     return match ? match[1] : undefined;

package/modules/requester/rate-limit/policy.js

@@ -243,8 +243,10 @@ function classifyRateLimit(policy, signal, options = {}) {
 }
 
 /**
- * Like classifyRateLimit, but always answers: with no hint it returns the
- * step of the fixed backoff ladder, with source "backoff".
+ * Like classifyRateLimit, but a throttled response always gets a hint: with
+ * none found it gets the step of the fixed backoff ladder, with source
+ * "backoff". Null when the response is not throttled: a status other than 429
+ * that classify did not recognise.
  */
 function resolveRateLimitHint(policy, signal, options = {}) {
     const {
@@ -258,6 +260,7 @@ function resolveRateLimitHint(policy, signal, options = {}) {
         onClassifyError,
     });
     if (hint) return hint;
+    if (signal.status !== 429 && !classified) return null;
 
     const waitMs =
         attempt < backOff.length ? Number(backOff[attempt]) * 1000 || 0 : 0;

package/modules/requester/requester.js

@@ -10,7 +10,6 @@ const { redactUrl } = require('../../logs/redact');
 const { toSanitizedSurrogate } = require('../../logs/serialize');
 const { getLoggerScope } = require('../../logs/context');
 const {
-    classifyRateLimit,
     computeScopeKey,
     computeWaitMs,
     headerValue,
@@ -20,7 +19,18 @@ const {
 } = require('./rate-limit');
 
 const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
-const JSON_CONTENT_TYPE = /^application\/(json|vnd\.api\+json|hal\+json)/;
+
+function isJsonMediaType(contentType) {
+    const mediaType = String(contentType ?? '')
+        .split(';')[0]
+        .trim()
+        .toLowerCase();
+    return (
+        mediaType === 'application/json' ||
+        mediaType === 'text/json' ||
+        mediaType.endsWith('+json')
+    );
+}
 
 // A node-fetch error message holds the raw URL, and util.inspect prints the
 // cause chain, so the FetchError keeps only a sanitized copy.
@@ -46,6 +56,7 @@ class Requester extends Delegate {
         super(params);
         this.backOff = get(params, 'backOff', [1, 3, 10, 30, 60, 180]);
         this._rateLimitPolicy = readRateLimitPolicy(this.constructor);
+        this._random = params?.random ?? Math.random;
         this.isRefreshable = false;
         this.refreshCount = 0;
         this.authGraceRetryCount = 0;
@@ -294,7 +305,12 @@ class Requester extends Delegate {
                     clearRequestTimer();
                     const delay = this.backOff[attempt] * 1000;
                     await new Promise((resolve) => setTimeout(resolve, delay));
-                    return this._rawRequest(url, options, attempt + 1);
+                    return this._rawRequest(
+                        url,
+                        options,
+                        attempt + 1,
+                        waitedMs
+                    );
                 }
                 const fetchError = await FetchError.create({
                     resource: encodedUrl,
@@ -318,70 +334,27 @@ class Requester extends Delegate {
                 status,
                 attempt
             );
-            if (throttle?.throttled) {
-                const { hint } = throttle;
-                if (hint.source === 'backoff') {
-                    if (attempt < this.backOff.length) {
-                        clearRequestTimer();
-                        const delay = this.backOff[attempt] * 1000;
-                        await new Promise((resolve) =>
-                            setTimeout(resolve, delay)
-                        );
-                        return this._rawRequest(
-                            url,
-                            options,
-                            attempt + 1,
-                            waitedMs
-                        );
-                    }
-                } else {
-                    const budgetMs = inProcessBudgetMs({
-                        policy: this._rateLimitPolicy,
-                        requestTimeoutMs: this.requestTimeoutMs,
-                        remainingMs: remainingInvocationMs(),
-                        waitedMs,
-                    });
-                    const { waitMs, fits } = computeWaitMs({
-                        hint,
-                        policy: this._rateLimitPolicy,
-                        budgetMs,
-                    });
-                    this._logRateLimited({
-                        status,
-                        hint,
-                        waitMs,
-                        attempt,
-                        waitedMs,
-                        action: fits ? 'wait' : 'throw',
-                    });
-                    if (fits) {
-                        clearRequestTimer();
-                        await new Promise((resolve) =>
-                            setTimeout(resolve, waitMs)
-                        );
-                        return this._rawRequest(
-                            url,
-                            options,
-                            attempt + 1,
-                            waitedMs + waitMs
-                        );
-                    }
-                    this._logRequestFailed(encodedUrl, options, status);
-                    const rateLimitError = await RateLimitError.create({
-                        resource: encodedUrl,
-                        init: options,
-                        response,
-                        responseBody: throttle.responseBody,
-                        hint,
-                        waitMs,
-                        module: this._telemetryModuleLabel(),
-                        scopeKey: computeScopeKey(this._rateLimitPolicy, this),
-                    });
-                    throw this._maybeFlagTimeoutDuringBodyRead(
-                        rateLimitError,
-                        timeoutMs
-                    );
-                }
+            const throttleRetry = await this._throttleRetry({
+                throttle,
+                status,
+                attempt,
+                waitedMs,
+                encodedUrl,
+                options,
+                response,
+                timeoutMs,
+            });
+            if (throttleRetry) {
+                clearRequestTimer();
+                await new Promise((resolve) =>
+                    setTimeout(resolve, throttleRetry.delayMs)
+                );
+                return this._rawRequest(
+                    url,
+                    options,
+                    attempt + 1,
+                    waitedMs + throttleRetry.hintedMs
+                );
             }
 
             // If the status is retriable and there are back off requests left, retry the request
@@ -389,7 +362,7 @@ class Requester extends Delegate {
                 clearRequestTimer();
                 const delay = this.backOff[attempt] * 1000;
                 await new Promise((resolve) => setTimeout(resolve, delay));
-                return this._rawRequest(url, options, attempt + 1);
+                return this._rawRequest(url, options, attempt + 1, waitedMs);
             }
 
             if (status === 401) {
@@ -410,7 +383,12 @@ class Requester extends Delegate {
                     // This request did not try the current token. Retry with
                     // it. Do not spend one more provider-side rotation.
                     clearRequestTimer();
-                    return this._rawRequest(url, options, attempt + 1);
+                    return this._rawRequest(
+                        url,
+                        options,
+                        attempt + 1,
+                        waitedMs
+                    );
                 }
 
                 if (!this.isRefreshable) {
@@ -427,7 +405,12 @@ class Requester extends Delegate {
                         await new Promise((resolve) =>
                             setTimeout(resolve, delay)
                         );
-                        return this._rawRequest(url, options, attempt + 1);
+                        return this._rawRequest(
+                            url,
+                            options,
+                            attempt + 1,
+                            waitedMs
+                        );
                     }
 
                     throw await this._invalidateAuth(
@@ -460,7 +443,12 @@ class Requester extends Delegate {
                 const refreshSucceeded = await this._refreshAuthOnce();
                 if (refreshSucceeded) {
                     clearRequestTimer();
-                    return this._rawRequest(url, options, attempt + 1);
+                    return this._rawRequest(
+                        url,
+                        options,
+                        attempt + 1,
+                        waitedMs
+                    );
                 }
 
                 throw await this._invalidateAuth(encodedUrl, options, response);
@@ -477,6 +465,10 @@ class Requester extends Delegate {
                     response,
                     responseBody: throttle?.responseBody,
                 });
+                if (throttle?.throttled && status !== 429) {
+                    fetchError.isRateLimited = true;
+                    fetchError.reason = throttle.hint.reason;
+                }
                 throw this._maybeFlagTimeoutDuringBodyRead(
                     fetchError,
                     timeoutMs
@@ -505,6 +497,69 @@ class Requester extends Delegate {
         }
     }
 
+    /**
+     * What to do about a throttled response: the delay before the next
+     * attempt, or null when there is no attempt left. A wait that a hint set
+     * and that does not fit the budget throws RateLimitError.
+     *
+     * @returns {Promise<{delayMs: number, hintedMs: number}|null>}
+     *   `hintedMs` is the part of the delay that a provider or a policy set,
+     *   which counts against the in-process cap.
+     */
+    async _throttleRetry({
+        throttle,
+        status,
+        attempt,
+        waitedMs,
+        encodedUrl,
+        options,
+        response,
+        timeoutMs,
+    }) {
+        if (!throttle?.throttled) return null;
+        const { hint } = throttle;
+        if (hint.source === 'backoff') {
+            return attempt < this.backOff.length
+                ? { delayMs: this.backOff[attempt] * 1000, hintedMs: 0 }
+                : null;
+        }
+
+        const budgetMs = inProcessBudgetMs({
+            policy: this._rateLimitPolicy,
+            requestTimeoutMs: this.requestTimeoutMs,
+            remainingMs: remainingInvocationMs(),
+            waitedMs,
+        });
+        const { waitMs, fits } = computeWaitMs({
+            hint,
+            policy: this._rateLimitPolicy,
+            budgetMs,
+            random: this._random,
+        });
+        this._logRateLimited({
+            status,
+            hint,
+            waitMs,
+            attempt,
+            waitedMs,
+            action: fits ? 'wait' : 'throw',
+        });
+        if (fits) return { delayMs: waitMs, hintedMs: waitMs };
+
+        this._logRequestFailed(encodedUrl, options, status);
+        const rateLimitError = await RateLimitError.create({
+            resource: encodedUrl,
+            init: options,
+            response,
+            responseBody: throttle.responseBody,
+            hint,
+            waitMs,
+            module: this._telemetryModuleLabel(),
+            scopeKey: computeScopeKey(this._rateLimitPolicy, this),
+        });
+        throw this._maybeFlagTimeoutDuringBodyRead(rateLimitError, timeoutMs);
+    }
+
     /**
      * Decides whether a response says a limit was hit. A 429 always does. A
      * 4xx or 5xx other than 401 does only when the module's classify() names
@@ -528,21 +583,17 @@ class Requester extends Delegate {
                 await this._readBodyForClassify(response));
         }
 
-        const signal = { status, headers: response.headers, body };
-        const options = {
-            attempt,
-            backOff: this.backOff,
-            now: Date.now(),
-            onClassifyError: (error) => this._logClassifyFailed(status, error),
-        };
-        if (status === 429) {
-            return {
-                throttled: true,
-                hint: resolveRateLimitHint(policy, signal, options),
-                responseBody,
-            };
-        }
-        const hint = classifyRateLimit(policy, signal, options);
+        const hint = resolveRateLimitHint(
+            policy,
+            { status, headers: response.headers, body },
+            {
+                attempt,
+                backOff: this.backOff,
+                now: Date.now(),
+                onClassifyError: (error) =>
+                    this._logClassifyFailed(status, error),
+            }
+        );
         return { throttled: Boolean(hint), hint, responseBody };
     }
 
@@ -551,11 +602,7 @@ class Requester extends Delegate {
             return {};
         }
         const text = await response.text();
-        if (
-            !JSON_CONTENT_TYPE.test(
-                headerValue(response.headers, 'content-type') || ''
-            )
-        ) {
+        if (!isJsonMediaType(headerValue(response.headers, 'content-type'))) {
             return { text };
         }
         try {

package/package.json

@@ -1,12 +1,13 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.656.10f1676.0",
+    "version": "2.0.0--canary.657.8f335eb.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
         "@aws-sdk/client-lambda": "^3.714.0",
         "@aws-sdk/client-s3": "^3.588.0",
+        "@aws-sdk/client-scheduler": "^3.588.0",
         "@aws-sdk/client-sqs": "^3.588.0",
         "@aws-sdk/client-ssm": "^3.588.0",
         "@aws-sdk/s3-request-presigner": "^3.588.0",
@@ -47,9 +48,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.656.10f1676.0",
-        "@friggframework/prettier-config": "2.0.0--canary.656.10f1676.0",
-        "@friggframework/test": "2.0.0--canary.656.10f1676.0",
+        "@friggframework/eslint-config": "2.0.0--canary.657.8f335eb.0",
+        "@friggframework/prettier-config": "2.0.0--canary.657.8f335eb.0",
+        "@friggframework/test": "2.0.0--canary.657.8f335eb.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -89,5 +90,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "10f1676a00a793a52a2f0a6edefb4d08ec677a15"
+    "gitHead": "8f335eb16f678038ad9448d9603c734ad663b3dc"
 }

package/queues/queue-deferral.js

@@ -0,0 +1,79 @@
+const MAX_DEFERRALS = 10;
+const MAX_DEFERRED_MS = 24 * 60 * 60 * 1000;
+const MAX_DELAY_SECONDS = 900;
+const MAX_VISIBILITY_TIMEOUT_SECONDS = 43_200;
+const MAX_DEFERRALS_ENV = 'FRIGG_QUEUE_MAX_DEFERRALS';
+const MAX_DEFERRED_MS_ENV = 'FRIGG_QUEUE_MAX_DEFERRED_MS';
+
+const toPositiveInteger = (value) => {
+    const number = Number(value);
+    return Number.isInteger(number) && number > 0 ? number : undefined;
+};
+
+/**
+ * @typedef {Object} DeferralLimits
+ * @property {number} maxDeferrals Times one message may be put back for a rate limit.
+ * @property {number} maxDeferredMs Longest total time from the first deferral to the last retry time.
+ */
+
+/**
+ * @param {Object} [env=process.env]
+ * @returns {DeferralLimits}
+ */
+const readDeferralLimits = (env = process.env) => ({
+    maxDeferrals: toPositiveInteger(env[MAX_DEFERRALS_ENV]) ?? MAX_DEFERRALS,
+    maxDeferredMs:
+        toPositiveInteger(env[MAX_DEFERRED_MS_ENV]) ?? MAX_DEFERRED_MS,
+});
+
+/**
+ * The deferral counters a message body carries in `_frigg`.
+ * @returns {{deferrals: number, firstDeferredAt: string|undefined}}
+ */
+const readDeferral = (body) => {
+    const frigg = body?._frigg;
+    const deferrals =
+        Number.isInteger(frigg?.deferrals) && frigg.deferrals > 0
+            ? frigg.deferrals
+            : 0;
+    const firstDeferredAt =
+        typeof frigg?.firstDeferredAt === 'string' &&
+        !Number.isNaN(Date.parse(frigg.firstDeferredAt))
+            ? frigg.firstDeferredAt
+            : undefined;
+    return { deferrals, firstDeferredAt };
+};
+
+const nextDeferral = (body, now = Date.now()) => {
+    const current = readDeferral(body);
+    return {
+        deferrals: current.deferrals + 1,
+        firstDeferredAt: current.firstDeferredAt ?? new Date(now).toISOString(),
+    };
+};
+
+const withDeferral = (body, deferral) => ({
+    ...body,
+    _frigg: { ...body._frigg, ...deferral },
+});
+
+const isDeferralCapped = (
+    { deferrals, firstDeferredAt, retryAt },
+    { maxDeferrals, maxDeferredMs }
+) =>
+    deferrals > maxDeferrals ||
+    retryAt.getTime() - Date.parse(firstDeferredAt) > maxDeferredMs;
+
+module.exports = {
+    MAX_DEFERRALS,
+    MAX_DEFERRALS_ENV,
+    MAX_DEFERRED_MS,
+    MAX_DEFERRED_MS_ENV,
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+};

package/queues/queuer-util.js

@@ -126,4 +126,4 @@ const QueuerUtil = {
     },
 };
 
-module.exports = { QueuerUtil };
+module.exports = { QueuerUtil, awsConfigOptions };

package/types/core/index.d.ts

@@ -47,8 +47,28 @@ declare module "@friggframework/core" {
     send(params: object & { QueueUrl: any }, delay?: number): Promise<string>;
 
     sendAsyncSQSMessage(params: SendSQSMessageParams): Promise<string>;
+
+    /**
+     * Hook: a message was put back for a rate limit. Runs after the message
+     * was sent. A failure is logged and never changes the outcome.
+     */
+    recordRateLimitWait(
+      body: object,
+      error: Error,
+      state: RateLimitWaitState
+    ): Promise<void>;
+
+    /** Hook: a message that was put back for a rate limit ran without error. */
+    clearRateLimitWait(body: object): Promise<void>;
   }
 
+  export type RateLimitWaitState = {
+    status: "WAITING" | "EXHAUSTED";
+    mechanism: "delay" | "schedule" | "visibility" | "none";
+    deferrals: number;
+    retryAt: Date;
+  };
+
   interface IWorker {
     getQueueURL(params: GetQueueURLParams): Promise<string | undefined>;
     run(params: { Records: any }, context?: object): Promise<BatchItemFailuresResponse>;

package/types/errors/index.d.ts

@@ -16,6 +16,12 @@ declare module "@friggframework/errors" {
     readonly body: any;
     isTimeout?: boolean;
     timeoutMs?: number;
+    /**
+     * True when `classify` named the response as a limit but gave no time. A
+     * `RateLimitError` always has it.
+     */
+    isRateLimited?: boolean;
+    reason?: RateLimitReason;
 
     static create(options?: CreateFetchErrorParams): Promise<FetchError>;
   }