@friggframework/core 2.0.0--canary.656.10f1676.0 → 2.0.0--canary.657.ebe5c60.0

package/CLAUDE.md

@@ -543,6 +543,58 @@ The single source of the max receive count is
 devtools integration builder uses it for the queue's `RedrivePolicy` and sets
 it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
 
+**Rate-limit deferral** (ADR-049): when a queue handler throws an error with
+`isRateLimited` and a `retryAt` (a `RateLimitError` from the Requester, or one a
+module throws), `Worker.run` does not give the message back to SQS for a 30
+minute redelivery that spends one of its three receives. It puts the message
+back so it runs at `retryAt`:
+
+| Wait | What the worker does | Record |
+|---|---|---|
+| Up to 900 s | Sends the same body again with `DelaySeconds`, then acknowledges the old message | handled |
+| Over 900 s, and `SCHEDULER_ROLE_ARN` is set | Creates a one-time EventBridge schedule at `retryAt` that sends the body to the queue, then acknowledges | handled |
+| Over 900 s, no scheduler | Extends the message's visibility timeout to `retryAt` (12 h at most). This uses one receive | reported failed |
+
+- The new message starts at receive count 1. A deferred body carries
+  `_frigg: { deferrals, firstDeferredAt }`; `_frigg` is reserved and the handler
+  never sees it.
+- Two caps bound it: `FRIGG_QUEUE_MAX_DEFERRALS` (default 10) and
+  `FRIGG_QUEUE_MAX_DEFERRED_MS` (default 24 h, from the first deferral to the
+  retry time). Past a cap, with no `retryAt`, with no event source ARN or on a
+  FIFO queue, the record fails like any retryable error and `delivery`
+  applies.
+- The mock scheduler is never used for a deferral: a schedule kept in memory
+  would be lost with the acknowledged message.
+- With a `processId` in the message, the worker writes
+  `Process.context.rateLimit = { status: 'WAITING' | 'EXHAUSTED', mechanism,
+  retryAt, reason, module, deferrals, updatedAt }`, and `null` after the message
+  ran. `EXHAUSTED` means a cap ended the deferrals, or a visibility change came
+  on the last delivery.
+- Delivery stays at least once. The new message is sent before the old one is
+  acknowledged, so a crash between the two can run the work twice.
+- Handlers check `delivery.isLastAttempt` first, then rethrow. The delay and
+  schedule tiers send a new message, so they never reach the last attempt. The
+  visibility tier and a cap use a receive: on the last one the worker writes an
+  ERROR (`frigg.worker.record_lost_rate_limited`), and the handler has this one
+  chance to end the run.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
+        throw error; // a RateLimitError too: core puts the message back at retryAt
+    }
+}
+```
+
+Records: `frigg.worker.record_deferred` (`mechanism` `delay` or `schedule`),
+`record_visibility_extended`, `record_deferral_capped`,
+`record_lost_rate_limited` and `rate_limit_state_failed`. A queue in a stack
+that does not own it (`ownership.queue: 'external'`) needs its own
+`sqs:ChangeMessageVisibility` and scheduler access.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.

package/core/CLAUDE.md

@@ -79,6 +79,25 @@ await worker.send({
 }, delaySeconds);
 ```
 
+**Rate-limit deferral** (ADR-049): `run()` handles an error with `isRateLimited`
+and a `retryAt` through `defer(record, body, error, delivery)`, which returns
+`{ outcome }`:
+- `acked`: the message was sent again with `DelaySeconds` (wait up to 900 s) or
+  scheduled with the one-time scheduler (longer wait). The record is handled.
+- `failed`: the visibility timeout was extended to `retryAt`, or a cap
+  (`FRIGG_QUEUE_MAX_DEFERRALS`, `FRIGG_QUEUE_MAX_DEFERRED_MS`) ended the
+  deferrals. The record is reported failed.
+- `skipped`: no `retryAt`, no `eventSourceARN`, a FIFO queue, or a failed send.
+  The record fails as any error does, and `record_failed` carries
+  `deferral: { skipped }`.
+
+Subclasses override two no-op hooks to keep run state: `recordRateLimitWait(body,
+error, state)` runs after the message was put back, and `clearRateLimitWait(body)`
+runs after a deferred or redelivered message succeeded. A failing hook logs
+`frigg.worker.rate_limit_state_failed` and never changes the outcome. The queue
+worker of an integration (`createQueueWorker`) writes `Process.context.rateLimit`
+in them. See `packages/core/CLAUDE.md` section 7.
+
 ### Delegate Pattern System (`Delegate.js:3-27`)
 
 **Purpose**: Observer/delegation pattern for decoupled component communication

package/core/Worker.js

@@ -1,14 +1,39 @@
-const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/client-sqs');
+const { randomUUID } = require('node:crypto');
+const {
+    SQSClient,
+    GetQueueUrlCommand,
+    SendMessageCommand,
+    ChangeMessageVisibilityCommand,
+} = require('@aws-sdk/client-sqs');
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
 const { readQueueDelivery } = require('../queues/queue-delivery');
+const { awsConfigOptions } = require('../queues/queuer-util');
+const {
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+} = require('../queues/queue-deferral');
 const { runMessageScope } = require('./invocation-scope');
-const { getLogger } = require('../logs');
+const { getLogger, serializeError } = require('../logs');
 
-const sqs = new SQSClient({ region: process.env.AWS_REGION });
+const sqs = new SQSClient({
+    region: process.env.AWS_REGION,
+    ...awsConfigOptions(),
+});
 const log = getLogger('frigg.worker');
 
+const validDate = (value) =>
+    value instanceof Date && !Number.isNaN(value.getTime()) ? value : null;
+const queueNameOf = (arn) =>
+    typeof arn === 'string' && arn ? arn.split(':').pop() : undefined;
+const causeOf = (error) => serializeError(error).message;
+
 class Worker {
     async getQueueURL(params) {
         // Passing params in because there will be multiple QueueNames
@@ -33,12 +58,31 @@ class Worker {
                 // messageId, receiveCount and the event come from the scope.
                 log.debug('Record started', { eventName: 'frigg.worker.record_started' });
 
+                const delivery = readQueueDelivery(record);
+                let runParams;
                 try {
-                    const runParams = JSON.parse(record.body);
+                    runParams = JSON.parse(record.body);
                     this._validateParams(runParams);
-                    await this._run(runParams, context, readQueueDelivery(record));
+                    await this._run(runParams, context, delivery);
+                    await this._clearDeferredState(runParams, delivery);
                     log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
                 } catch (error) {
+                    let deferral;
+                    if (error.isRateLimited && runParams) {
+                        deferral = await this.defer(
+                            record,
+                            runParams,
+                            error,
+                            delivery
+                        );
+                        if (deferral.outcome === 'acked') return;
+                        if (deferral.outcome === 'failed') {
+                            batchItemFailures.push({
+                                itemIdentifier: record.messageId,
+                            });
+                            return;
+                        }
+                    }
                     if (error.isHaltError) {
                         // HaltError means "discard this message, don't retry".
                         // Treat as success so SQS deletes it from the queue.
@@ -55,6 +99,14 @@ class Worker {
                     log.warn('Record failed, returned for retry', {
                         eventName: 'frigg.worker.record_failed',
                         error,
+                        ...(deferral && {
+                            deferral: {
+                                skipped: deferral.skipped,
+                                ...(deferral.cause && {
+                                    cause: causeOf(deferral.cause),
+                                }),
+                            },
+                        }),
                     });
                     batchItemFailures.push({ itemIdentifier: record.messageId });
                 }
@@ -75,6 +127,240 @@ class Worker {
         // parameters
     }
 
+    /**
+     * Puts a rate-limited message back so it runs at `error.retryAt`, without
+     * spending a receive. The message is sent again with a delay of up to 900 s,
+     * or scheduled with the one-time scheduler, or has its visibility timeout
+     * extended. The new message goes out before the old one is acknowledged, so
+     * delivery stays at least once.
+     *
+     * @returns {Promise<{outcome: 'acked'|'failed'|'skipped', skipped?: string, cause?: Error}>}
+     *   `acked`: the message is handled. `failed`: report the record failed and
+     *   let SQS redeliver it. `skipped`: nothing was done; fail the record as
+     *   any other error.
+     */
+    async defer(record, body, error, delivery) {
+        const retryAt = validDate(error.retryAt);
+        if (!retryAt) return { outcome: 'skipped', skipped: 'no_retry_at' };
+        const queueName = queueNameOf(record.eventSourceARN);
+        if (!queueName || queueName.endsWith('.fifo')) {
+            return { outcome: 'skipped', skipped: 'no_event_source' };
+        }
+
+        const now = Date.now();
+        const waitMs = Math.max(0, retryAt.getTime() - now);
+        const deferral = nextDeferral(body, now);
+        const details = {
+            waitMs,
+            retryAt: retryAt.toISOString(),
+            deferrals: deferral.deferrals,
+            firstDeferredAt: deferral.firstDeferredAt,
+            reason: error.reason,
+            module: error.module,
+            statusCode: error.statusCode,
+        };
+        const state = (status, mechanism) => ({
+            status,
+            mechanism,
+            deferrals: deferral.deferrals,
+            retryAt,
+        });
+
+        const limits = readDeferralLimits();
+        if (isDeferralCapped({ ...deferral, retryAt }, limits)) {
+            log.warn('Rate-limit deferral capped', {
+                eventName: 'frigg.worker.record_deferral_capped',
+                ...details,
+                maxDeferrals: limits.maxDeferrals,
+                maxDeferredMs: limits.maxDeferredMs,
+                deferredMs:
+                    retryAt.getTime() - Date.parse(deferral.firstDeferredAt),
+                error,
+            });
+            await this._recordWait(body, error, state('EXHAUSTED', 'none'));
+            this._logIfLost(delivery, error, details);
+            return { outcome: 'failed' };
+        }
+
+        const next = withDeferral(body, deferral);
+
+        if (waitMs <= MAX_DELAY_SECONDS * 1000) {
+            const delaySeconds = Math.ceil(waitMs / 1000);
+            try {
+                const queueUrl = await this._queueUrl(queueName);
+                await this.send({ ...next, QueueUrl: queueUrl }, delaySeconds);
+            } catch (cause) {
+                return { outcome: 'skipped', skipped: 'send_failed', cause };
+            }
+            await this._recordWait(body, error, state('WAITING', 'delay'));
+            log.warn('Record deferred', {
+                eventName: 'frigg.worker.record_deferred',
+                mechanism: 'delay',
+                delaySeconds,
+                ...details,
+                error,
+            });
+            return { outcome: 'acked' };
+        }
+
+        let scheduleError;
+        const scheduler = this.getSchedulerService();
+        if (scheduler) {
+            const scheduleName = `frigg-defer-${
+                record.messageId || randomUUID()
+            }`;
+            try {
+                await scheduler.scheduleOneTime({
+                    scheduleName,
+                    scheduleAt: retryAt,
+                    queueResourceId: record.eventSourceARN,
+                    payload: next,
+                });
+            } catch (cause) {
+                if (cause?.name !== 'ConflictException') scheduleError = cause;
+            }
+            if (!scheduleError) {
+                await this._recordWait(
+                    body,
+                    error,
+                    state('WAITING', 'schedule')
+                );
+                log.warn('Record deferred', {
+                    eventName: 'frigg.worker.record_deferred',
+                    mechanism: 'schedule',
+                    scheduleName,
+                    ...details,
+                    error,
+                });
+                return { outcome: 'acked' };
+            }
+        }
+
+        const visibilityTimeout = Math.min(
+            Math.ceil(waitMs / 1000),
+            MAX_VISIBILITY_TIMEOUT_SECONDS
+        );
+        try {
+            const queueUrl = await this._queueUrl(queueName);
+            await sqs.send(
+                new ChangeMessageVisibilityCommand({
+                    QueueUrl: queueUrl,
+                    ReceiptHandle: record.receiptHandle,
+                    VisibilityTimeout: visibilityTimeout,
+                })
+            );
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'visibility_failed', cause };
+        }
+        await this._recordWait(
+            body,
+            error,
+            state(
+                delivery.isLastAttempt ? 'EXHAUSTED' : 'WAITING',
+                'visibility'
+            )
+        );
+        log.warn('Record visibility extended', {
+            eventName: 'frigg.worker.record_visibility_extended',
+            mechanism: 'visibility',
+            visibilityTimeout,
+            ...details,
+            ...(scheduleError && { scheduleError: causeOf(scheduleError) }),
+            error,
+        });
+        this._logIfLost(delivery, error, details);
+        return { outcome: 'failed' };
+    }
+
+    /**
+     * The one-time scheduler, or null when the stack has none. The mock
+     * scheduler is never used: a schedule kept in memory would be lost with
+     * the message.
+     */
+    getSchedulerService() {
+        if (this._schedulerService !== undefined) return this._schedulerService;
+        this._schedulerService = null;
+        if (
+            process.env.SCHEDULER_ROLE_ARN &&
+            process.env.SCHEDULER_PROVIDER !== 'mock'
+        ) {
+            const {
+                createSchedulerService,
+                SCHEDULER_PROVIDERS,
+            } = require('../infrastructure/scheduler/scheduler-service-factory');
+            this._schedulerService = createSchedulerService({
+                provider: SCHEDULER_PROVIDERS.EVENTBRIDGE,
+            });
+        }
+        return this._schedulerService;
+    }
+
+    /**
+     * Hook: the message was put back for a rate limit. Runs after it was sent.
+     * @param {Object} body The parsed message body.
+     * @param {Error} error The RateLimitError.
+     * @param {{status: 'WAITING'|'EXHAUSTED', mechanism: string, deferrals: number, retryAt: Date}} state
+     */
+    async recordRateLimitWait() {}
+
+    /**
+     * Hook: a message that was put back for a rate limit ran without error.
+     * @param {Object} body The parsed message body.
+     */
+    async clearRateLimitWait() {}
+
+    _queueUrl(queueName) {
+        this._queueUrls ??= new Map();
+        if (!this._queueUrls.has(queueName)) {
+            const lookup = this.getQueueURL({ QueueName: queueName }).catch(
+                (error) => {
+                    this._queueUrls.delete(queueName);
+                    throw error;
+                }
+            );
+            this._queueUrls.set(queueName, lookup);
+        }
+        return this._queueUrls.get(queueName);
+    }
+
+    async _recordWait(body, error, state) {
+        await this._runStateHook('record', () =>
+            this.recordRateLimitWait(body, error, state)
+        );
+    }
+
+    async _clearDeferredState(body, delivery) {
+        if (readDeferral(body).deferrals === 0 && !(delivery.receiveCount > 1))
+            return;
+        await this._runStateHook('clear', () => this.clearRateLimitWait(body));
+    }
+
+    async _runStateHook(operation, hook) {
+        try {
+            await hook();
+        } catch (error) {
+            log.warn('Rate-limit run state not written', {
+                eventName: 'frigg.worker.rate_limit_state_failed',
+                operation,
+                error,
+            });
+        }
+    }
+
+    _logIfLost(delivery, error, details) {
+        if (!delivery.isLastAttempt) return;
+        log.error(
+            'Rate-limited record lost: the message goes to the dead-letter queue next',
+            {
+                eventName: 'frigg.worker.record_lost_rate_limited',
+                retryAt: details.retryAt,
+                reason: error.reason,
+                module: error.module,
+                error,
+            }
+        );
+    }
+
     // returns the message id
     async send(params, delay = 0) {
         this._validateParams(params);

package/handlers/backend-utils.js

@@ -165,6 +165,32 @@ const createQueueWorker = (integrationClass) => {
     const integrationName = integrationClass.Definition.name;
 
     class QueueWorker extends Worker {
+        async recordRateLimitWait(body, error, state) {
+            const processId = body.data?.processId;
+            if (!processId) return;
+            await createProcessRepository().applyProcessUpdate(processId, {
+                set: {
+                    'context.rateLimit': {
+                        status: state.status,
+                        mechanism: state.mechanism,
+                        retryAt: state.retryAt.toISOString(),
+                        reason: error.reason,
+                        module: error.module,
+                        deferrals: state.deferrals,
+                        updatedAt: new Date().toISOString(),
+                    },
+                },
+            });
+        }
+
+        async clearRateLimitWait(body) {
+            const processId = body?.data?.processId;
+            if (!processId) return;
+            await createProcessRepository().applyProcessUpdate(processId, {
+                set: { 'context.rateLimit': null },
+            });
+        }
+
         async _run(params, context, delivery) {
             const logCtx = {
                 integration: integrationName,

package/modules/requester/oauth-2.js

@@ -398,6 +398,12 @@ class OAuth2Requester extends Requester {
         );
         transportError.statusCode = status;
         transportError.isTokenRefreshTransportFailure = true;
+        if (error?.isRateLimited) {
+            transportError.isRateLimited = true;
+            transportError.retryAt = error.retryAt;
+            transportError.waitMs = error.waitMs;
+            transportError.reason = error.reason;
+        }
         return transportError;
     }
 

package/package.json

@@ -1,12 +1,13 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.656.10f1676.0",
+    "version": "2.0.0--canary.657.ebe5c60.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
         "@aws-sdk/client-lambda": "^3.714.0",
         "@aws-sdk/client-s3": "^3.588.0",
+        "@aws-sdk/client-scheduler": "^3.588.0",
         "@aws-sdk/client-sqs": "^3.588.0",
         "@aws-sdk/client-ssm": "^3.588.0",
         "@aws-sdk/s3-request-presigner": "^3.588.0",
@@ -47,9 +48,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.656.10f1676.0",
-        "@friggframework/prettier-config": "2.0.0--canary.656.10f1676.0",
-        "@friggframework/test": "2.0.0--canary.656.10f1676.0",
+        "@friggframework/eslint-config": "2.0.0--canary.657.ebe5c60.0",
+        "@friggframework/prettier-config": "2.0.0--canary.657.ebe5c60.0",
+        "@friggframework/test": "2.0.0--canary.657.ebe5c60.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -89,5 +90,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "10f1676a00a793a52a2f0a6edefb4d08ec677a15"
+    "gitHead": "ebe5c602d3aac76850912c731dd91851772856b3"
 }

package/queues/queue-deferral.js

@@ -0,0 +1,79 @@
+const MAX_DEFERRALS = 10;
+const MAX_DEFERRED_MS = 24 * 60 * 60 * 1000;
+const MAX_DELAY_SECONDS = 900;
+const MAX_VISIBILITY_TIMEOUT_SECONDS = 43_200;
+const MAX_DEFERRALS_ENV = 'FRIGG_QUEUE_MAX_DEFERRALS';
+const MAX_DEFERRED_MS_ENV = 'FRIGG_QUEUE_MAX_DEFERRED_MS';
+
+const toPositiveInteger = (value) => {
+    const number = Number(value);
+    return Number.isInteger(number) && number > 0 ? number : undefined;
+};
+
+/**
+ * @typedef {Object} DeferralLimits
+ * @property {number} maxDeferrals Times one message may be put back for a rate limit.
+ * @property {number} maxDeferredMs Longest total time from the first deferral to the last retry time.
+ */
+
+/**
+ * @param {Object} [env=process.env]
+ * @returns {DeferralLimits}
+ */
+const readDeferralLimits = (env = process.env) => ({
+    maxDeferrals: toPositiveInteger(env[MAX_DEFERRALS_ENV]) ?? MAX_DEFERRALS,
+    maxDeferredMs:
+        toPositiveInteger(env[MAX_DEFERRED_MS_ENV]) ?? MAX_DEFERRED_MS,
+});
+
+/**
+ * The deferral counters a message body carries in `_frigg`.
+ * @returns {{deferrals: number, firstDeferredAt: string|undefined}}
+ */
+const readDeferral = (body) => {
+    const frigg = body?._frigg;
+    const deferrals =
+        Number.isInteger(frigg?.deferrals) && frigg.deferrals > 0
+            ? frigg.deferrals
+            : 0;
+    const firstDeferredAt =
+        typeof frigg?.firstDeferredAt === 'string' &&
+        !Number.isNaN(Date.parse(frigg.firstDeferredAt))
+            ? frigg.firstDeferredAt
+            : undefined;
+    return { deferrals, firstDeferredAt };
+};
+
+const nextDeferral = (body, now = Date.now()) => {
+    const current = readDeferral(body);
+    return {
+        deferrals: current.deferrals + 1,
+        firstDeferredAt: current.firstDeferredAt ?? new Date(now).toISOString(),
+    };
+};
+
+const withDeferral = (body, deferral) => ({
+    ...body,
+    _frigg: { ...body._frigg, ...deferral },
+});
+
+const isDeferralCapped = (
+    { deferrals, firstDeferredAt, retryAt },
+    { maxDeferrals, maxDeferredMs }
+) =>
+    deferrals > maxDeferrals ||
+    retryAt.getTime() - Date.parse(firstDeferredAt) > maxDeferredMs;
+
+module.exports = {
+    MAX_DEFERRALS,
+    MAX_DEFERRALS_ENV,
+    MAX_DEFERRED_MS,
+    MAX_DEFERRED_MS_ENV,
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+};

package/queues/queuer-util.js

@@ -126,4 +126,4 @@ const QueuerUtil = {
     },
 };
 
-module.exports = { QueuerUtil };
+module.exports = { QueuerUtil, awsConfigOptions };

package/types/core/index.d.ts

@@ -47,8 +47,28 @@ declare module "@friggframework/core" {
     send(params: object & { QueueUrl: any }, delay?: number): Promise<string>;
 
     sendAsyncSQSMessage(params: SendSQSMessageParams): Promise<string>;
+
+    /**
+     * Hook: a message was put back for a rate limit. Runs after the message
+     * was sent. A failure is logged and never changes the outcome.
+     */
+    recordRateLimitWait(
+      body: object,
+      error: Error,
+      state: RateLimitWaitState
+    ): Promise<void>;
+
+    /** Hook: a message that was put back for a rate limit ran without error. */
+    clearRateLimitWait(body: object): Promise<void>;
   }
 
+  export type RateLimitWaitState = {
+    status: "WAITING" | "EXHAUSTED";
+    mechanism: "delay" | "schedule" | "visibility" | "none";
+    deferrals: number;
+    retryAt: Date;
+  };
+
   interface IWorker {
     getQueueURL(params: GetQueueURLParams): Promise<string | undefined>;
     run(params: { Records: any }, context?: object): Promise<BatchItemFailuresResponse>;