@friggframework/core 2.0.0--canary.656.4fb07b4.0 → 2.0.0--canary.657.6a81b60.0

package/CLAUDE.md

@@ -483,21 +483,22 @@ handlers (`{ req, res, next }`) and `this.on` events do not.
 
 `isLastAttempt` is `false` when either count is unknown: a local or non-SQS
 invocation, or a queue whose redrive policy the stack does not own
-(`ownership.queue: 'external'`). The value is information only: core still
-rethrows retryable errors and discards halt errors (4xx except 408, 429 and
-errors with `isRateLimited`).
+(`ownership.queue: 'external'`). The value does not change what core does: a
+retryable error returns the message to SQS, a rate-limited error puts it back
+to run at `retryAt` (see **Rate-limit deferral** below), and a halt error (4xx
+except 408, 429 and errors with `isRateLimited`) discards it.
 
-Use it to end a run or count lost work on the final try: when a retryable
-error (429, 5xx, network) is about to be rethrown and `isLastAttempt` is
-`true`, the message goes to the DLQ next, so mark the run failed or count the
-message's records as failed before rethrowing.
+Use it to end a run or count lost work on the final try. Check it first: when
+it is `true`, mark the run failed or count the message's records as failed, and
+return. Otherwise rethrow. Do not end the run and then rethrow: core can still
+send a `RateLimitError` again, so the message would run again on an ended run.
 
 ```javascript
 async processBatch({ data, delivery }) {
     try {
         await this.syncPage(data);
     } catch (error) {
-        if (delivery?.isLastAttempt) await this.failRun(data.processId, error);
+        if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
         throw error;
     }
 }
@@ -508,6 +509,71 @@ The single source of the max receive count is
 devtools integration builder uses it for the queue's `RedrivePolicy` and sets
 it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
 
+**Rate-limit deferral** (ADR-049): when a queue handler throws an error with
+`isRateLimited` and a `retryAt` (a `RateLimitError` from the Requester, or one a
+module throws), `Worker.run` does not give the message back to SQS for a 30
+minute redelivery. It puts the message back so it runs at `retryAt`:
+
+| Wait | What the worker does | Record |
+|---|---|---|
+| Up to 900 s | Sends the same body again with `DelaySeconds`, then acknowledges the old message | handled |
+| Over 900 s, and `SCHEDULER_ROLE_ARN` is set | Creates a one-time EventBridge schedule at `retryAt` that sends the body to the queue, then acknowledges | handled |
+| Over 900 s, no scheduler | Extends the message's visibility timeout to `retryAt` (12 h less 20 minutes at most). This uses one receive | reported failed |
+
+- The new message starts at receive count 1. A deferred body carries
+  `_frigg: { deferrals, firstDeferredAt }`; `_frigg` is reserved and the handler
+  never sees it.
+- The visibility tier sets a timeout of at most 12 h less 20 minutes
+  (42,000 s). SQS counts its 12 h maximum from the receive, and an extension
+  does not reset it. The margin covers the longest Lambda run (900 s) plus the
+  longest batching window (300 s).
+- Two caps bound it: `FRIGG_QUEUE_MAX_DEFERRALS` (default 30) and
+  `FRIGG_QUEUE_MAX_DEFERRED_MS` (default 26 h, from the first deferral to the
+  retry time). The defaults cover one full day of hourly waits, for a module
+  that probes a daily limit every hour. Past a cap, with no `retryAt`, with no
+  event source ARN or on a FIFO queue, the record fails like any retryable
+  error and `delivery` applies.
+- The mock scheduler is never used for a deferral: a schedule kept in memory
+  would be lost with the acknowledged message.
+- With a `processId` in the message, the worker writes
+  `Process.context.rateLimit = { status: 'WAITING' | 'EXHAUSTED', mechanism,
+  retryAt, reason, module, deferrals, updatedAt }`. `EXHAUSTED` means a cap ended
+  the deferrals, or a visibility change came on the last delivery. The value is
+  one per process, so it holds the latest known wait: a new wait replaces it only
+  when its `retryAt` is later, a `WAITING` value never hides an `EXHAUSTED` one,
+  and after a deferred or redelivered message ran the worker sets it to `null`
+  only when it is `WAITING` and its `retryAt` is past or within 5 s.
+- Delivery stays at least once. The new message is sent before the old one is
+  acknowledged, so a crash between the two can run the work twice.
+- Handlers check `delivery.isLastAttempt` first, then rethrow. The delay and
+  schedule tiers send a new message, so they never reach the last attempt. The
+  visibility tier and a cap use a receive: on the last one the worker writes an
+  ERROR (`frigg.worker.record_lost_rate_limited`), and the handler has this one
+  chance to end the run.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) return this.failRun(data.processId, error);
+        throw error; // a RateLimitError too: core puts the message back at retryAt
+    }
+}
+```
+
+Records: a rate-limited SQS record gets one log record, by its outcome.
+
+| Outcome | Log record |
+|---|---|
+| Acknowledged | WARN `frigg.worker.record_deferred` (`mechanism` `delay` or `schedule`) |
+| Not acknowledged, not the last attempt | One WARN: `record_deferral_capped`, `record_visibility_extended`, or `record_failed` with `deferral.skipped` |
+| Not acknowledged, last attempt | One ERROR, `record_lost_rate_limited`, with the details, and no WARN |
+
+A failing run state hook adds `rate_limit_state_failed`. A queue in a stack
+that does not own it (`ownership.queue: 'external'`) needs its own
+`sqs:ChangeMessageVisibility` and scheduler access.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.
@@ -759,6 +825,8 @@ Use test doubles from `@friggframework/test` package for consistent mocking.
 - `SECRET_ARN` - AWS Secrets Manager ARN for auto-injection
 - `FRIGG_LOG_LEVEL` - Minimum log level (`TRACE` … `FATAL`, default `INFO`; `DEBUG` on local runs)
 - `FRIGG_QUEUE_MAX_RECEIVE_COUNT` - Set by devtools on integration queue workers whose queue the stack owns; feeds `delivery.maxReceiveCount`
+- `FRIGG_QUEUE_MAX_DEFERRALS` - Most rate-limit deferrals of one message (default 30)
+- `FRIGG_QUEUE_MAX_DEFERRED_MS` - Most time in ms from a message's first rate-limit deferral to its retry time (default 26 h)
 
 ## Version Information
 

package/core/CLAUDE.md

@@ -79,6 +79,31 @@ await worker.send({
 }, delaySeconds);
 ```
 
+**Rate-limit deferral** (ADR-049): `run()` handles an error with `isRateLimited`
+and a `retryAt` through `defer(record, body, error, delivery)`, which returns
+`{ outcome }`:
+- `acked`: the message was sent again with `DelaySeconds` (wait up to 900 s) or
+  scheduled with the one-time scheduler (longer wait). The record is handled.
+- `failed`: the visibility timeout was extended to `retryAt`, or a cap
+  (`FRIGG_QUEUE_MAX_DEFERRALS`, `FRIGG_QUEUE_MAX_DEFERRED_MS`) ended the
+  deferrals. The record is reported failed.
+- `skipped`: no `retryAt` (`no_retry_at`), no `eventSourceARN` or a FIFO queue
+  (`no_event_source`), a failed send (`send_failed`), or a failed visibility
+  change (`visibility_failed`). The record fails as any error does.
+
+Each outcome writes one log record. `acked` writes WARN `record_deferred`.
+`failed` and `skipped` write one WARN (`record_visibility_extended`,
+`record_deferral_capped`, or `record_failed` with `deferral: { skipped }`). On
+the last attempt they write one ERROR, `record_lost_rate_limited`, with the
+details, and no WARN.
+
+Subclasses override two no-op hooks to keep run state: `recordRateLimitWait(body,
+error, state)` runs after the message was put back, and `clearRateLimitWait(body)`
+runs after a deferred or redelivered message succeeded. A failing hook logs
+`frigg.worker.rate_limit_state_failed` and never changes the outcome. The queue
+worker of an integration (`createQueueWorker`) writes `Process.context.rateLimit`
+in them. See `packages/core/CLAUDE.md` section 7.
+
 ### Delegate Pattern System (`Delegate.js:3-27`)
 
 **Purpose**: Observer/delegation pattern for decoupled component communication

package/core/Worker.js

@@ -1,14 +1,38 @@
-const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/client-sqs');
+const {
+    SQSClient,
+    GetQueueUrlCommand,
+    SendMessageCommand,
+    ChangeMessageVisibilityCommand,
+} = require('@aws-sdk/client-sqs');
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
 const { readQueueDelivery } = require('../queues/queue-delivery');
+const { awsConfigOptions } = require('../queues/queuer-util');
+const {
+    MAX_DEFERRED_VISIBILITY_SECONDS,
+    MAX_DELAY_SECONDS,
+    deferredMs,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+} = require('../queues/queue-deferral');
 const { runMessageScope } = require('./invocation-scope');
 const { getLogger } = require('../logs');
 
-const sqs = new SQSClient({ region: process.env.AWS_REGION });
+const sqs = new SQSClient({
+    region: process.env.AWS_REGION,
+    ...awsConfigOptions(),
+});
 const log = getLogger('frigg.worker');
 
+const validDate = (value) =>
+    value instanceof Date && !Number.isNaN(value.getTime()) ? value : null;
+const queueNameOf = (arn) =>
+    typeof arn === 'string' && arn ? arn.split(':').pop() : undefined;
+
 class Worker {
     async getQueueURL(params) {
         // Passing params in because there will be multiple QueueNames
@@ -33,12 +57,31 @@ class Worker {
                 // messageId, receiveCount and the event come from the scope.
                 log.debug('Record started', { eventName: 'frigg.worker.record_started' });
 
+                const delivery = readQueueDelivery(record);
+                let runParams;
                 try {
-                    const runParams = JSON.parse(record.body);
+                    runParams = JSON.parse(record.body);
                     this._validateParams(runParams);
-                    await this._run(runParams, context, readQueueDelivery(record));
+                    await this._run(runParams, context, delivery);
+                    await this._clearDeferredState(runParams, delivery);
                     log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
                 } catch (error) {
+                    let deferralResult;
+                    if (error.isRateLimited && runParams) {
+                        deferralResult = await this.defer(
+                            record,
+                            runParams,
+                            error,
+                            delivery
+                        );
+                        if (deferralResult.outcome === 'acked') return;
+                        if (deferralResult.outcome === 'failed') {
+                            batchItemFailures.push({
+                                itemIdentifier: record.messageId,
+                            });
+                            return;
+                        }
+                    }
                     if (error.isHaltError) {
                         // HaltError means "discard this message, don't retry".
                         // Treat as success so SQS deletes it from the queue.
@@ -51,11 +94,30 @@ class Worker {
                         });
                         return;
                     }
-                    // The message goes back to SQS, so WARN (ADR-048 §4).
-                    log.warn('Record failed, returned for retry', {
-                        eventName: 'frigg.worker.record_failed',
-                        error,
-                    });
+                    const skippedDeferral = deferralResult && {
+                        deferral: {
+                            skipped: deferralResult.skipped,
+                            ...(deferralResult.cause && {
+                                cause: deferralResult.cause,
+                            }),
+                        },
+                    };
+                    if (skippedDeferral && delivery.isLastAttempt) {
+                        this._logLost({
+                            retryAt: validDate(error.retryAt)?.toISOString(),
+                            reason: error.reason,
+                            module: error.module,
+                            ...skippedDeferral,
+                            error,
+                        });
+                    } else {
+                        // The message goes back to SQS, so WARN (ADR-048 §4).
+                        log.warn('Record failed, returned for retry', {
+                            eventName: 'frigg.worker.record_failed',
+                            error,
+                            ...skippedDeferral,
+                        });
+                    }
                     batchItemFailures.push({ itemIdentifier: record.messageId });
                 }
             });
@@ -75,6 +137,239 @@ class Worker {
         // parameters
     }
 
+    async defer(record, body, error, delivery) {
+        const retryAt = validDate(error.retryAt);
+        if (!retryAt) return { outcome: 'skipped', skipped: 'no_retry_at' };
+        const queueName = queueNameOf(record.eventSourceARN);
+        if (!queueName || queueName.endsWith('.fifo')) {
+            return { outcome: 'skipped', skipped: 'no_event_source' };
+        }
+
+        const now = Date.now();
+        const counters = nextDeferral(body, now);
+        const deferral = {
+            record,
+            queueName,
+            body,
+            deferredBody: withDeferral(body, counters),
+            error,
+            delivery,
+            retryAt,
+            waitMs: Math.max(0, retryAt.getTime() - now),
+            counters,
+        };
+
+        const limits = readDeferralLimits();
+        if (isDeferralCapped({ ...counters, retryAt }, limits)) {
+            await this._recordWait(deferral, 'EXHAUSTED', 'none');
+            this._logUnacked(
+                deferral,
+                'record_deferral_capped',
+                'Rate-limit deferral capped',
+                {
+                    maxDeferrals: limits.maxDeferrals,
+                    maxDeferredMs: limits.maxDeferredMs,
+                    deferredMs: deferredMs({ ...counters, retryAt }),
+                }
+            );
+            return { outcome: 'failed' };
+        }
+
+        if (deferral.waitMs <= MAX_DELAY_SECONDS * 1000)
+            return this._deferWithDelay(deferral);
+        const scheduled = await this._deferWithSchedule(deferral);
+        if (scheduled?.outcome) return scheduled;
+        return this._deferWithVisibility(deferral, scheduled?.scheduleError);
+    }
+
+    async _deferWithDelay(deferral) {
+        const delaySeconds = Math.ceil(deferral.waitMs / 1000);
+        try {
+            const queueUrl = await this._queueUrl(deferral.queueName);
+            await this.send(
+                { ...deferral.deferredBody, QueueUrl: queueUrl },
+                delaySeconds
+            );
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'send_failed', cause };
+        }
+        return this._acknowledgeDeferral(deferral, 'delay', { delaySeconds });
+    }
+
+    async _deferWithSchedule(deferral) {
+        const { record, retryAt, deferredBody } = deferral;
+        const scheduleName = `frigg-defer-${record.messageId}`;
+        try {
+            const scheduler = this.getSchedulerService();
+            if (!scheduler) return null;
+            await scheduler.scheduleOneTime({
+                scheduleName,
+                scheduleAt: retryAt,
+                queueResourceId: record.eventSourceARN,
+                payload: deferredBody,
+            });
+        } catch (cause) {
+            if (cause?.name !== 'ConflictException')
+                return { scheduleError: cause };
+        }
+        return this._acknowledgeDeferral(deferral, 'schedule', {
+            scheduleName,
+        });
+    }
+
+    async _deferWithVisibility(deferral, scheduleError) {
+        const { record, queueName, delivery, waitMs } = deferral;
+        const visibilityTimeout = Math.min(
+            Math.ceil(waitMs / 1000),
+            MAX_DEFERRED_VISIBILITY_SECONDS
+        );
+        try {
+            const queueUrl = await this._queueUrl(queueName);
+            await sqs.send(
+                new ChangeMessageVisibilityCommand({
+                    QueueUrl: queueUrl,
+                    ReceiptHandle: record.receiptHandle,
+                    VisibilityTimeout: visibilityTimeout,
+                })
+            );
+        } catch (cause) {
+            return { outcome: 'skipped', skipped: 'visibility_failed', cause };
+        }
+        await this._recordWait(
+            deferral,
+            delivery.isLastAttempt ? 'EXHAUSTED' : 'WAITING',
+            'visibility'
+        );
+        this._logUnacked(
+            deferral,
+            'record_visibility_extended',
+            'Record visibility extended',
+            {
+                mechanism: 'visibility',
+                visibilityTimeout,
+                ...(scheduleError && { scheduleError }),
+            }
+        );
+        return { outcome: 'failed' };
+    }
+
+    async _acknowledgeDeferral(deferral, mechanism, fields) {
+        await this._recordWait(deferral, 'WAITING', mechanism);
+        log.warn('Record deferred', {
+            eventName: 'frigg.worker.record_deferred',
+            mechanism,
+            ...fields,
+            ...this._deferralLogFields(deferral),
+            error: deferral.error,
+        });
+        return { outcome: 'acked' };
+    }
+
+    getSchedulerService() {
+        if (this._schedulerService !== undefined) return this._schedulerService;
+        if (
+            !process.env.SCHEDULER_ROLE_ARN ||
+            process.env.SCHEDULER_PROVIDER === 'mock'
+        ) {
+            this._schedulerService = null;
+            return null;
+        }
+        const {
+            createSchedulerService,
+            SCHEDULER_PROVIDERS,
+        } = require('../infrastructure/scheduler/scheduler-service-factory');
+        this._schedulerService = createSchedulerService({
+            provider: SCHEDULER_PROVIDERS.EVENTBRIDGE,
+        });
+        return this._schedulerService;
+    }
+
+    recordRateLimitWait() {
+        return Promise.resolve();
+    }
+
+    clearRateLimitWait() {
+        return Promise.resolve();
+    }
+
+    _queueUrl(queueName) {
+        this._queueUrls ??= new Map();
+        if (!this._queueUrls.has(queueName)) {
+            const lookup = this.getQueueURL({ QueueName: queueName }).catch(
+                (error) => {
+                    this._queueUrls.delete(queueName);
+                    throw error;
+                }
+            );
+            this._queueUrls.set(queueName, lookup);
+        }
+        return this._queueUrls.get(queueName);
+    }
+
+    async _recordWait({ body, error, counters, retryAt }, status, mechanism) {
+        await this._runStateHook('record', () =>
+            this.recordRateLimitWait(body, error, {
+                status,
+                mechanism,
+                deferrals: counters.deferrals,
+                retryAt,
+            })
+        );
+    }
+
+    async _clearDeferredState(body, delivery) {
+        const redelivered = delivery.receiveCount > 1;
+        if (readDeferral(body).deferrals === 0 && !redelivered) return;
+        await this._runStateHook('clear', () => this.clearRateLimitWait(body));
+    }
+
+    async _runStateHook(operation, hook) {
+        try {
+            await hook();
+        } catch (error) {
+            log.warn('Rate-limit run state not written', {
+                eventName: 'frigg.worker.rate_limit_state_failed',
+                operation,
+                error,
+            });
+        }
+    }
+
+    _deferralLogFields({ error, retryAt, waitMs, counters }) {
+        return {
+            waitMs,
+            retryAt: retryAt.toISOString(),
+            deferrals: counters.deferrals,
+            firstDeferredAt: counters.firstDeferredAt,
+            reason: error.reason,
+            module: error.module,
+            statusCode: error.statusCode,
+        };
+    }
+
+    _logUnacked(deferral, eventName, message, fields) {
+        const logFields = {
+            ...fields,
+            ...this._deferralLogFields(deferral),
+            error: deferral.error,
+        };
+        if (deferral.delivery.isLastAttempt) {
+            this._logLost(logFields);
+            return;
+        }
+        log.warn(message, {
+            eventName: `frigg.worker.${eventName}`,
+            ...logFields,
+        });
+    }
+
+    _logLost(fields) {
+        log.error(
+            'Rate-limited record lost: the message goes to the dead-letter queue next',
+            { eventName: 'frigg.worker.record_lost_rate_limited', ...fields }
+        );
+    }
+
     // returns the message id
     async send(params, delay = 0) {
         this._validateParams(params);

package/handlers/backend-utils.js

@@ -21,6 +21,12 @@ const {
     getModulesDefinitionFromIntegrationClasses,
 } = require('../integrations/utils/map-integration-dto');
 const { processNotFound } = require('../integrations/use-cases/process-errors');
+const {
+    RecordRateLimitWait,
+} = require('../integrations/use-cases/record-rate-limit-wait');
+const {
+    ClearRateLimitWait,
+} = require('../integrations/use-cases/clear-rate-limit-wait');
 
 const loadRouterFromObject = (IntegrationClass, routerObject) => {
     const router = Router();
@@ -165,6 +171,28 @@ const createQueueWorker = (integrationClass) => {
     const integrationName = integrationClass.Definition.name;
 
     class QueueWorker extends Worker {
+        async recordRateLimitWait(body, error, state) {
+            const processId = body.data?.processId;
+            if (!processId) return;
+            const recordRateLimitWait = new RecordRateLimitWait({
+                processRepository: createProcessRepository(),
+            });
+            await recordRateLimitWait.execute(processId, {
+                ...state,
+                reason: error.reason,
+                module: error.module,
+            });
+        }
+
+        async clearRateLimitWait(body) {
+            const processId = body?.data?.processId;
+            if (!processId) return;
+            const clearRateLimitWait = new ClearRateLimitWait({
+                processRepository: createProcessRepository(),
+            });
+            await clearRateLimitWait.execute(processId);
+        }
+
         async _run(params, context, delivery) {
             const logCtx = {
                 integration: integrationName,

package/integrations/use-cases/clear-rate-limit-wait.js

@@ -0,0 +1,23 @@
+const CLEAR_RATE_LIMIT_TOLERANCE_MS = 5_000;
+
+class ClearRateLimitWait {
+    constructor({ processRepository }) {
+        if (!processRepository) {
+            throw new Error('processRepository is required');
+        }
+        this.processRepository = processRepository;
+    }
+
+    async execute(processId) {
+        const process = await this.processRepository.findById(processId);
+        const current = process?.context?.rateLimit;
+        if (current?.status !== 'WAITING') return;
+        const remainingMs = Date.parse(current.retryAt) - Date.now();
+        if (remainingMs > CLEAR_RATE_LIMIT_TOLERANCE_MS) return;
+        await this.processRepository.applyProcessUpdate(processId, {
+            set: { 'context.rateLimit': null },
+        });
+    }
+}
+
+module.exports = { ClearRateLimitWait };

package/integrations/use-cases/index.js

@@ -6,6 +6,8 @@ const { CreateProcess } = require('./create-process');
 const { UpdateProcessState } = require('./update-process-state');
 const { UpdateProcessMetrics } = require('./update-process-metrics');
 const { GetProcess } = require('./get-process');
+const { RecordRateLimitWait } = require('./record-rate-limit-wait');
+const { ClearRateLimitWait } = require('./clear-rate-limit-wait');
 
 module.exports = {
     GetIntegrationsForUser,
@@ -16,4 +18,6 @@ module.exports = {
     UpdateProcessState,
     UpdateProcessMetrics,
     GetProcess,
-}; 
+    RecordRateLimitWait,
+    ClearRateLimitWait,
+};

package/integrations/use-cases/record-rate-limit-wait.js

@@ -0,0 +1,35 @@
+const keepsCurrentRateLimit = (current, next) => {
+    if (!current) return false;
+    if (next.status === 'EXHAUSTED') return false;
+    if (current.status === 'EXHAUSTED') return true;
+    return Date.parse(current.retryAt) > next.retryAt.getTime();
+};
+
+class RecordRateLimitWait {
+    constructor({ processRepository }) {
+        if (!processRepository) {
+            throw new Error('processRepository is required');
+        }
+        this.processRepository = processRepository;
+    }
+
+    async execute(processId, wait) {
+        const process = await this.processRepository.findById(processId);
+        if (keepsCurrentRateLimit(process?.context?.rateLimit, wait)) return;
+        await this.processRepository.applyProcessUpdate(processId, {
+            set: {
+                'context.rateLimit': {
+                    status: wait.status,
+                    mechanism: wait.mechanism,
+                    retryAt: wait.retryAt.toISOString(),
+                    reason: wait.reason,
+                    module: wait.module,
+                    deferrals: wait.deferrals,
+                    updatedAt: new Date().toISOString(),
+                },
+            },
+        });
+    }
+}
+
+module.exports = { RecordRateLimitWait };

package/modules/requester/oauth-2.js

@@ -398,6 +398,13 @@ class OAuth2Requester extends Requester {
         );
         transportError.statusCode = status;
         transportError.isTokenRefreshTransportFailure = true;
+        if (error?.isRateLimited) {
+            transportError.isRateLimited = true;
+            transportError.retryAt = error.retryAt;
+            transportError.waitMs = error.waitMs;
+            transportError.reason = error.reason;
+            transportError.module = error.module;
+        }
         return transportError;
     }
 

package/package.json

@@ -1,12 +1,13 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.656.4fb07b4.0",
+    "version": "2.0.0--canary.657.6a81b60.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
         "@aws-sdk/client-lambda": "^3.714.0",
         "@aws-sdk/client-s3": "^3.588.0",
+        "@aws-sdk/client-scheduler": "^3.588.0",
         "@aws-sdk/client-sqs": "^3.588.0",
         "@aws-sdk/client-ssm": "^3.588.0",
         "@aws-sdk/s3-request-presigner": "^3.588.0",
@@ -47,12 +48,13 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.656.4fb07b4.0",
-        "@friggframework/prettier-config": "2.0.0--canary.656.4fb07b4.0",
-        "@friggframework/test": "2.0.0--canary.656.4fb07b4.0",
+        "@friggframework/eslint-config": "2.0.0--canary.657.6a81b60.0",
+        "@friggframework/prettier-config": "2.0.0--canary.657.6a81b60.0",
+        "@friggframework/test": "2.0.0--canary.657.6a81b60.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
+        "aws-sdk-client-mock": "^4.1.0",
         "chai": "^4.3.6",
         "eslint": "^8.22.0",
         "eslint-plugin-import": "^2.29.1",
@@ -89,5 +91,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "4fb07b4e92324bbbf0d19153e2168642facc1182"
+    "gitHead": "6a81b603e102e18dd02eaca5cb89bf04f43b18a0"
 }

package/queues/queue-deferral.js

@@ -0,0 +1,69 @@
+const { toPositiveInteger } = require('./queue-delivery');
+
+const MAX_DEFERRALS = 30;
+const MAX_DEFERRED_MS = 26 * 60 * 60 * 1000;
+const MAX_DELAY_SECONDS = 900;
+const MAX_VISIBILITY_TIMEOUT_SECONDS = 43_200;
+const MAX_LAMBDA_RUN_SECONDS = 900;
+const MAX_BATCHING_WINDOW_SECONDS = 300;
+const MAX_DEFERRED_VISIBILITY_SECONDS =
+    MAX_VISIBILITY_TIMEOUT_SECONDS -
+    MAX_LAMBDA_RUN_SECONDS -
+    MAX_BATCHING_WINDOW_SECONDS;
+const MAX_DEFERRALS_ENV = 'FRIGG_QUEUE_MAX_DEFERRALS';
+const MAX_DEFERRED_MS_ENV = 'FRIGG_QUEUE_MAX_DEFERRED_MS';
+
+const readDeferralLimits = (env = process.env) => ({
+    maxDeferrals: toPositiveInteger(env[MAX_DEFERRALS_ENV]) ?? MAX_DEFERRALS,
+    maxDeferredMs:
+        toPositiveInteger(env[MAX_DEFERRED_MS_ENV]) ?? MAX_DEFERRED_MS,
+});
+
+const readDeferral = (body) => {
+    const frigg = body?._frigg;
+    const deferrals =
+        Number.isInteger(frigg?.deferrals) && frigg.deferrals > 0
+            ? frigg.deferrals
+            : 0;
+    const firstDeferredAt =
+        typeof frigg?.firstDeferredAt === 'string' &&
+        !Number.isNaN(Date.parse(frigg.firstDeferredAt))
+            ? frigg.firstDeferredAt
+            : undefined;
+    return { deferrals, firstDeferredAt };
+};
+
+const nextDeferral = (body, now = Date.now()) => {
+    const current = readDeferral(body);
+    return {
+        deferrals: current.deferrals + 1,
+        firstDeferredAt: current.firstDeferredAt ?? new Date(now).toISOString(),
+    };
+};
+
+const withDeferral = (body, deferral) => ({
+    ...body,
+    _frigg: { ...body._frigg, ...deferral },
+});
+
+const deferredMs = ({ firstDeferredAt, retryAt }) =>
+    retryAt.getTime() - Date.parse(firstDeferredAt);
+
+const isDeferralCapped = (deferral, { maxDeferrals, maxDeferredMs }) =>
+    deferral.deferrals > maxDeferrals || deferredMs(deferral) > maxDeferredMs;
+
+module.exports = {
+    MAX_DEFERRALS,
+    MAX_DEFERRALS_ENV,
+    MAX_DEFERRED_MS,
+    MAX_DEFERRED_MS_ENV,
+    MAX_DEFERRED_VISIBILITY_SECONDS,
+    MAX_DELAY_SECONDS,
+    MAX_VISIBILITY_TIMEOUT_SECONDS,
+    deferredMs,
+    isDeferralCapped,
+    nextDeferral,
+    readDeferral,
+    readDeferralLimits,
+    withDeferral,
+};

package/queues/queue-delivery.js

@@ -38,4 +38,5 @@ module.exports = {
     INTEGRATION_QUEUE_MAX_RECEIVE_COUNT,
     QUEUE_MAX_RECEIVE_COUNT_ENV,
     readQueueDelivery,
+    toPositiveInteger,
 };

package/queues/queuer-util.js

@@ -126,4 +126,4 @@ const QueuerUtil = {
     },
 };
 
-module.exports = { QueuerUtil };
+module.exports = { QueuerUtil, awsConfigOptions };

package/types/core/index.d.ts

@@ -47,8 +47,23 @@ declare module "@friggframework/core" {
     send(params: object & { QueueUrl: any }, delay?: number): Promise<string>;
 
     sendAsyncSQSMessage(params: SendSQSMessageParams): Promise<string>;
+
+    recordRateLimitWait(
+      body: object,
+      error: Error,
+      state: RateLimitWaitState
+    ): Promise<void>;
+
+    clearRateLimitWait(body: object): Promise<void>;
   }
 
+  export type RateLimitWaitState = {
+    status: "WAITING" | "EXHAUSTED";
+    mechanism: "delay" | "schedule" | "visibility" | "none";
+    deferrals: number;
+    retryAt: Date;
+  };
+
   interface IWorker {
     getQueueURL(params: GetQueueURLParams): Promise<string | undefined>;
     run(params: { Records: any }, context?: object): Promise<BatchItemFailuresResponse>;