@friggframework/core 2.0.0--canary.654.70623e2.0 → 2.0.0--canary.656.10f1676.0

package/CLAUDE.md

@@ -211,7 +211,56 @@ packages/core/
 - `integration-repository-factory.js` - Creates database-specific repositories
 - `integration-repository-mongo.js` - MongoDB implementation
 - `integration-repository-postgres.js` - PostgreSQL implementation
-- `integration-mapping-repository-*.js` - Mapping data persistence
+- `integration-mapping-repository-*.js` - Mapping data persistence.
+  `queryMappings(integrationId, { where, orderBy, skip, take, omit })`
+  returns `{ mappings, total }`: one filtered, ordered page of an
+  integration's mappings, for callers that cannot load every row through
+  `findMappingsByIntegration`. `where` is an array of ANDed conditions
+  (at most 20); an entry may be `{ anyOf: [...] }`, ORed, one level deep.
+  Conditions are `{ path: 'mapping.<segment>...', op: 'exists' | 'notExists' }`
+  (JSON null counts as absent), `{ path: 'mapping.…', op: 'in', value: string[] }`
+  (1–500 strings) and `{ path: 'sourceId', op: 'notStartsWith', value }`
+  (a NULL sourceId matches). `orderBy` is `{ path: 'mapping.…', direction:
+  'asc' | 'desc' }`, nulls last, ties broken by id in the same direction;
+  without it rows come in id order. `take` is 1–500. `omit` lists top-level
+  mapping keys to leave out of the rows; never write such rows back. Path
+  segments must match `^[A-Za-z_][A-Za-z0-9_]*$`. The PostgreSQL, MongoDB and
+  DocumentDB adapters give the same pages and totals;
+  `integration-mapping-repository-query-parity.test.js` checks that against
+  real databases when `QUERY_MAPPINGS_PARITY_MONGO_URL` /
+  `QUERY_MAPPINGS_PARITY_POSTGRES_URL` are set. Two orderings still differ:
+  strings compare by the database collation on PostgreSQL and by code point
+  on MongoDB and DocumentDB, and arrays or objects at the sort path order
+  among themselves only on PostgreSQL (the other adapters fall back to id).
+  The legacy `IntegrationMappingRepository` inherits the port's
+  "not supported by this database adapter yet" error. Every adapter refuses
+  to run while field-level encryption still encrypts
+  `IntegrationMapping.mapping` on write (see `database/encryption/README.md`),
+  and returns rows decrypted like `findMappingsByIntegration`. PostgreSQL and
+  MongoDB decrypt through `decryptQueriedMappings`
+  (`database/encryption/integration-mapping-encryption.js`), a no-op while
+  encryption is off or lists no `IntegrationMapping` field besides `mapping`.
+  Validation lives in `integration-mapping-query.js`; a new operator is one
+  entry in its `OPERATORS` table, one in the Postgres adapter's
+  `CONDITION_SQL` and one in `CONDITION_EXPRESSIONS` in
+  `integration-mapping-query-pipeline.js`, the aggregation stages both
+  MongoDB-protocol adapters share. Those stages may only use what Amazon
+  DocumentDB 4.0 and 5.0 support (no `$facet`, `$getField`, `$set` or
+  `$unset`).
+  **Cost**: PostgreSQL answers in one SQL statement, which reads every row
+  of the integration and evaluates the JSON paths per row, because no JSON
+  index exists. With ~4 KB mappings on PostgreSQL 16 that is about 0.3 s per
+  call at 10⁴ rows per integration and 2.5–3 s at 10⁵. MongoDB answers in one
+  aggregate: the `integrationId` index narrows `$match` to the integration,
+  the `$expr` is then evaluated per document, and `$facet` returns the page
+  with its `$count`, so a page (after `omit`) must fit the 16 MB document
+  limit. DocumentDB has no `$facet`: the page and the count are two
+  concurrent aggregates, so the total does not come from the same snapshot
+  as the page. Both sort in memory on a computed key (`allowDiskUse`). With
+  ~4 KB mappings on MongoDB 7 that is about 0.05–0.1 s per call at 10⁴ rows
+  per integration and 0.5–0.7 s at 10⁵ (the DocumentDB pipeline, run on
+  MongoDB, 0.6–0.8 s). On every adapter a deep offset or an empty page past
+  the end costs about the same as the first page.
 - `process-repository-*.js` - Process (long-running job) persistence.
   Implements `applyProcessUpdate(processId, ops)` — a race-safe alternative
   to `update(id, patch)` that routes increments, sets, and bounded-array
@@ -320,6 +369,48 @@ class MyIntegration extends IntegrationBase {
 - `ApiKeyRequester` - API key authentication
 - `BasicAuthRequester` - Basic authentication
 
+**Rate limits** (ADR-049): an API module declares `static rateLimit` on its
+`Requester` subclass. For a throttled response the Requester looks for a hint
+in this order:
+
+| Step | Source |
+|---|---|
+| 1 | `classify({ status, headers, body })` of the module. It recognises a limit that is not a plain 429 (a 403 limit code, a `policyName` in the body) and may name only a `reason` |
+| 2 | The header parsers in `parsers` order. Default: `retryAfter`, `resetHeaders`, `ietf` |
+| 3 | The static policy: `minRetryAfterMs`, or the `perMs` of the window named by the `reason` |
+| 4 | The fixed `backOff` ladder (1, 3, 10, 30, 60, 180 s) |
+
+- A 429 with no hint from steps 1 to 3 keeps today's ladder: the same calls,
+  the same delays, then a plain `FetchError`. Nothing budgets it.
+- With a hint, the wait is `max(hint, minRetryAfterMs, 1 s)` plus at most 10 %
+  jitter. The total sleep of one request is capped at `maxInProcessWaitMs`
+  (default 5 minutes) and at the time left in the invocation less one request
+  timeout (`remainingInvocationMs()`).
+- A wait that does not fit throws `RateLimitError` (`isRateLimited`, `retryAt`,
+  `waitMs`, `reason`, `policy`, `source`, `module`, `scopeKey`). The queue
+  worker does not halt it, whatever its status.
+- `Retry-After` is read on a 429 only. Another status counts only when
+  `classify` recognises it. A 5xx keeps the 5xx ladder.
+- `maxInProcessWaitMs: 0` turns off sleeping: every hinted wait throws.
+- A module that does not use the Requester (for example one on jsforce) calls
+  `classifyRateLimit(policy, { status, headers, body })` around its own client
+  and throws `RateLimitError` itself.
+
+```javascript
+class Api extends OAuth2Requester {
+    static rateLimit = {
+        scope: 'entity',
+        minRetryAfterMs: 1_000,
+        classify({ status, body }) {
+            if (status === 403 && body?.code === 'DAILY_LIMIT') {
+                return { reason: 'daily', waitMs: 60 * 60_000 };
+            }
+            return null;
+        },
+    };
+}
+```
+
 **Module Factory**:
 - `ModuleFactory` - Creates and configures API module instances
 - Handles credential injection
@@ -414,6 +505,44 @@ const decrypted = await cryptor.decrypt(encrypted);
 - `integration-event-dispatcher.js` - Routes events to integration handlers
 - Supports lifecycle events and user actions
 
+**Queue handler delivery**: a handler dispatched from the integration queue
+(any `this.events` entry reached through SQS, including `ON_WEBHOOK` and
+scheduled jobs) receives `{ data, context, delivery }`. HTTP-dispatched
+handlers (`{ req, res, next }`) and `this.on` events do not.
+
+| Field | Type | Value |
+|---|---|---|
+| `delivery.receiveCount` | `number \| undefined` | SQS `ApproximateReceiveCount` of this delivery |
+| `delivery.maxReceiveCount` | `number \| undefined` | Receives allowed before SQS moves the message to the DLQ (`FRIGG_QUEUE_MAX_RECEIVE_COUNT`) |
+| `delivery.isLastAttempt` | `boolean` | `true` only when both counts are known and `receiveCount >= maxReceiveCount` |
+
+`isLastAttempt` is `false` when either count is unknown: a local or non-SQS
+invocation, or a queue whose redrive policy the stack does not own
+(`ownership.queue: 'external'`). The value is information only: core still
+rethrows retryable errors and discards halt errors (4xx except 408, 429 and
+errors with `isRateLimited`).
+
+Use it to end a run or count lost work on the final try: when a retryable
+error (429, 5xx, network) is about to be rethrown and `isLastAttempt` is
+`true`, the message goes to the DLQ next, so mark the run failed or count the
+message's records as failed before rethrowing.
+
+```javascript
+async processBatch({ data, delivery }) {
+    try {
+        await this.syncPage(data);
+    } catch (error) {
+        if (delivery?.isLastAttempt) await this.failRun(data.processId, error);
+        throw error;
+    }
+}
+```
+
+The single source of the max receive count is
+`INTEGRATION_QUEUE_MAX_RECEIVE_COUNT` in `queues/queue-delivery.js`. The
+devtools integration builder uses it for the queue's `RedrivePolicy` and sets
+it as `FRIGG_QUEUE_MAX_RECEIVE_COUNT` on the queue worker function.
+
 ### 8. Error Handling (`/errors`)
 
 **Purpose**: Standardized error types with proper HTTP semantics.
@@ -421,6 +550,7 @@ const decrypted = await cryptor.decrypt(encrypted);
 **Error Types**:
 - `BaseError` - Base error class
 - `FetchError` - HTTP request failures
+- `RateLimitError` - A `FetchError` for a limit the provider or the module's policy says when to retry (`retryAt`)
 - `HaltError` - Stop processing without retry
 - `RequiredPropertyError` - Missing required parameters
 - `ParameterTypeError` - Invalid parameter type
@@ -663,6 +793,7 @@ Use test doubles from `@friggframework/test` package for consistent mocking.
 
 - `SECRET_ARN` - AWS Secrets Manager ARN for auto-injection
 - `FRIGG_LOG_LEVEL` - Minimum log level (`TRACE` … `FATAL`, default `INFO`; `DEBUG` on local runs)
+- `FRIGG_QUEUE_MAX_RECEIVE_COUNT` - Set by devtools on integration queue workers whose queue the stack owns; feeds `delivery.maxReceiveCount`
 
 ## Version Information
 

package/core/CLAUDE.md

@@ -52,9 +52,10 @@ const handler = createHandler({
 **Usage Pattern**:
 ```javascript
 class MyWorker extends Worker {
-    async _run(params, context = {}) {
+    async _run(params, context = {}, delivery) {
         // Your job processing logic here
         // params are already JSON.parsed from SQS message body
+        // delivery is { receiveCount, maxReceiveCount, isLastAttempt }
     }
     
     _validateParams(params) {
@@ -129,7 +130,7 @@ class MyIntegration extends Delegate {
 ### Lambda Handler Lifecycle
 1. **Pre-Execution Setup**:
    ```javascript
-   runInvocationScope({ requestId, handlerName, method, route, invocation }, …); // Logger scope for the invocation
+   runInvocationScope({ requestId, handlerName, method, route, invocation }, …); // Logger scope + invocation deadline (remainingInvocationMs())
    log.info('Handler invoked', { eventName: 'frigg.handler.invoked' });
    await secretsToEnv();                     // Secrets Manager injection
    await parametersToEnv();                  // SSM Parameter Store fetch (only when SSM_PARAMETER_PREFIX + FRIGG_SSM_OFFLOADED_KEYS are set)
@@ -225,6 +226,8 @@ const handler = createHandler({
    - Logs error but returns success
    - Prevents infinite retries for known issues
    - Used for graceful degradation scenarios
+   - The queue worker sets it on a 4xx except 408, 429 and errors with
+     `isRateLimited` (a provider limit signalled with 403 clears with time)
 
 ### Logging Strategy
 ```javascript

package/core/Worker.js

@@ -2,6 +2,7 @@ const { SQSClient, GetQueueUrlCommand, SendMessageCommand } = require('@aws-sdk/
 const _ = require('lodash');
 const { RequiredPropertyError } = require('../errors');
 const { get } = require('../assertions');
+const { readQueueDelivery } = require('../queues/queue-delivery');
 const { runMessageScope } = require('./invocation-scope');
 const { getLogger } = require('../logs');
 
@@ -35,7 +36,7 @@ class Worker {
                 try {
                     const runParams = JSON.parse(record.body);
                     this._validateParams(runParams);
-                    await this._run(runParams, context);
+                    await this._run(runParams, context, readQueueDelivery(record));
                     log.debug('Record succeeded', { eventName: 'frigg.worker.record_succeeded' });
                 } catch (error) {
                     if (error.isHaltError) {

package/core/index.js

@@ -3,6 +3,10 @@ const { Worker } = require('./Worker');
 const { loadInstalledModules } = require('./load-installed-modules');
 const { createHandler } = require('./create-handler');
 const { runInvocationScope } = require('./invocation-scope');
+const {
+    runWithInvocationDeadline,
+    remainingInvocationMs,
+} = require('./invocation-deadline');
 
 module.exports = {
     Delegate,
@@ -10,4 +14,6 @@ module.exports = {
     loadInstalledModules,
     createHandler,
     runInvocationScope,
+    runWithInvocationDeadline,
+    remainingInvocationMs,
 };

package/core/invocation-deadline.js

@@ -0,0 +1,56 @@
+const { AsyncLocalStorage } = require('node:async_hooks');
+
+const STORE_KEY = Symbol.for('@friggframework/core.invocation-deadline');
+
+function storage() {
+    if (!globalThis[STORE_KEY]) {
+        globalThis[STORE_KEY] = new AsyncLocalStorage();
+    }
+    return globalThis[STORE_KEY];
+}
+
+/**
+ * Runs fn with the time the invocation ends, as epoch milliseconds. A nested
+ * scope can shorten the deadline and never extend it. A deadline that is not
+ * a finite number runs fn with no limit.
+ */
+function runWithInvocationDeadline(deadlineAt, fn) {
+    if (typeof deadlineAt !== 'number' || !Number.isFinite(deadlineAt)) {
+        return fn();
+    }
+    const outer = storage().getStore();
+    const effective = outer
+        ? Math.min(outer.deadlineAt, deadlineAt)
+        : deadlineAt;
+    return storage().run(Object.freeze({ deadlineAt: effective }), fn);
+}
+
+/**
+ * The end of a Lambda invocation, from its context. Undefined when the
+ * context cannot tell.
+ */
+function deadlineFromContext(context) {
+    try {
+        const remaining = context?.getRemainingTimeInMillis?.();
+        return typeof remaining === 'number' && Number.isFinite(remaining)
+            ? Date.now() + remaining
+            : undefined;
+    } catch {
+        return undefined;
+    }
+}
+
+/**
+ * Milliseconds left in the invocation. Infinity outside a deadline scope
+ * (tests, `frigg start`, scripts).
+ */
+function remainingInvocationMs(now = Date.now()) {
+    const store = storage().getStore();
+    return store ? Math.max(0, store.deadlineAt - now) : Infinity;
+}
+
+module.exports = {
+    deadlineFromContext,
+    remainingInvocationMs,
+    runWithInvocationDeadline,
+};

package/core/invocation-scope.js

@@ -1,6 +1,10 @@
 const { runInContext, LOGGER_SCOPE_KEY } = require('../logs/context');
 const { summarizeMessageBody } = require('../logs/summarize-event');
 const { flushSinks, hasFlushableSinks } = require('../logs/logger-runtime');
+const {
+    deadlineFromContext,
+    runWithInvocationDeadline,
+} = require('./invocation-deadline');
 
 // Bounds the tail latency telemetry adds to every warm invocation. Kept low so
 // an unreachable OTLP endpoint (e.g. a VPC Lambda with no NAT/egress) costs at
@@ -168,21 +172,26 @@ async function runInvocationScope(fields, fn, opts = {}) {
         flushTimeoutMs = DEFAULT_FLUSH_TIMEOUT_MS,
         context,
     } = opts;
-    return runInContext({ [LOGGER_SCOPE_KEY]: { ...(fields || {}) } }, async () => {
-        try {
-            return await fn();
-        } finally {
-            // Every step is guarded, so a flush never changes the result.
-            await flushInvocation({
-                telemetry,
-                usageRollup,
-                eventSummary,
-                shouldUseDatabase,
-                flushTimeoutMs,
-                context,
-            });
-        }
-    });
+    return runWithInvocationDeadline(deadlineFromContext(context), () =>
+        runInContext(
+            { [LOGGER_SCOPE_KEY]: { ...(fields || {}) } },
+            async () => {
+                try {
+                    return await fn();
+                } finally {
+                    // Every step is guarded, so a flush never changes the result.
+                    await flushInvocation({
+                        telemetry,
+                        usageRollup,
+                        eventSummary,
+                        shouldUseDatabase,
+                        flushTimeoutMs,
+                        context,
+                    });
+                }
+            }
+        )
+    );
 }
 
 function toCount(value) {

package/database/documentdb-encryption-service.js

@@ -1,5 +1,9 @@
 const { Cryptor } = require('../encrypt/Cryptor');
-const { getEncryptedFields, loadCustomEncryptionSchema } = require('./encryption/encryption-schema-registry');
+const {
+    getEncryptedFields,
+    getFieldsToEncryptOnWrite,
+    loadCustomEncryptionSchema,
+} = require('./encryption/encryption-schema-registry');
 
 /**
  * Encryption service specifically for DocumentDB repositories
@@ -107,7 +111,7 @@ class DocumentDBEncryptionService {
         }
 
         // Get encrypted fields from registry
-        const encryptedFieldsConfig = getEncryptedFields(modelName);
+        const encryptedFieldsConfig = getFieldsToEncryptOnWrite(modelName);
         if (!encryptedFieldsConfig || encryptedFieldsConfig.length === 0) {
             return document;
         }

package/database/documentdb-utils.js

@@ -166,11 +166,12 @@ async function findManyDrained(client, collection, filter = {}, options = {}) {
     return drainCursor(client, collection, first);
 }
 
-async function aggregateDrained(client, collection, pipeline) {
+async function aggregateDrained(client, collection, pipeline, options = {}) {
     const first = await client.$runCommandRaw({
         aggregate: collection,
         pipeline,
         cursor: { batchSize: DRAIN_BATCH_SIZE },
+        ...options,
     });
     return drainCursor(client, collection, first);
 }

package/database/encryption/README.md

@@ -818,6 +818,26 @@ export AES_KEY=$(openssl rand -hex 16)  # Generate 32-char key
 ❌ Query on encrypted fields (not supported)
 ❌ Manually decrypt data (use extension)
 
+The integration mapping repositories' `queryMappings()` (PostgreSQL, MongoDB
+and DocumentDB) filters and sorts inside the `mapping` JSON, so it refuses to
+run while field-level encryption still encrypts `IntegrationMapping.mapping` on
+write. Opt the field out in the app definition, together with any nested
+`mapping.*` path that `encryption.schema` encrypts; the Prisma extension and
+`DocumentDBEncryptionService` both honor the opt-out:
+
+```javascript
+encryption: {
+    disable: { IntegrationMapping: ['mapping'] },
+}
+```
+
+The opt-out applies to writes only. Rows written encrypted before it stay
+readable, but `queryMappings()` does not match them until they are written
+again. It returns the rows it matches decrypted, like
+`findMappingsByIntegration()`: a nested `mapping.*` path written encrypted
+before its opt-out reads plain, though conditions and ordering on that path
+still see the ciphertext until the row is written again.
+
 ## Future Enhancements
 
 ### Planned

package/database/encryption/integration-mapping-encryption.js

@@ -0,0 +1,97 @@
+const { getEncryptionConfig } = require('../prisma');
+const { Cryptor } = require('../../encrypt/Cryptor');
+const {
+    getFieldsToDecryptOnRead,
+    getFieldsToEncryptOnWrite,
+    loadCustomEncryptionSchema,
+} = require('./encryption-schema-registry');
+const {
+    createFieldEncryptionService,
+} = require('./prisma-encryption-extension');
+
+let plainMappingEncryption = null;
+
+/**
+ * The `IntegrationMapping` fields, `mapping` itself or a nested `mapping.*`
+ * path, that field-level encryption encrypts on write. An empty result is
+ * kept for the life of the process.
+ *
+ * @returns {string[]} Empty when every mapping path is written as plain JSON
+ */
+function getMappingFieldsEncryptedOnWrite() {
+    return mappingEncryption().encryptedOnWrite;
+}
+
+/**
+ * @throws {Error} When field-level encryption still encrypts `mapping`, or a
+ *   nested `mapping.*` path, on write, naming the opt-out that lifts it
+ */
+function assertMappingWrittenUnencrypted() {
+    const fields = getMappingFieldsEncryptedOnWrite();
+    if (fields.length === 0) return;
+
+    const encrypted = fields
+        .map((field) => `IntegrationMapping.${field}`)
+        .join(', ');
+    const optOut = fields.map((field) => `'${field}'`).join(', ');
+    throw new Error(
+        `queryMappings: field-level encryption still encrypts ${encrypted} on write, so it cannot be queried. Opt out by adding ${optOut} to appDefinition.encryption.disable.IntegrationMapping.`
+    );
+}
+
+/**
+ * Decrypts rows that `queryMappings` read around the Prisma encryption
+ * extension, the way reads through the extension do: every
+ * `IntegrationMapping` field the schema lists, opted-out paths included, so a
+ * path written encrypted before its opt-out comes back plain.
+ *
+ * @param {Object[]} rows - Rows whose `mapping` is a JSON object
+ * @returns {Promise<Object[]>} `rows` itself when encryption is off or the
+ *   schema lists no field besides `mapping`
+ */
+async function decryptQueriedMappings(rows) {
+    const { decryptor } = mappingEncryption();
+    if (!decryptor) return rows;
+    return decryptor.decryptFieldsInBulk('IntegrationMapping', rows);
+}
+
+function mappingEncryption() {
+    if (plainMappingEncryption) return plainMappingEncryption;
+
+    const encryption = currentMappingEncryption();
+    if (encryption.encryptedOnWrite.length === 0) {
+        plainMappingEncryption = encryption;
+    }
+    return encryption;
+}
+
+function currentMappingEncryption() {
+    const config = getEncryptionConfig();
+    if (!config.enabled) return { encryptedOnWrite: [], decryptor: null };
+
+    loadCustomEncryptionSchema();
+    const encryptedOnWrite = getFieldsToEncryptOnWrite(
+        'IntegrationMapping'
+    ).filter((field) => field === 'mapping' || field.startsWith('mapping.'));
+    const decryptsBesidesMapping = getFieldsToDecryptOnRead(
+        'IntegrationMapping'
+    ).some((field) => field !== 'mapping');
+    const decryptor = decryptsBesidesMapping
+        ? createFieldEncryptionService(
+              new Cryptor({ shouldUseAws: config.method === 'kms' })
+          )
+        : null;
+    return { encryptedOnWrite, decryptor };
+}
+
+/** Test helper: forget a kept result. */
+function resetMappingEncryptionCheck() {
+    plainMappingEncryption = null;
+}
+
+module.exports = {
+    assertMappingWrittenUnencrypted,
+    decryptQueriedMappings,
+    getMappingFieldsEncryptedOnWrite,
+    resetMappingEncryptionCheck,
+};

package/database/encryption/prisma-encryption-extension.js

@@ -10,6 +10,24 @@ const {
 } = require('./encryption-schema-registry');
 const { FieldEncryptionService } = require('./field-encryption-service');
 
+/**
+ * The FieldEncryptionService the extension runs, with field paths from the
+ * encryption schema registry.
+ *
+ * @param {import('../../encrypt/Cryptor').Cryptor} cryptor
+ * @returns {FieldEncryptionService}
+ */
+function createFieldEncryptionService(cryptor) {
+    return new FieldEncryptionService({
+        cryptor,
+        schema: {
+            getEncryptedFields,
+            getFieldsToEncryptOnWrite,
+            getFieldsToDecryptOnRead,
+        },
+    });
+}
+
 function createEncryptionExtension({ cryptor, enabled = true }) {
     if (!enabled) {
         return (client) => client;
@@ -21,14 +39,7 @@ function createEncryptionExtension({ cryptor, enabled = true }) {
         );
     }
 
-    const encryptionService = new FieldEncryptionService({
-        cryptor,
-        schema: {
-            getEncryptedFields,
-            getFieldsToEncryptOnWrite,
-            getFieldsToDecryptOnRead,
-        },
-    });
+    const encryptionService = createFieldEncryptionService(cryptor);
 
     return {
         name: 'frigg-field-encryption',
@@ -227,4 +238,4 @@ function createEncryptionExtension({ cryptor, enabled = true }) {
     };
 }
 
-module.exports = { createEncryptionExtension };
+module.exports = { createEncryptionExtension, createFieldEncryptionService };

package/errors/fetch-error.js

@@ -55,12 +55,18 @@ class FetchError extends BaseError {
 
     static async create(options = {}) {
         const { response } = options;
-        let responseBody =
-            response && !response.bodyUsed && typeof response.text === 'function'
-                ? await response.text()
-                : null;
-        if (!responseBody) responseBody = options.responseBody ?? options.body;
-        return new FetchError({ ...options, responseBody });
+        const provided = options.responseBody ?? options.body;
+        let responseBody = provided;
+        if (
+            !responseBody &&
+            response &&
+            !response.bodyUsed &&
+            typeof response.text === 'function'
+        ) {
+            responseBody = (await response.text()) || provided;
+        }
+        const ErrorClass = typeof this === 'function' ? this : FetchError;
+        return new ErrorClass({ ...options, responseBody });
     }
 }
 

package/errors/index.js

@@ -1,6 +1,7 @@
 const { BaseError } = require('./base-error');
 const { FetchError } = require('./fetch-error');
 const { HaltError } = require('./halt-error');
+const { RateLimitError } = require('./rate-limit-error');
 const {
     RequiredPropertyError,
     ParameterTypeError,
@@ -11,6 +12,7 @@ module.exports = {
     BaseError,
     FetchError,
     HaltError,
+    RateLimitError,
     RequiredPropertyError,
     ParameterTypeError,
     ClientSafeError,

package/errors/rate-limit-error.js

@@ -0,0 +1,44 @@
+const { FetchError } = require('./fetch-error');
+
+function resolveTiming({ hint, waitMs, now }) {
+    if (waitMs !== undefined) {
+        return { waitMs, retryAt: new Date(now + waitMs) };
+    }
+    if (hint?.retryAt) {
+        const retryAt = new Date(hint.retryAt);
+        if (!Number.isNaN(retryAt.getTime())) {
+            return { waitMs: Math.max(0, retryAt.getTime() - now), retryAt };
+        }
+    }
+    const fallbackWaitMs = hint?.waitMs ?? 0;
+    return { waitMs: fallbackWaitMs, retryAt: new Date(now + fallbackWaitMs) };
+}
+
+/**
+ * A FetchError for a response that said a limit was hit and told us, or the
+ * module's policy told us, when to call again. `retryAt` is that time.
+ */
+class RateLimitError extends FetchError {
+    constructor({
+        hint,
+        waitMs,
+        module,
+        scopeKey,
+        now = Date.now(),
+        ...fetchErrorArgs
+    } = {}) {
+        super(fetchErrorArgs);
+
+        const timing = resolveTiming({ hint, waitMs, now });
+        this.isRateLimited = true;
+        this.waitMs = timing.waitMs;
+        this.retryAt = timing.retryAt;
+        this.reason = hint?.reason ?? 'unknown';
+        this.policy = hint?.policy;
+        this.source = hint?.source ?? 'unknown';
+        this.module = module;
+        this.scopeKey = scopeKey;
+    }
+}
+
+module.exports = { RateLimitError };

package/handlers/backend-utils.js

@@ -165,7 +165,7 @@ const createQueueWorker = (integrationClass) => {
     const integrationName = integrationClass.Definition.name;
 
     class QueueWorker extends Worker {
-        async _run(params, context) {
+        async _run(params, context, delivery) {
             const logCtx = {
                 integration: integrationName,
                 event: params.event,
@@ -255,6 +255,7 @@ const createQueueWorker = (integrationClass) => {
                     event: params.event,
                     data: params.data,
                     context: context,
+                    delivery,
                 });
                 console.log(
                     `[QueueWorker] ${params.event} dispatched ok`,
@@ -284,7 +285,8 @@ const createQueueWorker = (integrationClass) => {
                     status >= 400 &&
                     status < 500 &&
                     status !== 408 &&
-                    status !== 429
+                    status !== 429 &&
+                    !error.isRateLimited
                 ) {
                     error.isHaltError = true;
                     console.warn(