@friggframework/core 2.0.0--canary.657.b14221f.0 → 2.0.0--canary.658.93c8e07.0

package/integrations/repositories/integration-repository-documentdb.js

@@ -14,6 +14,7 @@ const {
     IntegrationRepositoryInterface,
 } = require('./integration-repository-interface');
 const { validateConfigPatch } = require('./config-patch-shared');
+const { toMessageItem } = require('./message-item-shared');
 
 class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
     constructor() {
@@ -94,7 +95,7 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
     async updateIntegrationMessages(
         integrationId,
         messageType,
-        messageTitle,
+        messageTitleOrItem,
         messageBody,
         messageTimestamp
     ) {
@@ -108,11 +109,12 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
         }
         const messages = this._extractMessages(existing);
         const list = Array.isArray(messages[messageType]) ? [...messages[messageType]] : [];
-        list.push({
-            title: messageTitle ?? null,
-            message: messageBody,
-            timestamp: messageTimestamp,
-        });
+        const item = toMessageItem(
+            messageTitleOrItem,
+            messageBody,
+            messageTimestamp
+        );
+        list.push({ ...item, title: item.title ?? null });
         const updatedMessages = { ...messages, [messageType]: list };
         await updateOne(
             this.prisma,
@@ -132,6 +134,23 @@ class IntegrationRepositoryDocumentDB extends IntegrationRepositoryInterface {
         return true;
     }
 
+    async findIntegrationMessages(integrationId, messageType) {
+        const objectId = toObjectId(integrationId);
+        if (!objectId) {
+            throw new Error(`Integration ${integrationId} not found`);
+        }
+        const existing = await findOne(this.prisma, 'Integration', {
+            _id: objectId,
+        });
+        if (!existing) {
+            throw new Error(`Integration ${integrationId} not found`);
+        }
+        const messages = this._extractMessages(existing);
+        return Array.isArray(messages[messageType])
+            ? messages[messageType]
+            : [];
+    }
+
     async createIntegration(entities, userId, config) {
         const now = new Date();
         const document = {

package/integrations/repositories/integration-repository-interface.js

@@ -97,26 +97,43 @@ class IntegrationRepositoryInterface {
     }
 
     /**
-     * Update integration messages
+     * Append a message to the stored messages of one type. Takes the
+     * positional form (title, body, timestamp) or one message item object,
+     * whose keys are stored as they are.
      *
      * @param {string|number} integrationId - Integration ID
      * @param {string} messageType - Type of message (errors, warnings, info, logs)
-     * @param {string} messageTitle - Message title
-     * @param {string} messageBody - Message body
-     * @param {Date} messageTimestamp - Message timestamp
+     * @param {string|Object} messageTitleOrItem - Message title, or the whole message item
+     * @param {string} [messageBody] - Message body (positional form)
+     * @param {Date} [messageTimestamp] - Message timestamp (positional form)
      * @returns {Promise<boolean>} Success indicator
      * @abstract
      */
     async updateIntegrationMessages(
         integrationId,
         messageType,
-        messageTitle,
+        messageTitleOrItem,
         messageBody,
         messageTimestamp
     ) {
         throw new Error('Method updateIntegrationMessages must be implemented by subclass');
     }
 
+    /**
+     * Find the stored messages of one type. Reads the stored items, not the
+     * `messages` of an integration record, which some adapters do not fill.
+     *
+     * @param {string|number} integrationId - Integration ID
+     * @param {string} messageType - Type of message (errors, warnings, info, logs)
+     * @returns {Promise<Array>} The stored message items of that type, oldest first
+     * @abstract
+     */
+    async findIntegrationMessages(integrationId, messageType) {
+        throw new Error(
+            'Method findIntegrationMessages must be implemented by subclass'
+        );
+    }
+
     /**
      * Create a new integration
      *

package/integrations/repositories/integration-repository-mongo.js

@@ -3,6 +3,7 @@ const {
     IntegrationRepositoryInterface,
 } = require('./integration-repository-interface');
 const { validateConfigPatch } = require('./config-patch-shared');
+const { toMessageItem } = require('./message-item-shared');
 
 /**
  * MongoDB Integration Repository Adapter
@@ -229,15 +230,15 @@ class IntegrationRepositoryMongo extends IntegrationRepositoryInterface {
      *
      * @param {string} integrationId - Integration ID
      * @param {string} messageType - Type of message (errors, warnings, info, logs)
-     * @param {string} messageTitle - Message title
-     * @param {string} messageBody - Message body
-     * @param {Date} messageTimestamp - Message timestamp
+     * @param {string|Object} messageTitleOrItem - Message title, or the whole message item
+     * @param {string} [messageBody] - Message body (positional form)
+     * @param {Date} [messageTimestamp] - Message timestamp (positional form)
      * @returns {Promise<boolean>} Success indicator
      */
     async updateIntegrationMessages(
         integrationId,
         messageType,
-        messageTitle,
+        messageTitleOrItem,
         messageBody,
         messageTimestamp
     ) {
@@ -251,17 +252,14 @@ class IntegrationRepositoryMongo extends IntegrationRepositoryInterface {
         }
 
         // Parse existing messages (JSON field)
-        const messages = integration.messages || {};
-        const messageArray = Array.isArray(messages[messageType])
-            ? messages[messageType]
+        const messageArray = Array.isArray(integration[messageType])
+            ? integration[messageType]
             : [];
 
         // Add new message
-        messageArray.push({
-            title: messageTitle,
-            message: messageBody,
-            timestamp: messageTimestamp,
-        });
+        messageArray.push(
+            toMessageItem(messageTitleOrItem, messageBody, messageTimestamp)
+        );
 
         // Update messages
         await this.prisma.integration.update({
@@ -274,6 +272,28 @@ class IntegrationRepositoryMongo extends IntegrationRepositoryInterface {
         return true; // Mongoose compatibility
     }
 
+    /**
+     * Find the stored messages of one type
+     *
+     * @param {string} integrationId - Integration ID
+     * @param {string} messageType - Type of message (errors, warnings, info, logs)
+     * @returns {Promise<Array>} The stored message items of that type, oldest first
+     */
+    async findIntegrationMessages(integrationId, messageType) {
+        const integration = await this.prisma.integration.findUnique({
+            where: { id: integrationId },
+            select: { [messageType]: true },
+        });
+
+        if (!integration) {
+            throw new Error(`Integration ${integrationId} not found`);
+        }
+
+        return Array.isArray(integration[messageType])
+            ? integration[messageType]
+            : [];
+    }
+
     /**
      * Find all integrations whose entity set includes the given entity ID.
      *

package/integrations/repositories/integration-repository-postgres.js

@@ -3,6 +3,7 @@ const {
     IntegrationRepositoryInterface,
 } = require('./integration-repository-interface');
 const { validateConfigPatch } = require('./config-patch-shared');
+const { toMessageItem } = require('./message-item-shared');
 const { strictIntId } = require('./report-id');
 
 /**
@@ -273,15 +274,15 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface {
      *
      * @param {string} integrationId - Integration ID (string from application layer)
      * @param {string} messageType - Type of message (errors, warnings, info, logs)
-     * @param {string} messageTitle - Message title
-     * @param {string} messageBody - Message body
-     * @param {Date} messageTimestamp - Message timestamp
+     * @param {string|Object} messageTitleOrItem - Message title, or the whole message item
+     * @param {string} [messageBody] - Message body (positional form)
+     * @param {Date} [messageTimestamp] - Message timestamp (positional form)
      * @returns {Promise<boolean>} Success indicator
      */
     async updateIntegrationMessages(
         integrationId,
         messageType,
-        messageTitle,
+        messageTitleOrItem,
         messageBody,
         messageTimestamp
     ) {
@@ -297,17 +298,14 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface {
         }
 
         // Parse existing messages (JSON field)
-        const messages = integration.messages || {};
-        const messageArray = Array.isArray(messages[messageType])
-            ? messages[messageType]
+        const messageArray = Array.isArray(integration[messageType])
+            ? integration[messageType]
             : [];
 
         // Add new message
-        messageArray.push({
-            title: messageTitle,
-            message: messageBody,
-            timestamp: messageTimestamp,
-        });
+        messageArray.push(
+            toMessageItem(messageTitleOrItem, messageBody, messageTimestamp)
+        );
 
         // Update messages
         await this.prisma.integration.update({
@@ -320,6 +318,29 @@ class IntegrationRepositoryPostgres extends IntegrationRepositoryInterface {
         return true; // Mongoose compatibility
     }
 
+    /**
+     * Find the stored messages of one type
+     *
+     * @param {string} integrationId - Integration ID (string from application layer)
+     * @param {string} messageType - Type of message (errors, warnings, info, logs)
+     * @returns {Promise<Array>} The stored message items of that type, oldest first
+     */
+    async findIntegrationMessages(integrationId, messageType) {
+        const intId = this._convertId(integrationId);
+        const integration = await this.prisma.integration.findUnique({
+            where: { id: intId },
+            select: { [messageType]: true },
+        });
+
+        if (!integration) {
+            throw new Error(`Integration ${integrationId} not found`);
+        }
+
+        return Array.isArray(integration[messageType])
+            ? integration[messageType]
+            : [];
+    }
+
     /**
      * Create a new integration
      *

package/integrations/repositories/message-item-shared.js

@@ -0,0 +1,24 @@
+const isItem = (value) =>
+    value !== null && typeof value === 'object' && !Array.isArray(value);
+
+/**
+ * The stored shape of a message, for IntegrationRepository
+ * .updateIntegrationMessages(). The method takes the positional form (title,
+ * body, timestamp) or one item object. The keys of an item object are stored
+ * as they are, so a caller can attach what a client acts on, like a code or
+ * a list of actions.
+ * @param {string|Object} titleOrItem - The title, or the whole item.
+ * @param {string} [messageBody] - Ignored when an item is given.
+ * @param {number|Date} [messageTimestamp] - Ignored when an item is given.
+ * @returns {Object} A new object; the given item is never returned itself.
+ */
+function toMessageItem(titleOrItem, messageBody, messageTimestamp) {
+    if (isItem(titleOrItem)) return { ...titleOrItem };
+    return {
+        title: titleOrItem,
+        message: messageBody,
+        timestamp: messageTimestamp,
+    };
+}
+
+module.exports = { toMessageItem };

package/integrations/tests/doubles/test-integration-repository.js

@@ -1,5 +1,6 @@
 const { v4: uuid } = require('uuid');
 const { validateConfigPatch } = require('../../repositories/config-patch-shared');
+const { toMessageItem } = require('../../repositories/message-item-shared');
 
 class TestIntegrationRepository {
     constructor() {
@@ -72,18 +73,30 @@ class TestIntegrationRepository {
         return results;
     }
 
-    async updateIntegrationMessages(id, type, title, body, timestamp) {
+    async updateIntegrationMessages(id, type, titleOrItem, body, timestamp) {
         const rec = this.store.get(id);
         if (!rec) {
             this.operationHistory.push({ operation: 'updateMessages', id, success: false });
             return false;
         }
         if (!rec.messages[type]) rec.messages[type] = [];
-        rec.messages[type].push({ title, message: body, timestamp });
+        rec.messages[type].push(toMessageItem(titleOrItem, body, timestamp));
         this.operationHistory.push({ operation: 'updateMessages', id, type, success: true });
         return true;
     }
 
+    async findIntegrationMessages(id, type) {
+        const rec = this.store.get(id);
+        this.operationHistory.push({
+            operation: 'findMessages',
+            id,
+            type,
+            found: !!rec,
+        });
+        if (!rec) throw new Error(`Integration ${id} not found`);
+        return [...(rec.messages[type] ?? [])];
+    }
+
     async updateIntegrationConfig(id, config) {
         if (config === null || config === undefined) {
             throw new Error('Config parameter is required');

package/integrations/use-cases/record-rate-limit-message.js

@@ -0,0 +1,85 @@
+const RATE_LIMITED = 'RATE_LIMITED';
+const SAME_LIMIT_WINDOW_MS = 60_000;
+const MINUTE_MS = 60_000;
+
+const isLink = (link) =>
+    typeof link?.label === 'string' && typeof link?.url === 'string';
+
+const linkActions = (links) =>
+    (Array.isArray(links) ? links : [])
+        .filter(isLink)
+        .map(({ label, url }) => ({ type: 'LINK', label, url }));
+
+function utcMinuteAfter(date) {
+    const minute = new Date(Math.ceil(date.getTime() / MINUTE_MS) * MINUTE_MS);
+    return `${minute.toISOString().slice(0, 16).replace('T', ' ')} UTC`;
+}
+
+/**
+ * Use case that records one user-facing warning when a module reports a rate
+ * limit that is too long to wait for. The text is fixed and the payload's
+ * message, url, body and headers are never read, because the message reaches
+ * end users. It changes no integration status: a rate limit is not an error.
+ * @class RecordRateLimitMessage
+ */
+class RecordRateLimitMessage {
+    /**
+     * @param {Object} params - Configuration parameters.
+     * @param {import('../repositories/integration-repository-interface').IntegrationRepositoryInterface} params.integrationRepository - Repository for integration data operations.
+     */
+    constructor({ integrationRepository }) {
+        this.integrationRepository = integrationRepository;
+    }
+
+    /**
+     * Writes the warning, unless the stored warnings already hold one for the
+     * same module whose reset time is within a minute of this one. Many workers
+     * hit one limit together and each reports it, so the check reads the
+     * stored warnings and not the messages of this instance.
+     * @async
+     * @param {string} integrationId - ID of the integration to warn on.
+     * @param {Object} payload - The RATE_LIMITED payload of the Module.
+     * @param {string} payload.moduleName
+     * @param {string} payload.reason
+     * @param {Date|string|number} payload.retryAt
+     * @param {Array<{label: string, url: string}>} [payload.links]
+     * @returns {Promise<boolean>} True when it wrote a warning.
+     * @throws When the read or the write fails; the caller decides what a
+     *   failed warning is worth.
+     */
+    async execute(integrationId, { moduleName, reason, retryAt, links }) {
+        const resetAt = new Date(retryAt);
+        const warnings =
+            await this.integrationRepository.findIntegrationMessages(
+                integrationId,
+                'warnings'
+            );
+        const alreadyRecorded = warnings.some(
+            (warning) =>
+                warning?.code === RATE_LIMITED &&
+                warning.module === moduleName &&
+                Math.abs(Date.parse(warning.retryAt) - resetAt.getTime()) <=
+                    SAME_LIMIT_WINDOW_MS
+        );
+        if (alreadyRecorded) return false;
+
+        const resetsAt = utcMinuteAfter(resetAt);
+        await this.integrationRepository.updateIntegrationMessages(
+            integrationId,
+            'warnings',
+            {
+                title: 'Rate limit reached',
+                message: `The ${moduleName} API rate limit was reached and resets at ${resetsAt}.`,
+                timestamp: Date.now(),
+                code: RATE_LIMITED,
+                module: moduleName,
+                reason,
+                retryAt: resetAt.toISOString(),
+                actions: [{ type: 'RETRY_WHEN_READY' }, ...linkActions(links)],
+            }
+        );
+        return true;
+    }
+}
+
+module.exports = { RecordRateLimitMessage };

package/integrations/use-cases/update-integration-messages.js

@@ -13,19 +13,21 @@ class UpdateIntegrationMessages {
     }
 
     /**
-     * Executes the integration messages update.
+     * Executes the integration messages update. Pass the positional form
+     * (title, body, timestamp) or one message item object; the keys of an item
+     * object are stored as they are.
      * @async
      * @param {string} integrationId - ID of the integration to update.
      * @param {string} messageType - Type of message: 'errors', 'warnings', 'info', or 'logs'.
-     * @param {string} messageTitle - Title of the message.
-     * @param {string} messageBody - Body content of the message.
-     * @param {string} messageTimestamp - Timestamp when the message was created.
+     * @param {string|Object} messageTitleOrItem - Title of the message, or the whole message item.
+     * @param {string} [messageBody] - Body content of the message (positional form).
+     * @param {string} [messageTimestamp] - Timestamp when the message was created (positional form).
      * @returns {Promise<Object>} The updated integration record.
      */
     async execute(
         integrationId,
         messageType,
-        messageTitle,
+        messageTitleOrItem,
         messageBody,
         messageTimestamp
     ) {
@@ -33,7 +35,7 @@ class UpdateIntegrationMessages {
             await this.integrationRepository.updateIntegrationMessages(
                 integrationId,
                 messageType,
-                messageTitle,
+                messageTitleOrItem,
                 messageBody,
                 messageTimestamp
             );

package/modules/module.js

@@ -51,6 +51,8 @@ class Module extends Delegate {
         this.delegateTypes.push(this.DLGT_CREDENTIAL_INVALIDATED);
         this.DLGT_CREDENTIAL_VALIDATED = 'CREDENTIAL_VALIDATED';
         this.delegateTypes.push(this.DLGT_CREDENTIAL_VALIDATED);
+        this.DLGT_RATE_LIMITED = 'RATE_LIMITED';
+        this.delegateTypes.push(this.DLGT_RATE_LIMITED);
 
         Object.assign(this, this.definition.requiredAuthMethods);
 
@@ -160,6 +162,8 @@ class Module extends Delegate {
             await this.deauthorize();
         } else if (delegateString === this.api.DLGT_INVALID_AUTH) {
             await this.markCredentialsInvalid(object);
+        } else if (delegateString === this.api.DLGT_RATE_LIMITED) {
+            await this.reportRateLimit(object);
         } else if (delegateString === this.api.DLGT_CREDENTIAL_RELOAD) {
             return this.reloadCredential();
         }
@@ -234,6 +238,32 @@ class Module extends Delegate {
         }
     }
 
+    /**
+     * Passes a rate limit the api reported to the parent delegate, best
+     * effort. The payload never holds the error's message, url, body or
+     * headers, because the parent shows it to end users.
+     * @param {import('../errors').RateLimitError} rateLimitError
+     */
+    async reportRateLimit(rateLimitError) {
+        const { reason, retryAt, policy, statusCode } = rateLimitError;
+        const links = this.apiClass.rateLimit?.userHints?.[reason]?.links ?? [];
+        try {
+            await this.notify(this.DLGT_RATE_LIMITED, {
+                moduleName: this.name,
+                reason,
+                retryAt,
+                policy,
+                statusCode,
+                links,
+            });
+        } catch (err) {
+            this.logger.error('Failed to propagate RATE_LIMITED', {
+                eventName: `${this.logger.name}.rate_limited_propagation_failed`,
+                error: err,
+            });
+        }
+    }
+
     async deauthorize() {
         //todo: Check if this is correct, we're instantiating a new api without params (credentials, tokens, etc...)
         this.api = new this.apiClass();

package/modules/requester/oauth-2.js

@@ -398,12 +398,6 @@ class OAuth2Requester extends Requester {
         );
         transportError.statusCode = status;
         transportError.isTokenRefreshTransportFailure = true;
-        if (error?.isRateLimited) {
-            transportError.isRateLimited = true;
-            transportError.retryAt = error.retryAt;
-            transportError.waitMs = error.waitMs;
-            transportError.reason = error.reason;
-        }
         return transportError;
     }
 

package/modules/requester/requester.js

@@ -67,6 +67,8 @@ class Requester extends Delegate {
         this._authGeneration = 0;
         this.DLGT_INVALID_AUTH = 'INVALID_AUTH';
         this.delegateTypes.push(this.DLGT_INVALID_AUTH);
+        this.DLGT_RATE_LIMITED = 'RATE_LIMITED';
+        this.delegateTypes.push(this.DLGT_RATE_LIMITED);
         this.agent = get(params, 'agent', null);
 
         // Per-attempt HTTP timeout. Without this the framework called fetch()
@@ -557,6 +559,7 @@ class Requester extends Delegate {
             module: this._telemetryModuleLabel(),
             scopeKey: computeScopeKey(this._rateLimitPolicy, this),
         });
+        await this._notifyRateLimited(rateLimitError);
         throw this._maybeFlagTimeoutDuringBodyRead(rateLimitError, timeoutMs);
     }
 
@@ -636,6 +639,24 @@ class Requester extends Delegate {
         });
     }
 
+    /**
+     * Tells the delegate a wait was too long to sleep. Best effort: a failed
+     * notification is logged and never replaces the RateLimitError the caller
+     * is about to receive.
+     */
+    async _notifyRateLimited(rateLimitError) {
+        try {
+            await this.notify(this.DLGT_RATE_LIMITED, rateLimitError);
+        } catch (error) {
+            const logger = this.logger;
+            logger.warn('Rate limit notification failed', {
+                eventName: `${logger.name}.rate_limit_notify_failed`,
+                statusCode: rateLimitError.statusCode,
+                error,
+            });
+        }
+    }
+
     _logRequestFailed(encodedUrl, options, status) {
         const logger = this.logger;
         if (!logger.isLevelEnabled('DEBUG')) return;

package/package.json

@@ -1,13 +1,12 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0--canary.657.b14221f.0",
+    "version": "2.0.0--canary.658.93c8e07.0",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
         "@aws-sdk/client-lambda": "^3.714.0",
         "@aws-sdk/client-s3": "^3.588.0",
-        "@aws-sdk/client-scheduler": "^3.588.0",
         "@aws-sdk/client-sqs": "^3.588.0",
         "@aws-sdk/client-ssm": "^3.588.0",
         "@aws-sdk/s3-request-presigner": "^3.588.0",
@@ -48,9 +47,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0--canary.657.b14221f.0",
-        "@friggframework/prettier-config": "2.0.0--canary.657.b14221f.0",
-        "@friggframework/test": "2.0.0--canary.657.b14221f.0",
+        "@friggframework/eslint-config": "2.0.0--canary.658.93c8e07.0",
+        "@friggframework/prettier-config": "2.0.0--canary.658.93c8e07.0",
+        "@friggframework/test": "2.0.0--canary.658.93c8e07.0",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -90,5 +89,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "b14221fca51737284a61407b42e003637930465e"
+    "gitHead": "93c8e078d4ff1bf00ed831152712b442abe615cd"
 }

package/queues/queuer-util.js

@@ -126,4 +126,4 @@ const QueuerUtil = {
     },
 };
 
-module.exports = { QueuerUtil, awsConfigOptions };
+module.exports = { QueuerUtil };

package/types/core/index.d.ts

@@ -47,28 +47,8 @@ declare module "@friggframework/core" {
     send(params: object & { QueueUrl: any }, delay?: number): Promise<string>;
 
     sendAsyncSQSMessage(params: SendSQSMessageParams): Promise<string>;
-
-    /**
-     * Hook: a message was put back for a rate limit. Runs after the message
-     * was sent. A failure is logged and never changes the outcome.
-     */
-    recordRateLimitWait(
-      body: object,
-      error: Error,
-      state: RateLimitWaitState
-    ): Promise<void>;
-
-    /** Hook: a message that was put back for a rate limit ran without error. */
-    clearRateLimitWait(body: object): Promise<void>;
   }
 
-  export type RateLimitWaitState = {
-    status: "WAITING" | "EXHAUSTED";
-    mechanism: "delay" | "schedule" | "visibility" | "none";
-    deferrals: number;
-    retryAt: Date;
-  };
-
   interface IWorker {
     getQueueURL(params: GetQueueURLParams): Promise<string | undefined>;
     run(params: { Records: any }, context?: object): Promise<BatchItemFailuresResponse>;