@friggframework/core 2.0.0-next.109 → 2.0.0-next.110

package/modules/module.js

@@ -148,9 +148,29 @@ class Module extends Delegate {
             await this.deauthorize();
         } else if (delegateString === this.api.DLGT_INVALID_AUTH) {
             await this.markCredentialsInvalid(object);
+        } else if (delegateString === this.api.DLGT_CREDENTIAL_RELOAD) {
+            return this.reloadCredential();
         }
     }
 
+    /**
+     * Re-reads the credential row from the database. The requester can then
+     * adopt a concurrent invocation's refresh and does not race it.
+     * @returns {Promise<Object|null>} The persisted token fields, or null
+     *   when no credential row is available. The requester treats null as
+     *   "nothing to adopt" and continues with its own refresh.
+     */
+    async reloadCredential() {
+        if (!this.credential?.id) return null;
+        const freshFromDatabase =
+            await this.credentialRepository.findCredentialById(
+                this.credential.id
+            );
+        if (!freshFromDatabase) return null;
+        this.credential = freshFromDatabase;
+        return this.apiParamsFromCredential(freshFromDatabase);
+    }
+
     async markCredentialsInvalid(diagnosticInfo = null) {
         if (!this.credential) return;
 

package/modules/requester/oauth-2.js

@@ -60,9 +60,26 @@ class OAuth2Requester extends Requester {
         this.DLGT_TOKEN_UPDATE = 'TOKEN_UPDATE';
         /** @type {string} Delegate type for token deauthorization notifications */
         this.DLGT_TOKEN_DEAUTHORIZED = 'TOKEN_DEAUTHORIZED';
+        /**
+         * @type {string} Delegate type that asks the Module for the stored
+         * credential. The requester can then adopt a concurrent invocation's
+         * refresh and does not race it. See _adoptNewerCredential.
+         */
+        this.DLGT_CREDENTIAL_RELOAD = 'CREDENTIAL_RELOAD';
 
         this.delegateTypes.push(this.DLGT_TOKEN_UPDATE);
         this.delegateTypes.push(this.DLGT_TOKEN_DEAUTHORIZED);
+        this.delegateTypes.push(this.DLGT_CREDENTIAL_RELOAD);
+
+        /**
+         * Re-read delays after an invalid_grant, in ms. The winner's write
+         * can arrive after the loser's rejection (716 ms measured in
+         * production). Wait between re-reads before you decide that the
+         * credential is dead. Tests can inject other values.
+         */
+        this.credentialReloadBackoffMs = params?.credentialReloadBackoffMs ?? [
+            500, 1000, 1500,
+        ];
 
         /** @type {string} OAuth grant type */
         this.grant_type = get(params, 'grant_type', 'authorization_code');
@@ -295,6 +312,16 @@ class OAuth2Requester extends Requester {
      * @returns {Promise<boolean>} True if refresh succeeded, false if failed
      */
     async refreshAuth() {
+        // The wrapper runs this check before it calls refreshAuth(). Callers
+        // that reach this method directly skip the wrapper, so the check runs
+        // here for them. The guard keeps it at one read per refresh.
+        if (
+            !this._isInsideRefreshFlow() &&
+            (await this._adoptNewerCredential())
+        ) {
+            return true;
+        }
+
         try {
             console.log('[Frigg] Starting token refresh', {
                 grant_type: this.grant_type,
@@ -310,7 +337,11 @@ class OAuth2Requester extends Requester {
                     refresh_token: this.refresh_token,
                 });
             } else {
-                await this.getTokenFromClientCredentials();
+                // getTokenFromClientCredentials() reports a failed token
+                // request itself and resolves to undefined. Without this
+                // check, the refresh counts as a success with no new token.
+                const tokenRes = await this.getTokenFromClientCredentials();
+                if (!tokenRes) return false;
             }
             console.log('[Frigg] Token refresh succeeded');
             return true;
@@ -322,8 +353,21 @@ class OAuth2Requester extends Requester {
                 response_status: error?.response?.status,
                 response_data: error?.response?.data,
             });
-            // Status only: the refresh body carries client_secret, and
-            // FetchError embeds the body in its message outside prod.
+
+            if (!this._isDefinitiveAuthRejection(error)) {
+                throw this._transportFailureError(error, moduleName);
+            }
+
+            if (await this._adoptNewerCredentialWithBackoff()) return true;
+
+            // The provider rejected the grant, and the store has nothing
+            // newer. The credential is dead.
+            this.telemetry?.count?.('frigg.auth.refresh_race_lost', 1, {
+                module: this._telemetryModuleLabel(),
+            });
+            // Send the status only. The refresh body contains the
+            // client_secret, and FetchError puts the body in its message
+            // outside prod.
             await this.notify(this.DLGT_INVALID_AUTH, {
                 statusCode: error?.statusCode,
             });
@@ -331,6 +375,106 @@ class OAuth2Requester extends Requester {
         }
     }
 
+    /**
+     * A timeout, a 429, or a 5xx from the token endpoint says nothing about
+     * the credential. The error must stay retryable: the caller fails loudly
+     * (worker throw → SQS retry → DLQ) and does not flag a healthy
+     * credential. This is a fresh Error on purpose. Outside prod, the
+     * original message can contain the request body, and the body carries
+     * the client_secret.
+     */
+    _transportFailureError(error, moduleName) {
+        const status =
+            error?.statusCode ?? error?.status ?? error?.response?.status;
+        const transportError = new Error(
+            `[Frigg] Token refresh transport failure for ${moduleName}` +
+                (status != null ? ` (status ${status})` : '')
+        );
+        transportError.statusCode = status;
+        transportError.isTokenRefreshTransportFailure = true;
+        return transportError;
+    }
+
+    /**
+     * A definitive rejection can mean that another invocation consumed this
+     * refresh token first. That invocation's write can be unreadable for a
+     * short time. Re-read with a bounded backoff before you decide that the
+     * credential is dead.
+     */
+    async _adoptNewerCredentialWithBackoff() {
+        for (const delayMs of this.credentialReloadBackoffMs) {
+            await new Promise((resolve) => setTimeout(resolve, delayMs));
+            if (await this._adoptNewerCredential()) {
+                console.log(
+                    '[Frigg] Adopted a newer credential after a refresh rejection',
+                    { module: this._telemetryModuleLabel() }
+                );
+                this.telemetry?.count?.('frigg.auth.refresh_race_recovered', 1, {
+                    module: this._telemetryModuleLabel(),
+                });
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * True when the token endpoint refused the grant. RFC 6749 §5.2 sets
+     * the status: 400, or 401 for invalid_client. A 429 or a 5xx is never a
+     * verdict on the credential. Body markers are only a fallback for SDK
+     * errors that have no status code. Production FetchErrors have a
+     * sanitized body, so a marker cannot be the primary signal.
+     */
+    _isDefinitiveAuthRejection(error) {
+        const status =
+            error?.statusCode ?? error?.status ?? error?.response?.status;
+        if (status !== undefined && status !== null) {
+            return status === 400 || status === 401;
+        }
+        const haystack = [
+            error?.message,
+            error?.body,
+            typeof error?.error === 'string' ? error.error : null,
+            error?.response?.data && JSON.stringify(error.response.data),
+        ]
+            .filter(Boolean)
+            .join(' ');
+        return /\b(invalid_grant|invalid_client)\b/i.test(haystack);
+    }
+
+    /**
+     * Adopts the credential stored in the database if it is newer than the
+     * credential from the instance. The refresh token decides "newer": a
+     * provider can rotate it and return an identical access-token string.
+     * The reload only reads. A reload failure is not fatal: a database blip
+     * must not change the auth behavior.
+     *
+     * @returns {Promise<boolean>} True if the module adopted a newer
+     *   credential.
+     */
+    async _adoptNewerCredential() {
+        let stored = null;
+        try {
+            stored = await this.notify(this.DLGT_CREDENTIAL_RELOAD);
+        } catch (_) {
+            return false;
+        }
+        if (!stored?.refresh_token) return false;
+        if (stored.refresh_token === this.refresh_token) return false;
+
+        if (stored.access_token) {
+            this.access_token = stored.access_token;
+        }
+        this.refresh_token = stored.refresh_token;
+        if (stored.accessTokenExpire !== undefined) {
+            this.accessTokenExpire = stored.accessTokenExpire;
+        }
+        if (stored.refreshTokenExpire !== undefined) {
+            this.refreshTokenExpire = stored.refreshTokenExpire;
+        }
+        return true;
+    }
+
     /**
      * Obtains tokens using the Resource Owner Password Credentials grant.
      * Requires username and password to be set.

package/modules/requester/requester.js

@@ -1,4 +1,5 @@
 const fetch = require('node-fetch');
+const { AsyncLocalStorage } = require('async_hooks');
 const { Delegate } = require('../../core');
 const { FetchError } = require('../../errors');
 const { get } = require('../../assertions');
@@ -7,6 +8,14 @@ const { getTelemetry } = require('../../telemetry/telemetry-runtime');
 const DEFAULT_REQUEST_TIMEOUT_MS = 60_000;
 const MAX_AUTH_RETRIES = 3;
 
+// This context marks the async call chain that holds the refresh slot, and
+// names the requester that holds it. Token requests re-enter _rawRequest
+// through this._post. A 401 from inside that chain must fail fast: if it
+// joins the refresh in flight, it awaits its own promise and hangs. The
+// identity keeps a second requester used inside the chain out of the check.
+// AsyncLocalStorage reaches the nested calls without help from the subclasses.
+const refreshContext = new AsyncLocalStorage();
+
 class Requester extends Delegate {
     constructor(params) {
         super(params);
@@ -14,6 +23,11 @@ class Requester extends Delegate {
         this.isRefreshable = false;
         this.refreshCount = 0;
         this.authGraceRetryCount = 0;
+        // Concurrent 401s share one refreshAuth() run. See _refreshAuthOnce().
+        this._inFlightRefresh = null;
+        // This counter increases when the tokens change. A stale 401 (the
+        // token changed already) then retries and does not refresh again.
+        this._authGeneration = 0;
         this.DLGT_INVALID_AUTH = 'INVALID_AUTH';
         this.delegateTypes.push(this.DLGT_INVALID_AUTH);
         this.agent = get(params, 'agent', null);
@@ -184,6 +198,10 @@ class Requester extends Delegate {
 
         options.headers = await this.addAuthHeaders(options.headers);
 
+        // A 401 that arrives after a concurrent refresh is stale. It is not
+        // proof that the new token failed.
+        const authGenerationAtDispatch = this._authGeneration;
+
         if (this.agent) options.agent = this.agent;
 
         // Per-attempt timeout — fresh AbortController per call so the retry
@@ -266,6 +284,26 @@ class Requester extends Delegate {
             }
 
             if (status === 401) {
+                // A 401 from inside the refresh flow means the provider
+                // rejected the credential itself (invalid_client). A new
+                // refresh cannot help. A join would await this same call.
+                if (this._isInsideRefreshFlow()) {
+                    throw await this._invalidateAuth(
+                        encodedUrl,
+                        options,
+                        response
+                    );
+                }
+
+                const tokenReplacedWhileInFlight =
+                    this._authGeneration !== authGenerationAtDispatch;
+                if (this.isRefreshable && tokenReplacedWhileInFlight) {
+                    // This request did not try the current token. Retry with
+                    // it. Do not spend one more provider-side rotation.
+                    clearRequestTimer();
+                    return this._rawRequest(url, options, attempt + 1);
+                }
+
                 if (!this.isRefreshable) {
                     // Up to MAX_AUTH_RETRIES grace retries before invalidating
                     // — a 401 alone isn't proof the credential is bad.
@@ -290,14 +328,15 @@ class Requester extends Delegate {
                     );
                 }
 
-                if (this.refreshCount < MAX_AUTH_RETRIES) {
-                    this.refreshCount++;
-                    const refreshSucceeded = await this.refreshAuth();
-                    if (refreshSucceeded) {
-                        clearRequestTimer();
-                        return this._rawRequest(url, options, attempt + 1);
-                    }
+                // Concurrent 401s share one refresh. Independent refreshes
+                // kill each other, because many providers use single-use
+                // refresh tokens. Only the initiator spends the retry budget.
+                const refreshAlreadyInFlight = Boolean(this._inFlightRefresh);
 
+                if (
+                    !refreshAlreadyInFlight &&
+                    this.refreshCount >= MAX_AUTH_RETRIES
+                ) {
                     throw await this._invalidateAuth(
                         encodedUrl,
                         options,
@@ -305,6 +344,16 @@ class Requester extends Delegate {
                     );
                 }
 
+                if (!refreshAlreadyInFlight) {
+                    this.refreshCount++;
+                }
+
+                const refreshSucceeded = await this._refreshAuthOnce();
+                if (refreshSucceeded) {
+                    clearRequestTimer();
+                    return this._rawRequest(url, options, attempt + 1);
+                }
+
                 throw await this._invalidateAuth(encodedUrl, options, response);
             }
 
@@ -427,6 +476,64 @@ class Requester extends Delegate {
         return this._request(options.url, fetchOptions);
     }
 
+    /**
+     * Runs one refreshAuth() at a time. The first caller starts the refresh.
+     * Callers that arrive during the refresh await the same promise. The
+     * check-and-store step is synchronous. Thus two concurrent callers
+     * cannot both start a refresh.
+     *
+     * @returns {Promise<boolean>} True if the refresh succeeded.
+     */
+    _refreshAuthOnce() {
+        if (!this._inFlightRefresh) {
+            // Keep .finally last. The stored promise must be the promise
+            // that clears the slot. Then the slot is free before a waiter
+            // resumes.
+            this._inFlightRefresh = this._adoptOrRefresh().finally(() => {
+                this._inFlightRefresh = null;
+            });
+        }
+        return this._inFlightRefresh;
+    }
+
+    /**
+     * Adopts a newer stored credential, or refreshes. Both steps run inside
+     * the marker, because the marker must cover the whole time this instance
+     * holds the slot. The adoption lives here, not in refreshAuth(), so a
+     * module that overrides refreshAuth() still gets it.
+     *
+     * @returns {Promise<boolean>} True if the instance holds a usable token.
+     */
+    async _adoptOrRefresh() {
+        const refreshSucceeded = await refreshContext.run(
+            { requester: this },
+            async () => {
+                if (await this._adoptNewerCredential()) return true;
+                return this.refreshAuth();
+            }
+        );
+        if (refreshSucceeded) this._authGeneration++;
+        return refreshSucceeded;
+    }
+
+    /**
+     * Hook for requesters that hold a rotating stored credential. Return true
+     * when the instance adopted a newer stored credential and needs no
+     * refresh. A module backed by a vendor SDK overrides this, calls super,
+     * and then copies the adopted tokens into its SDK client.
+     *
+     * @returns {Promise<boolean>} True if the instance adopted a newer
+     *   credential.
+     */
+    async _adoptNewerCredential() {
+        return false;
+    }
+
+    /** True while this instance runs its own refresh. */
+    _isInsideRefreshFlow() {
+        return refreshContext.getStore()?.requester === this;
+    }
+
     async refreshAuth() {
         throw new Error('refreshAuth not yet defined in child of Requester');
     }

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@friggframework/core",
     "prettier": "@friggframework/prettier-config",
-    "version": "2.0.0-next.109",
+    "version": "2.0.0-next.110",
     "dependencies": {
         "@aws-sdk/client-apigatewaymanagementapi": "^3.588.0",
         "@aws-sdk/client-kms": "^3.588.0",
@@ -48,9 +48,9 @@
         }
     },
     "devDependencies": {
-        "@friggframework/eslint-config": "2.0.0-next.109",
-        "@friggframework/prettier-config": "2.0.0-next.109",
-        "@friggframework/test": "2.0.0-next.109",
+        "@friggframework/eslint-config": "2.0.0-next.110",
+        "@friggframework/prettier-config": "2.0.0-next.110",
+        "@friggframework/test": "2.0.0-next.110",
         "@prisma/client": "^6.19.3",
         "@types/lodash": "4.17.15",
         "@typescript-eslint/eslint-plugin": "^8.0.0",
@@ -90,5 +90,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "193f4e024490cd25256c0fa33fd05bc8f323a58e"
+    "gitHead": "90c538f9523764b80b71a49d2d037e89cf426e0a"
 }

package/types/module-plugin/index.d.ts

@@ -41,6 +41,7 @@ declare module "@friggframework/module-plugin" {
     ): Promise<any>;
     parseBody(response: any): Promise<any>;
     refreshAuth(): Promise<any>;
+    _adoptNewerCredential(): Promise<boolean>;
 
     delegate: any;
     delegateTypes: any[];
@@ -72,6 +73,7 @@ declare module "@friggframework/module-plugin" {
     _put(options: RequestOptions): Promise<any>;
     _delete(options: RequestOptions): Promise<any>;
     refreshAuth(): Promise<any>;
+    _adoptNewerCredential(): Promise<boolean>;
   }
 
   type RequestOptions = {
@@ -201,6 +203,7 @@ declare module "@friggframework/module-plugin" {
     addAuthHeaders(headers: object): Promise<object>;
     isAuthenticated(): boolean;
     refreshAuth(): Promise<void>;
+    _adoptNewerCredential(): Promise<boolean>;
     getTokenFromUsernamePassword(): Promise<Token>;
     getTokenFromClientCredentials(): Promise<Token>;
   }