@rushstack/rush-http-build-cache-plugin 5.97.1-pr3481.18 → 5.98.0

package/lib/{HttpBuildCacheProvider.test.js → test/HttpBuildCacheProvider.test.js}

@@ -1,4 +1,6 @@
 "use strict";
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -27,9 +29,9 @@ jest.mock('node-fetch', function () {
     return Object.assign(jest.fn(), jest.requireActual('node-fetch'));
 });
 const node_fetch_1 = __importStar(require("node-fetch"));
-const HttpBuildCacheProvider_1 = require("./HttpBuildCacheProvider");
 const rush_sdk_1 = require("@rushstack/rush-sdk");
 const node_core_library_1 = require("@rushstack/node-core-library");
+const HttpBuildCacheProvider_1 = require("../HttpBuildCacheProvider");
 const EXAMPLE_OPTIONS = {
     url: 'https://buildcache.example.acme.com',
     tokenHandler: {
@@ -39,7 +41,8 @@ const EXAMPLE_OPTIONS = {
     uploadMethod: 'POST',
     isCacheWriteAllowed: false,
     pluginName: 'example-plugin',
-    rushProjectRoot: '/repo'
+    rushProjectRoot: '/repo',
+    minHttpRetryDelayMs: 1
 };
 describe('HttpBuildCacheProvider', () => {
     let terminalBuffer;
@@ -59,7 +62,8 @@ describe('HttpBuildCacheProvider', () => {
             }));
             const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');
             expect(result).toBe(undefined);
-            expect(node_fetch_1.default).toHaveBeenCalledWith('https://buildcache.example.acme.com/some-key', {
+            expect(node_fetch_1.default).toHaveBeenCalledTimes(1);
+            expect(node_fetch_1.default).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {
                 body: undefined,
                 headers: {},
                 method: 'GET',
@@ -67,6 +71,45 @@ describe('HttpBuildCacheProvider', () => {
             });
             expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(`"Error getting cache entry: Error: Credentials for https://buildcache.example.acme.com/ have not been provided.[n]In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.[n][n]For local developers, run:[n][n]    rush update-cloud-credentials --interactive[n][n]"`);
         });
+        it('attempts up to 3 times to download a cache entry', async () => {
+            jest.spyOn(rush_sdk_1.EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);
+            const session = {};
+            const provider = new HttpBuildCacheProvider_1.HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);
+            mocked(node_fetch_1.default).mockResolvedValueOnce(new node_fetch_1.Response('InternalServiceError', {
+                status: 500,
+                statusText: 'InternalServiceError'
+            }));
+            mocked(node_fetch_1.default).mockResolvedValueOnce(new node_fetch_1.Response('ServiceUnavailable', {
+                status: 503,
+                statusText: 'ServiceUnavailable'
+            }));
+            mocked(node_fetch_1.default).mockResolvedValueOnce(new node_fetch_1.Response('BadGateway', {
+                status: 504,
+                statusText: 'BadGateway'
+            }));
+            const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');
+            expect(result).toBe(undefined);
+            expect(node_fetch_1.default).toHaveBeenCalledTimes(3);
+            expect(node_fetch_1.default).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {
+                body: undefined,
+                headers: {},
+                method: 'GET',
+                redirect: 'follow'
+            });
+            expect(node_fetch_1.default).toHaveBeenNthCalledWith(2, 'https://buildcache.example.acme.com/some-key', {
+                body: undefined,
+                headers: {},
+                method: 'GET',
+                redirect: 'follow'
+            });
+            expect(node_fetch_1.default).toHaveBeenNthCalledWith(3, 'https://buildcache.example.acme.com/some-key', {
+                body: undefined,
+                headers: {},
+                method: 'GET',
+                redirect: 'follow'
+            });
+            expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(`"Could not get cache entry: HTTP 504: BadGateway[n]"`);
+        });
     });
 });
 //# sourceMappingURL=HttpBuildCacheProvider.test.js.map

package/lib/test/HttpBuildCacheProvider.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"HttpBuildCacheProvider.test.js","sourceRoot":"","sources":["../../src/test/HttpBuildCacheProvider.test.ts"],"names":[],"mappings":";AAAA,4FAA4F;AAC5F,2DAA2D;;;;;;;;;;;;;;;;;;;;;;;;;AAE3D,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE;IACtB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;AACpE,CAAC,CAAC,CAAC;AAEH,yDAA6C;AAC7C,kDAA4E;AAC5E,oEAAsF;AAEtF,sEAAmE;AAEnE,MAAM,eAAe,GAAG;IACtB,GAAG,EAAE,qCAAqC;IAC1C,YAAY,EAAE;QACZ,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,CAAC,iBAAiB,CAAC;KAC1B;IACD,YAAY,EAAE,MAAM;IACpB,mBAAmB,EAAE,KAAK;IAC1B,UAAU,EAAE,gBAAgB;IAC5B,eAAe,EAAE,OAAO;IACxB,mBAAmB,EAAE,CAAC;CACvB,CAAC;AAEF,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,IAAI,cAA4C,CAAC;IACjD,IAAI,QAAmB,CAAC;IAExB,UAAU,CAAC,GAAG,EAAE;QACd,cAAc,GAAG,IAAI,gDAA4B,EAAE,CAAC;QACpD,QAAQ,GAAG,IAAI,4BAAQ,CAAC,cAAc,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iCAAiC,EAAE,GAAG,EAAE;QAC/C,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,IAAI,CAAC,KAAK,CAAC,mCAAwB,EAAE,sBAAsB,EAAE,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAE/F,MAAM,OAAO,GAAgB,EAAiB,CAAC;YAC/C,MAAM,QAAQ,GAAG,IAAI,+CAAsB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YAEtE,MAAM,CAAC,oBAAK,CAAC,CAAC,iBAAiB,CAC7B,IAAI,qBAAQ,CAAC,cAAc,EAAE;gBAC3B,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,cAAc;aAC3B,CAAC,CACH,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,+BAA+B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,oBAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,oBAAK,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,8CAA8C,EAAE;gBACvF,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,CAAC,CAAC,qBAAqB,CAC7D,2SAA2S,CAC5S,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,IAAI,CAAC,KAAK,CAAC,mCAAwB,EAAE,sBAAsB,EAAE,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAE/F,MAAM,OAAO,GAAgB,EAAiB,CAAC;YAC/C,MAAM,QAAQ,GAAG,IAAI,+CAAsB,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YAEtE,MAAM,CAAC,oBAAK,CAAC,CAAC,qBAAqB,CACjC,IAAI,qBAAQ,CAAC,sBAAsB,EAAE;gBACnC,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,sBAAsB;aACnC,CAAC,CACH,CAAC;YACF,MAAM,CAAC,oBAAK,CAAC,CAAC,qBAAqB,CACjC,IAAI,qBAAQ,CAAC,oBAAoB,EAAE;gBACjC,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,oBAAoB;aACjC,CAAC,CACH,CAAC;YACF,MAAM,CAAC,oBAAK,CAAC,CAAC,qBAAqB,CACjC,IAAI,qBAAQ,CAAC,YAAY,EAAE;gBACzB,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,YAAY;aACzB,CAAC,CACH,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,+BAA+B,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACpF,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC/B,MAAM,CAAC,oBAAK,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,CAAC,oBAAK,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,8CAA8C,EAAE;gBACvF,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,oBAAK,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,8CAA8C,EAAE;gBACvF,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,oBAAK,CAAC,CAAC,uBAAuB,CAAC,CAAC,EAAE,8CAA8C,EAAE;gBACvF,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,QAAQ;aACnB,CAAC,CAAC;YACH,MAAM,CAAC,cAAc,CAAC,gBAAgB,EAAE,CAAC,CAAC,qBAAqB,CAC7D,sDAAsD,CACvD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.\n// See LICENSE in the project root for license information.\n\njest.mock('node-fetch', function () {\n  return Object.assign(jest.fn(), jest.requireActual('node-fetch'));\n});\n\nimport fetch, { Response } from 'node-fetch';\nimport { RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk';\nimport { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library';\n\nimport { HttpBuildCacheProvider } from '../HttpBuildCacheProvider';\n\nconst EXAMPLE_OPTIONS = {\n  url: 'https://buildcache.example.acme.com',\n  tokenHandler: {\n    exec: 'node',\n    args: ['tokenHandler.js']\n  },\n  uploadMethod: 'POST',\n  isCacheWriteAllowed: false,\n  pluginName: 'example-plugin',\n  rushProjectRoot: '/repo',\n  minHttpRetryDelayMs: 1\n};\n\ndescribe('HttpBuildCacheProvider', () => {\n  let terminalBuffer: StringBufferTerminalProvider;\n  let terminal!: Terminal;\n\n  beforeEach(() => {\n    terminalBuffer = new StringBufferTerminalProvider();\n    terminal = new Terminal(terminalBuffer);\n  });\n\n  describe('tryGetCacheEntryBufferByIdAsync', () => {\n    it('prints warning if read credentials are not available', async () => {\n      jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);\n\n      const session: RushSession = {} as RushSession;\n      const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);\n\n      mocked(fetch).mockResolvedValue(\n        new Response('Unauthorized', {\n          status: 401,\n          statusText: 'Unauthorized'\n        })\n      );\n\n      const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');\n      expect(result).toBe(undefined);\n      expect(fetch).toHaveBeenCalledTimes(1);\n      expect(fetch).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {\n        body: undefined,\n        headers: {},\n        method: 'GET',\n        redirect: 'follow'\n      });\n      expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(\n        `\"Error getting cache entry: Error: Credentials for https://buildcache.example.acme.com/ have not been provided.[n]In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.[n][n]For local developers, run:[n][n]    rush update-cloud-credentials --interactive[n][n]\"`\n      );\n    });\n\n    it('attempts up to 3 times to download a cache entry', async () => {\n      jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);\n\n      const session: RushSession = {} as RushSession;\n      const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);\n\n      mocked(fetch).mockResolvedValueOnce(\n        new Response('InternalServiceError', {\n          status: 500,\n          statusText: 'InternalServiceError'\n        })\n      );\n      mocked(fetch).mockResolvedValueOnce(\n        new Response('ServiceUnavailable', {\n          status: 503,\n          statusText: 'ServiceUnavailable'\n        })\n      );\n      mocked(fetch).mockResolvedValueOnce(\n        new Response('BadGateway', {\n          status: 504,\n          statusText: 'BadGateway'\n        })\n      );\n\n      const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');\n      expect(result).toBe(undefined);\n      expect(fetch).toHaveBeenCalledTimes(3);\n      expect(fetch).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {\n        body: undefined,\n        headers: {},\n        method: 'GET',\n        redirect: 'follow'\n      });\n      expect(fetch).toHaveBeenNthCalledWith(2, 'https://buildcache.example.acme.com/some-key', {\n        body: undefined,\n        headers: {},\n        method: 'GET',\n        redirect: 'follow'\n      });\n      expect(fetch).toHaveBeenNthCalledWith(3, 'https://buildcache.example.acme.com/some-key', {\n        body: undefined,\n        headers: {},\n        method: 'GET',\n        redirect: 'follow'\n      });\n      expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(\n        `\"Could not get cache entry: HTTP 504: BadGateway[n]\"`\n      );\n    });\n  });\n});\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rushstack/rush-http-build-cache-plugin",
-  "version": "5.97.1-pr3481.18",
+  "version": "5.98.0",
   "description": "Rush plugin for generic HTTP cloud build cache",
   "repository": {
     "type": "git",
@@ -14,17 +14,17 @@
   "dependencies": {
     "https-proxy-agent": "~5.0.0",
     "node-fetch": "2.6.7",
-    "@rushstack/node-core-library": "3.55.2",
-    "@rushstack/rush-sdk": "5.97.1-pr3481.18"
+    "@rushstack/node-core-library": "3.59.0",
+    "@rushstack/rush-sdk": "5.98.0"
   },
   "devDependencies": {
     "@types/heft-jest": "1.0.1",
     "@types/node": "14.18.36",
     "@types/node-fetch": "2.6.2",
-    "@microsoft/rush-lib": "5.97.1-pr3481.18",
+    "@microsoft/rush-lib": "5.98.0",
     "@rushstack/eslint-config": "3.2.0",
-    "@rushstack/heft": "0.50.0",
-    "@rushstack/heft-node-rig": "1.12.6"
+    "@rushstack/heft": "0.50.5",
+    "@rushstack/heft-node-rig": "1.12.11"
   },
   "scripts": {
     "build": "heft build --clean",

package/rush-logs/rush-http-build-cache-plugin._phase_build.log

@@ -1,24 +1,5 @@
-Invoking: heft build --clean --production
-Using local Heft from /home/vsts/work/1/s/rush-plugins/rush-http-build-cache-plugin/node_modules/@rushstack/heft
+Build cache hit.
+Clearing cached folders: dist, lib, lib-commonjs, temp
+Successfully restored output from the build cache.
 
-Project build folder is "/home/vsts/work/1/s/rush-plugins/rush-http-build-cache-plugin"
-Using rig configuration from ./node_modules/@rushstack/heft-node-rig/profiles/default
-Starting build
- ---- Clean started ---- 
- ---- Clean finished (2ms) ---- 
- ---- Pre-compile started ---- 
- ---- Pre-compile finished (0ms) ---- 
- ---- Compile started ---- 
-[copy-static-assets] Copied 1 file and linked 0 files in 12ms
-[typescript] Using TypeScript version 4.8.4
- ---- Compile finished (5243ms) ---- 
- ---- Bundle started ---- 
- ---- Bundle finished (1ms) ---- 
- ---- Post-build started ---- 
- ---- Post-build finished (2ms) ---- 
--------------------- Finished (5.609s) --------------------
-Project: @rushstack/rush-http-build-cache-plugin@5.97.1-pr3481.18
-Heft version: 0.50.0
-Node version: v14.21.3
-Caching build output folders: lib
-Successfully set cache entry.
+Restoring cached log file at /home/vsts/work/1/s/rush-plugins/rush-http-build-cache-plugin/.rush/temp/operation/_phase_build/all.log

package/rush-logs/rush-http-build-cache-plugin._phase_test.log

@@ -1,3 +1,4 @@
+This project does not define the caching behavior of the "_phase:test" command, so caching has been disabled.
 Invoking: heft test --no-build --production
 Using local Heft from /home/vsts/work/1/s/rush-plugins/rush-http-build-cache-plugin/node_modules/@rushstack/heft
 
@@ -7,23 +8,22 @@ Starting test
 [jest] Using Jest version 29.5.0
 
 Run start. 1 test suite
-START src/HttpBuildCacheProvider.test.ts
-PASS src/HttpBuildCacheProvider.test.ts (duration: 1.231s, 1 passed, 0 failed)
+START src/test/HttpBuildCacheProvider.test.ts
+PASS src/test/HttpBuildCacheProvider.test.ts (duration: 0.693s, 2 passed, 0 failed)
 
 Tests finished:
-  Successes: 1
+  Successes: 2
   Failures: 0
-  Total: 1
------------------------------|---------|----------|---------|---------|---------------------------------------------------
-File                         | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s                                 
------------------------------|---------|----------|---------|---------|---------------------------------------------------
-All files                    |   42.74 |    39.02 |    40.9 |   42.74 |                                                   
- HttpBuildCacheProvider.ts   |   47.45 |    39.02 |      50 |   47.45 | 58,89,101-174,184,209,213-214,237-249,258,297-380 
- RushHttpBuildCachePlugin.ts |       0 |      100 |       0 |       0 | 1-81                                              
- index.ts                    |       0 |      100 |     100 |       0 | 1-3                                               
------------------------------|---------|----------|---------|---------|---------------------------------------------------
--------------------- Finished (2.192s) --------------------
-Project: @rushstack/rush-http-build-cache-plugin@5.97.1-pr3481.18
-Heft version: 0.50.0
+  Total: 2
+-----------------------------|---------|----------|---------|---------|----------------------------------------------------------------------------
+File                         | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s                                                          
+-----------------------------|---------|----------|---------|---------|----------------------------------------------------------------------------
+All files                    |   53.23 |    45.55 |   47.82 |   53.23 |                                                                            
+ HttpBuildCacheProvider.ts   |   58.26 |    45.55 |   61.11 |   58.26 | 63,108-182,192,218,222-223,270-278,287,326-328,335,348-371,389-390,399-409 
+ RushHttpBuildCachePlugin.ts |       0 |      100 |       0 |       0 | 7-77                                                                       
+ index.ts                    |       0 |      100 |     100 |       0 | 1-3                                                                        
+-----------------------------|---------|----------|---------|---------|----------------------------------------------------------------------------
+-------------------- Finished (1.935s) --------------------
+Project: @rushstack/rush-http-build-cache-plugin@5.97.1
+Heft version: 0.50.5
 Node version: v14.21.3
-This project does not define the caching behavior of the "_phase:test" command, so caching has been disabled.

package/src/HttpBuildCacheProvider.ts

@@ -1,4 +1,4 @@
-import { ITerminal, Executable } from '@rushstack/node-core-library';
+import { ITerminal, Executable, Async } from '@rushstack/node-core-library';
 import {
   ICloudBuildCacheProvider,
   ICredentialCacheEntry,
@@ -34,6 +34,7 @@ export interface IHttpBuildCacheProviderOptions {
   url: string;
   tokenHandler?: IHttpBuildCacheTokenHandler;
   uploadMethod?: string;
+  minHttpRetryDelayMs?: number;
   headers?: Record<string, string>;
   cacheKeyPrefix?: string;
   isCacheWriteAllowed: boolean;
@@ -41,6 +42,9 @@ export interface IHttpBuildCacheProviderOptions {
   rushProjectRoot: string;
 }
 
+const MAX_HTTP_CACHE_ATTEMPTS: number = 3;
+const DEFAULT_MIN_HTTP_RETRY_DELAY_MS = 2500;
+
 export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
   private readonly _pluginName: string;
   private readonly _rushSession: RushSession;
@@ -52,6 +56,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
   private readonly _headers: Record<string, string>;
   private readonly _cacheKeyPrefix: string;
   private readonly _tokenHandler: IHttpBuildCacheTokenHandler | undefined;
+  private readonly _minHttpRetryDelayMs: number;
   private __credentialCacheId: string | undefined;
 
   public get isCacheWriteAllowed(): boolean {
@@ -70,6 +75,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
     this._headers = options.headers ?? {};
     this._tokenHandler = options.tokenHandler;
     this._cacheKeyPrefix = options.cacheKeyPrefix ?? '';
+    this._minHttpRetryDelayMs = options.minHttpRetryDelayMs ?? DEFAULT_MIN_HTTP_RETRY_DELAY_MS;
   }
 
   public async tryGetCacheEntryBufferByIdAsync(
@@ -83,7 +89,8 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
         method: 'GET',
         body: undefined,
         warningText: 'Could not get cache entry',
-        readBody: true
+        readBody: true,
+        maxAttempts: MAX_HTTP_CACHE_ATTEMPTS
       });
 
       return Buffer.isBuffer(result) ? result : undefined;
@@ -112,7 +119,8 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
         method: this._uploadMethod,
         body: objectBuffer,
         warningText: 'Could not write cache entry',
-        readBody: false
+        readBody: false,
+        maxAttempts: MAX_HTTP_CACHE_ATTEMPTS
       });
 
       return result !== false;
@@ -197,6 +205,7 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
     body: BodyInit | undefined;
     warningText: string;
     readBody: boolean;
+    maxAttempts: number;
     credentialOptions?: CredentialsOptions;
   }): Promise<Buffer | boolean> {
     const { terminal, relUrl, method, body, warningText, readBody, credentialOptions } = options;
@@ -227,13 +236,33 @@ export class HttpBuildCacheProvider implements ICloudBuildCacheProvider {
     });
 
     if (!response.ok) {
-      if (typeof credentials !== 'string' && safeCredentialOptions === CredentialsOptions.Optional) {
-        // We tried fetching the resource without credentials and that did not work out
-        // Try again but require credentials this time
-        // This will trigger the provider to request credentials
+      const isNonCredentialResponse = response.status >= 500 && response.status < 600;
+
+      if (
+        !isNonCredentialResponse &&
+        typeof credentials !== 'string' &&
+        safeCredentialOptions === CredentialsOptions.Optional
+      ) {
+        // If we don't already have credentials yet, and we got a response from the server
+        // that is a "normal" failure (4xx), then we assume that credentials are probably
+        // required. Re-attempt the request, requiring credentials this time.
+        //
+        // This counts as part of the "first attempt", so it is not included in the max attempts
         return await this._http({ ...options, credentialOptions: CredentialsOptions.Required });
       }
 
+      if (options.maxAttempts > 1) {
+        // Pause a bit before retrying in case the server is busy
+        // Add some random jitter to the retry so we can spread out load on the remote service
+        // A proper solution might add exponential back off in case the retry count is high (10 or more)
+        const factor = 1.0 + Math.random(); // A random number between 1.0 and 2.0
+        const retryDelay = Math.floor(factor * this._minHttpRetryDelayMs);
+
+        await Async.sleep(retryDelay);
+
+        return await this._http({ ...options, maxAttempts: options.maxAttempts - 1 });
+      }
+
       this._reportFailure(terminal, method, response, false, warningText);
       return false;
     }

package/src/RushHttpBuildCachePlugin.ts

@@ -1,12 +1,9 @@
-import { Import } from '@rushstack/node-core-library';
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
 import type { IRushPlugin, RushSession, RushConfiguration } from '@rushstack/rush-sdk';
 import type { HttpBuildCacheProvider, IHttpBuildCacheProviderOptions } from './HttpBuildCacheProvider';
 
-const HttpBuildCacheProviderModule: typeof import('./HttpBuildCacheProvider') = Import.lazy(
-  './HttpBuildCacheProvider',
-  require
-);
-
 const PLUGIN_NAME: string = 'HttpBuildCachePlugin';
 
 /**
@@ -56,31 +53,29 @@ export class RushHttpBuildCachePlugin implements IRushPlugin {
 
   public apply(rushSession: RushSession, rushConfig: RushConfiguration): void {
     rushSession.hooks.initialize.tap(this.pluginName, () => {
-      rushSession.registerCloudBuildCacheProviderFactory(
-        'http',
-        (buildCacheConfig): HttpBuildCacheProvider => {
-          const config: IRushHttpBuildCachePluginConfig = (
-            buildCacheConfig as typeof buildCacheConfig & {
-              httpConfiguration: IRushHttpBuildCachePluginConfig;
-            }
-          ).httpConfiguration;
+      rushSession.registerCloudBuildCacheProviderFactory('http', async (buildCacheConfig) => {
+        const config: IRushHttpBuildCachePluginConfig = (
+          buildCacheConfig as typeof buildCacheConfig & {
+            httpConfiguration: IRushHttpBuildCachePluginConfig;
+          }
+        ).httpConfiguration;
 
-          const { url, uploadMethod, headers, tokenHandler, cacheKeyPrefix, isCacheWriteAllowed } = config;
+        const { url, uploadMethod, headers, tokenHandler, cacheKeyPrefix, isCacheWriteAllowed } = config;
 
-          const options: IHttpBuildCacheProviderOptions = {
-            pluginName: this.pluginName,
-            rushProjectRoot: rushConfig.rushJsonFolder,
-            url: url,
-            uploadMethod: uploadMethod,
-            headers: headers,
-            tokenHandler: tokenHandler,
-            cacheKeyPrefix: cacheKeyPrefix,
-            isCacheWriteAllowed: !!isCacheWriteAllowed
-          };
+        const options: IHttpBuildCacheProviderOptions = {
+          pluginName: this.pluginName,
+          rushProjectRoot: rushConfig.rushJsonFolder,
+          url: url,
+          uploadMethod: uploadMethod,
+          headers: headers,
+          tokenHandler: tokenHandler,
+          cacheKeyPrefix: cacheKeyPrefix,
+          isCacheWriteAllowed: !!isCacheWriteAllowed
+        };
 
-          return new HttpBuildCacheProviderModule.HttpBuildCacheProvider(options, rushSession);
-        }
-      );
+        const { HttpBuildCacheProvider } = await import('./HttpBuildCacheProvider');
+        return new HttpBuildCacheProvider(options, rushSession);
+      });
     });
   }
 }

package/src/test/HttpBuildCacheProvider.test.ts

@@ -0,0 +1,115 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+jest.mock('node-fetch', function () {
+  return Object.assign(jest.fn(), jest.requireActual('node-fetch'));
+});
+
+import fetch, { Response } from 'node-fetch';
+import { RushSession, EnvironmentConfiguration } from '@rushstack/rush-sdk';
+import { StringBufferTerminalProvider, Terminal } from '@rushstack/node-core-library';
+
+import { HttpBuildCacheProvider } from '../HttpBuildCacheProvider';
+
+const EXAMPLE_OPTIONS = {
+  url: 'https://buildcache.example.acme.com',
+  tokenHandler: {
+    exec: 'node',
+    args: ['tokenHandler.js']
+  },
+  uploadMethod: 'POST',
+  isCacheWriteAllowed: false,
+  pluginName: 'example-plugin',
+  rushProjectRoot: '/repo',
+  minHttpRetryDelayMs: 1
+};
+
+describe('HttpBuildCacheProvider', () => {
+  let terminalBuffer: StringBufferTerminalProvider;
+  let terminal!: Terminal;
+
+  beforeEach(() => {
+    terminalBuffer = new StringBufferTerminalProvider();
+    terminal = new Terminal(terminalBuffer);
+  });
+
+  describe('tryGetCacheEntryBufferByIdAsync', () => {
+    it('prints warning if read credentials are not available', async () => {
+      jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);
+
+      const session: RushSession = {} as RushSession;
+      const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);
+
+      mocked(fetch).mockResolvedValue(
+        new Response('Unauthorized', {
+          status: 401,
+          statusText: 'Unauthorized'
+        })
+      );
+
+      const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');
+      expect(result).toBe(undefined);
+      expect(fetch).toHaveBeenCalledTimes(1);
+      expect(fetch).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {
+        body: undefined,
+        headers: {},
+        method: 'GET',
+        redirect: 'follow'
+      });
+      expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(
+        `"Error getting cache entry: Error: Credentials for https://buildcache.example.acme.com/ have not been provided.[n]In CI, verify that RUSH_BUILD_CACHE_CREDENTIAL contains a valid Authorization header value.[n][n]For local developers, run:[n][n]    rush update-cloud-credentials --interactive[n][n]"`
+      );
+    });
+
+    it('attempts up to 3 times to download a cache entry', async () => {
+      jest.spyOn(EnvironmentConfiguration, 'buildCacheCredential', 'get').mockReturnValue(undefined);
+
+      const session: RushSession = {} as RushSession;
+      const provider = new HttpBuildCacheProvider(EXAMPLE_OPTIONS, session);
+
+      mocked(fetch).mockResolvedValueOnce(
+        new Response('InternalServiceError', {
+          status: 500,
+          statusText: 'InternalServiceError'
+        })
+      );
+      mocked(fetch).mockResolvedValueOnce(
+        new Response('ServiceUnavailable', {
+          status: 503,
+          statusText: 'ServiceUnavailable'
+        })
+      );
+      mocked(fetch).mockResolvedValueOnce(
+        new Response('BadGateway', {
+          status: 504,
+          statusText: 'BadGateway'
+        })
+      );
+
+      const result = await provider.tryGetCacheEntryBufferByIdAsync(terminal, 'some-key');
+      expect(result).toBe(undefined);
+      expect(fetch).toHaveBeenCalledTimes(3);
+      expect(fetch).toHaveBeenNthCalledWith(1, 'https://buildcache.example.acme.com/some-key', {
+        body: undefined,
+        headers: {},
+        method: 'GET',
+        redirect: 'follow'
+      });
+      expect(fetch).toHaveBeenNthCalledWith(2, 'https://buildcache.example.acme.com/some-key', {
+        body: undefined,
+        headers: {},
+        method: 'GET',
+        redirect: 'follow'
+      });
+      expect(fetch).toHaveBeenNthCalledWith(3, 'https://buildcache.example.acme.com/some-key', {
+        body: undefined,
+        headers: {},
+        method: 'GET',
+        redirect: 'follow'
+      });
+      expect(terminalBuffer.getWarningOutput()).toMatchInlineSnapshot(
+        `"Could not get cache entry: HTTP 504: BadGateway[n]"`
+      );
+    });
+  });
+});