@internetarchive/fetch-handler 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +29 -0
- package/.github/workflows/ci.yml +27 -0
- package/.github/workflows/gh-pages-main.yml +40 -0
- package/.github/workflows/pr-preview.yml +63 -0
- package/.husky/pre-commit +4 -0
- package/.prettierignore +1 -0
- package/.vscode/extensions.json +12 -0
- package/.vscode/tasks.json +12 -0
- package/LICENSE +661 -0
- package/README.md +82 -0
- package/coverage/.gitignore +3 -0
- package/coverage/.placeholder +0 -0
- package/demo/app-root.ts +84 -0
- package/demo/index.html +27 -0
- package/dist/demo/app-root.d.ts +12 -0
- package/dist/demo/app-root.js +90 -0
- package/dist/demo/app-root.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -0
- package/dist/src/fetch-handler-interface.d.ts +33 -0
- package/dist/src/fetch-handler-interface.js +2 -0
- package/dist/src/fetch-handler-interface.js.map +1 -0
- package/dist/src/ia-fetch-handler.d.ts +36 -0
- package/dist/src/ia-fetch-handler.js +70 -0
- package/dist/src/ia-fetch-handler.js.map +1 -0
- package/dist/src/utils/fetch-retrier.d.ts +36 -0
- package/dist/src/utils/fetch-retrier.js +94 -0
- package/dist/src/utils/fetch-retrier.js.map +1 -0
- package/dist/src/utils/promised-sleep.d.ts +13 -0
- package/dist/src/utils/promised-sleep.js +16 -0
- package/dist/src/utils/promised-sleep.js.map +1 -0
- package/dist/test/fetch-retrier.test.d.ts +1 -0
- package/dist/test/fetch-retrier.test.js +139 -0
- package/dist/test/fetch-retrier.test.js.map +1 -0
- package/dist/test/ia-fetch-handler.test.d.ts +1 -0
- package/dist/test/ia-fetch-handler.test.js +50 -0
- package/dist/test/ia-fetch-handler.test.js.map +1 -0
- package/dist/test/mocks/mock-analytics-handler.d.ts +20 -0
- package/dist/test/mocks/mock-analytics-handler.js +28 -0
- package/dist/test/mocks/mock-analytics-handler.js.map +1 -0
- package/dist/vite.config.d.ts +2 -0
- package/dist/vite.config.js +25 -0
- package/dist/vite.config.js.map +1 -0
- package/eslint.config.mjs +53 -0
- package/index.ts +2 -0
- package/package.json +74 -0
- package/renovate.json +6 -0
- package/screenshot/gh-pages-settings.png +0 -0
- package/src/fetch-handler-interface.ts +39 -0
- package/src/ia-fetch-handler.ts +94 -0
- package/src/utils/fetch-retrier.ts +141 -0
- package/src/utils/promised-sleep.ts +15 -0
- package/ssl/server.crt +22 -0
- package/ssl/server.key +28 -0
- package/test/fetch-retrier.test.ts +173 -0
- package/test/ia-fetch-handler.test.ts +66 -0
- package/test/mocks/mock-analytics-handler.ts +43 -0
- package/tsconfig.json +31 -0
- package/vite.config.ts +25 -0
- package/web-dev-server.config.mjs +32 -0
- package/web-test-runner.config.mjs +41 -0
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
import type { AnalyticsHandlerInterface } from '@internetarchive/analytics-manager';
|
|
2
|
+
import { AnalyticsHandler } from '@internetarchive/analytics-manager';
|
|
3
|
+
import { promisedSleep } from './promised-sleep';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A class that retries a fetch request.
|
|
7
|
+
*/
|
|
8
|
+
export interface FetchRetrierInterface {
|
|
9
|
+
/**
|
|
10
|
+
* Execute a fetch with retry.
|
|
11
|
+
*
|
|
12
|
+
* @param requestInfo RequestInfo
|
|
13
|
+
* @param init Optional RequestInit
|
|
14
|
+
* @param retries Optional number of retries to attempt
|
|
15
|
+
* @returns Promise<Response>
|
|
16
|
+
*/
|
|
17
|
+
fetchRetry(
|
|
18
|
+
requestInfo: RequestInfo,
|
|
19
|
+
init?: RequestInit,
|
|
20
|
+
retries?: number,
|
|
21
|
+
): Promise<Response>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** @inheritdoc */
|
|
25
|
+
export class FetchRetrier implements FetchRetrierInterface {
|
|
26
|
+
private analyticsHandler = new AnalyticsHandler({ enableAnalytics: true });
|
|
27
|
+
private sleep = promisedSleep; // default sleep function
|
|
28
|
+
|
|
29
|
+
private retryCount = 2;
|
|
30
|
+
|
|
31
|
+
private retryDelay = 1000;
|
|
32
|
+
|
|
33
|
+
constructor(options?: {
|
|
34
|
+
analyticsHandler?: AnalyticsHandlerInterface;
|
|
35
|
+
retryCount?: number;
|
|
36
|
+
retryDelay?: number;
|
|
37
|
+
sleepFn?: (ms: number) => Promise<void>; // <-- add this!
|
|
38
|
+
}) {
|
|
39
|
+
if (options?.analyticsHandler)
|
|
40
|
+
this.analyticsHandler = options.analyticsHandler;
|
|
41
|
+
if (options?.retryCount) this.retryCount = options.retryCount;
|
|
42
|
+
if (options?.retryDelay) this.retryDelay = options.retryDelay;
|
|
43
|
+
if (options?.sleepFn) this.sleep = options.sleepFn; // override if provided
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** @inheritdoc */
|
|
47
|
+
public async fetchRetry(
|
|
48
|
+
requestInfo: RequestInfo,
|
|
49
|
+
init?: RequestInit,
|
|
50
|
+
retries: number = this.retryCount,
|
|
51
|
+
): Promise<Response> {
|
|
52
|
+
const urlString =
|
|
53
|
+
typeof requestInfo === 'string' ? requestInfo : requestInfo.url;
|
|
54
|
+
const retryNumber = this.retryCount - retries + 1;
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
const response = await fetch(requestInfo, init);
|
|
58
|
+
if (response.ok) return response;
|
|
59
|
+
// don't retry on a 404 since this will never succeed
|
|
60
|
+
if (response.status === 404) {
|
|
61
|
+
this.log404Event(urlString);
|
|
62
|
+
return response;
|
|
63
|
+
}
|
|
64
|
+
if (retries > 0) {
|
|
65
|
+
await this.sleep(this.retryDelay);
|
|
66
|
+
this.logRetryEvent(
|
|
67
|
+
urlString,
|
|
68
|
+
retryNumber,
|
|
69
|
+
response.statusText,
|
|
70
|
+
response.status,
|
|
71
|
+
);
|
|
72
|
+
return this.fetchRetry(requestInfo, init, retries - 1);
|
|
73
|
+
}
|
|
74
|
+
this.logFailureEvent(urlString, response.status);
|
|
75
|
+
return response;
|
|
76
|
+
} catch (error) {
|
|
77
|
+
// if a content blocker is detected, log it and don't retry
|
|
78
|
+
if (this.isContentBlockerError(error)) {
|
|
79
|
+
this.logContentBlockingEvent(urlString, error);
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (retries > 0) {
|
|
84
|
+
await this.sleep(this.retryDelay);
|
|
85
|
+
// intentionally duplicating the error message here since we want something
|
|
86
|
+
// in the status code even though we won't have an actual code
|
|
87
|
+
this.logRetryEvent(urlString, retryNumber, error, error);
|
|
88
|
+
return this.fetchRetry(requestInfo, init, retries - 1);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.logFailureEvent(urlString, error);
|
|
92
|
+
throw error;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
private isContentBlockerError(error: unknown): boolean {
|
|
97
|
+
// all of the content blocker errors are `TypeError`
|
|
98
|
+
if (!(error instanceof TypeError)) return false;
|
|
99
|
+
const message = error.message.toLowerCase();
|
|
100
|
+
return message.includes('content blocker');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private readonly eventCategory = 'offshootFetchRetry';
|
|
104
|
+
|
|
105
|
+
private logRetryEvent(
|
|
106
|
+
urlString: string,
|
|
107
|
+
retryNumber: number,
|
|
108
|
+
status: unknown,
|
|
109
|
+
code: unknown,
|
|
110
|
+
) {
|
|
111
|
+
this.analyticsHandler.sendEvent({
|
|
112
|
+
category: this.eventCategory,
|
|
113
|
+
action: 'retryingFetch',
|
|
114
|
+
label: `retryNumber: ${retryNumber} / ${this.retryCount}, code: ${code}, status: ${status}, url: ${urlString}`,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private logFailureEvent(urlString: string, error: unknown) {
|
|
119
|
+
this.analyticsHandler.sendEvent({
|
|
120
|
+
category: this.eventCategory,
|
|
121
|
+
action: 'fetchFailed',
|
|
122
|
+
label: `error: ${error}, url: ${urlString}`,
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private log404Event(urlString: string) {
|
|
127
|
+
this.analyticsHandler.sendEvent({
|
|
128
|
+
category: this.eventCategory,
|
|
129
|
+
action: 'status404NotRetrying',
|
|
130
|
+
label: `url: ${urlString}`,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
private logContentBlockingEvent(urlString: string, error: unknown) {
|
|
135
|
+
this.analyticsHandler.sendEvent({
|
|
136
|
+
category: this.eventCategory,
|
|
137
|
+
action: 'contentBlockerDetectedNotRetrying',
|
|
138
|
+
label: `error: ${error}, url: ${urlString}`,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Asynchronously pause execution for a given number of milliseconds.
|
|
3
|
+
*
|
|
4
|
+
* Used for waiting for some asynchronous updates.
|
|
5
|
+
*
|
|
6
|
+
* Usage:
|
|
7
|
+
* await promisedSleep(100)
|
|
8
|
+
*
|
|
9
|
+
* @export
|
|
10
|
+
* @param {number} ms
|
|
11
|
+
* @returns {Promise<void>}
|
|
12
|
+
*/
|
|
13
|
+
export function promisedSleep(ms: number): Promise<void> {
|
|
14
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
15
|
+
}
|
package/ssl/server.crt
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
|
2
|
+
MIIDszCCApugAwIBAgIUYr5csS2ntaMTww0b5QoFI0xlOpUwDQYJKoZIhvcNAQEL
|
|
3
|
+
BQAwaTELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAkNBMRYwFAYDVQQHDA1TYW4gRnJh
|
|
4
|
+
bmNpc2NvMRkwFwYDVQQKDBBJbnRlcm5ldCBBcmNoaXZlMRowGAYDVQQDDBFsb2Nh
|
|
5
|
+
bC5hcmNoaXZlLm9yZzAeFw0yNTAzMDcxOTM1MzRaFw0yNTA0MDYxOTM1MzRaMGkx
|
|
6
|
+
CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNj
|
|
7
|
+
bzEZMBcGA1UECgwQSW50ZXJuZXQgQXJjaGl2ZTEaMBgGA1UEAwwRbG9jYWwuYXJj
|
|
8
|
+
aGl2ZS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDZqbS9hNS3
|
|
9
|
+
8AtcA7Uszq7EA7RavuHc8AcpbURa6Y7NORs0SmfDD0oLptQ7DX5pSuHyOeDHzQ3U
|
|
10
|
+
XRCUDAhAD1XlaYQ6PPdiJPRAe4ACELzgDzepxVz3pdKPcfXd5/mHV3ghUZya2XW5
|
|
11
|
+
DSF1CRYUX2j10a+JORJIG4bhvTg4HxTJZZDuaNfBMC6ZSn7d9YLQoAf1MLLJm43Y
|
|
12
|
+
cj1WlOnyG4DHxNEqEk260BlbBywJH+SJA6GA0GR2bnzYZaDnZ0wqea0Zamx8ijpd
|
|
13
|
+
JxBUxYOqLfzs2KREVkkGOmsFEu82W6G3D4bKsY1UNaWXb/jcl3UkXIJJij6CdbbM
|
|
14
|
+
QkUJc8dow3sVAgMBAAGjUzBRMB0GA1UdDgQWBBSla0L0Yhpx9/DEV4uJ6OoHH92Q
|
|
15
|
+
bTAfBgNVHSMEGDAWgBSla0L0Yhpx9/DEV4uJ6OoHH92QbTAPBgNVHRMBAf8EBTAD
|
|
16
|
+
AQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBEzt9sgWtwjLI7AVrejDSE6F/01ng4bTni
|
|
17
|
+
0PJXa7IpwjhAApVWcjxDKp8SkDUAcZNULqBGvpPicqjnKf+wxk3xTMc7pvLqzOor
|
|
18
|
+
pm+plDUBfCEbrv1ouLDMgOtxQxbwbBUPySsLoNmDUGAGnL2zENX7qidOOmmSQEN4
|
|
19
|
+
jZuPtiEkgTqmtkfHaRX/7jhmGVfxMxj0Krlar1/cSbvwaVs4kRSjiL9/avT+AX59
|
|
20
|
+
0QoNYTLGcBP0RKia8NF2A3cgUivIMFZQDvdsiAvqY+DbC7Mnyv4Sj/ZHHwVa9+8k
|
|
21
|
+
iRU56DrOt7sgaa/oQfzWnvRtL0F5eXC3veqM4qZrXs9hHeGlgPFo
|
|
22
|
+
-----END CERTIFICATE-----
|
package/ssl/server.key
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
-----BEGIN PRIVATE KEY-----
|
|
2
|
+
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDZqbS9hNS38Atc
|
|
3
|
+
A7Uszq7EA7RavuHc8AcpbURa6Y7NORs0SmfDD0oLptQ7DX5pSuHyOeDHzQ3UXRCU
|
|
4
|
+
DAhAD1XlaYQ6PPdiJPRAe4ACELzgDzepxVz3pdKPcfXd5/mHV3ghUZya2XW5DSF1
|
|
5
|
+
CRYUX2j10a+JORJIG4bhvTg4HxTJZZDuaNfBMC6ZSn7d9YLQoAf1MLLJm43Ycj1W
|
|
6
|
+
lOnyG4DHxNEqEk260BlbBywJH+SJA6GA0GR2bnzYZaDnZ0wqea0Zamx8ijpdJxBU
|
|
7
|
+
xYOqLfzs2KREVkkGOmsFEu82W6G3D4bKsY1UNaWXb/jcl3UkXIJJij6CdbbMQkUJ
|
|
8
|
+
c8dow3sVAgMBAAECggEANgLpITAhcuVDhFk9L3m4H1bF/dCpDmCXfl2pXR/guicm
|
|
9
|
+
C4M9HUeheaOzvVWbXThiOe/HyfylpmFTmFEmCPNlPrDAyYzQXE/MNmYO/TQ3Eihk
|
|
10
|
+
iSG68I764XKHbsG+Byoa2rW8NSaqEjniZ/7Rtkt4qasXMmdxlGgUP9bq6O45g8HV
|
|
11
|
+
kHxneFlA2KbrvmWnBi7NTea9+tp61NWiq5n97UgHacQR6KkYIpbxd7uNSnCSdmXt
|
|
12
|
+
pcwzO4ZPabJ2/DKRjePhU5OggPh+9AhJ3jsBo99GwYPgSZDh8E3vh2OWOtLBMUH3
|
|
13
|
+
rmAYwlRT2aZ5hy5yi+4QD98WoUQtsr+9n757F8V6MQKBgQDzZycdgaKwWd5d34NN
|
|
14
|
+
0TkFtPQPwxxJTCCs3I3q02uWcS3svQzBK0cWb4nISO04TnJ3MnXo83dgGhCMF+uZ
|
|
15
|
+
FCXxAA53z8F92iZo+ELlXFDFwNeNYih/afDA42tWZ6TlVsDnZ4zQgRjHS7jEF/JV
|
|
16
|
+
0ZgwGpw5725JQt64dic8wTOcDQKBgQDk7YYACQcWTnmjDhZQ3PZSX4fcTtzPZKZj
|
|
17
|
+
fa1GrXEaUH1hSyc9VmY6qJpUmXexpvtr194nXE+O5wbThOHcBjVlo2Qv+vswmUX3
|
|
18
|
+
WEcVzTVN4/fODCLCqFcMNIr+BzwZfwpT+p0u8g9FxDy1gGmvkxwIu8DCpnUT12Xj
|
|
19
|
+
hm2wO+UxKQKBgFxCSDBF9+2SUtgQJYv0dwGzwiLLWMhro6MCAoT02D3w7nBihBgg
|
|
20
|
+
GFTnuDkDc285ROfrZ4gB6MizeHwxgOrIGU2NMO62/+d9LbvyBiE76Z3bZ5i+kQ0i
|
|
21
|
+
kc/7I69fn8ASLxJHTLenh0XbbNBfJ0riJCZvn7HSEGKShysyFdNQhAhtAoGAFYVi
|
|
22
|
+
0IQIv4cXFkYPwQBUw7+pVQOw7GpI3heFf5x0goXIk6nuAW0q5R7Oi192CiRphGTh
|
|
23
|
+
xI+ABy4ezSmz1exbfreShpQwowv1sOACpsEI3s6skBlB90y+Ci6yVlk1xCvWO7jW
|
|
24
|
+
qAAngWaGUoXE6bWJsCR+ZY4ieYAJWw9bJnMrA6kCgYAzV4Xeoh5ofENZM21wKW3W
|
|
25
|
+
iCzRibPObv6Vb/j9A9yT57yzI3BdfvsX5zmuSvOJm1DimgGY9nCJUzUEYG0a1Dhh
|
|
26
|
+
/rqObPoVIFGesmvwflVYFktmVHk7ycEDVreSTz23cvmraPz1fnpdKeuEs4sRQJV7
|
|
27
|
+
iJhLoxX5SJlJc0RXMhgHGQ==
|
|
28
|
+
-----END PRIVATE KEY-----
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { expect } from '@open-wc/testing';
|
|
2
|
+
import sinon from 'sinon';
|
|
3
|
+
import { FetchRetrier } from '../src/utils/fetch-retrier';
|
|
4
|
+
import { MockAnalyticsHandler } from './mocks/mock-analytics-handler';
|
|
5
|
+
|
|
6
|
+
describe('FetchRetrier', () => {
|
|
7
|
+
let fetchStub: sinon.SinonStub;
|
|
8
|
+
let sleepStub: sinon.SinonStub;
|
|
9
|
+
let analytics: MockAnalyticsHandler;
|
|
10
|
+
|
|
11
|
+
beforeEach(() => {
|
|
12
|
+
analytics = new MockAnalyticsHandler();
|
|
13
|
+
fetchStub = sinon.stub(globalThis, 'fetch');
|
|
14
|
+
sleepStub = sinon.stub().resolves(); // stubbed promisedSleep
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
afterEach(() => {
|
|
18
|
+
fetchStub.restore();
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('returns response on first success', async () => {
|
|
22
|
+
fetchStub.resolves(new Response('ok', { status: 200 }));
|
|
23
|
+
const retrier = new FetchRetrier({
|
|
24
|
+
analyticsHandler: analytics,
|
|
25
|
+
sleepFn: sleepStub,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
const res = await retrier.fetchRetry('https://foo.org/data');
|
|
29
|
+
|
|
30
|
+
expect(res.status).to.equal(200);
|
|
31
|
+
expect(fetchStub.callCount).to.equal(1);
|
|
32
|
+
expect(sleepStub.callCount).to.equal(0);
|
|
33
|
+
expect(analytics.events.length).to.equal(0);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('does not retry on 404 and logs event', async () => {
|
|
37
|
+
fetchStub.resolves(new Response('not found', { status: 404 }));
|
|
38
|
+
const retrier = new FetchRetrier({
|
|
39
|
+
analyticsHandler: analytics,
|
|
40
|
+
sleepFn: sleepStub,
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const res = await retrier.fetchRetry('https://foo.org/404');
|
|
44
|
+
|
|
45
|
+
expect(res.status).to.equal(404);
|
|
46
|
+
expect(fetchStub.callCount).to.equal(1);
|
|
47
|
+
expect(sleepStub.callCount).to.equal(0);
|
|
48
|
+
expect(analytics.events[0].action).to.equal('status404NotRetrying');
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('retries on 500 and logs retry/failure events', async () => {
|
|
52
|
+
fetchStub.onCall(0).resolves(new Response('fail', { status: 500 }));
|
|
53
|
+
fetchStub.onCall(1).resolves(new Response('fail again', { status: 500 }));
|
|
54
|
+
fetchStub.onCall(2).resolves(new Response('still fail', { status: 500 }));
|
|
55
|
+
|
|
56
|
+
const retrier = new FetchRetrier({
|
|
57
|
+
analyticsHandler: analytics,
|
|
58
|
+
retryCount: 2,
|
|
59
|
+
retryDelay: 1,
|
|
60
|
+
sleepFn: sleepStub,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const res = await retrier.fetchRetry('https://foo.org/fail');
|
|
64
|
+
|
|
65
|
+
expect(res.status).to.equal(500);
|
|
66
|
+
expect(fetchStub.callCount).to.equal(3);
|
|
67
|
+
expect(sleepStub.callCount).to.equal(2);
|
|
68
|
+
expect(analytics.events.some(e => e.action === 'retryingFetch')).to.be.true;
|
|
69
|
+
expect(analytics.events.some(e => e.action === 'fetchFailed')).to.be.true;
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('retries on fetch error and eventually succeeds', async () => {
|
|
73
|
+
fetchStub.onCall(0).rejects(new Error('Network error'));
|
|
74
|
+
fetchStub.onCall(1).resolves(new Response('ok', { status: 200 }));
|
|
75
|
+
|
|
76
|
+
const retrier = new FetchRetrier({
|
|
77
|
+
analyticsHandler: analytics,
|
|
78
|
+
retryCount: 1,
|
|
79
|
+
retryDelay: 1,
|
|
80
|
+
sleepFn: sleepStub,
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const res = await retrier.fetchRetry('https://foo.org/retry');
|
|
84
|
+
|
|
85
|
+
expect(res.status).to.equal(200);
|
|
86
|
+
expect(fetchStub.callCount).to.equal(2);
|
|
87
|
+
expect(sleepStub.calledOnce).to.be.true;
|
|
88
|
+
expect(analytics.events.some(e => e.action === 'retryingFetch')).to.be.true;
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('throws and logs when retries are exhausted due to network error', async () => {
|
|
92
|
+
fetchStub.rejects(new Error('Boom'));
|
|
93
|
+
|
|
94
|
+
const retrier = new FetchRetrier({
|
|
95
|
+
analyticsHandler: analytics,
|
|
96
|
+
retryCount: 1,
|
|
97
|
+
retryDelay: 1,
|
|
98
|
+
sleepFn: sleepStub,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
await retrier.fetchRetry('https://foo.org/networkfail');
|
|
103
|
+
throw new Error('Should have thrown');
|
|
104
|
+
} catch (err: any) {
|
|
105
|
+
expect(err.message).to.equal('Boom');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
expect(fetchStub.callCount).to.equal(2);
|
|
109
|
+
expect(sleepStub.callCount).to.equal(1);
|
|
110
|
+
expect(analytics.events.some(e => e.action === 'fetchFailed')).to.be.true;
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it('detects content blocker error and does not retry', async () => {
|
|
114
|
+
const blockerError = new TypeError('Content Blocker denied request');
|
|
115
|
+
fetchStub.rejects(blockerError);
|
|
116
|
+
|
|
117
|
+
const retrier = new FetchRetrier({
|
|
118
|
+
analyticsHandler: analytics,
|
|
119
|
+
retryCount: 2,
|
|
120
|
+
sleepFn: sleepStub,
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
try {
|
|
124
|
+
await retrier.fetchRetry('https://foo.org/blocked');
|
|
125
|
+
throw new Error('Should have thrown');
|
|
126
|
+
} catch (err: any) {
|
|
127
|
+
expect(err).to.equal(blockerError);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
expect(fetchStub.callCount).to.equal(1);
|
|
131
|
+
expect(sleepStub.callCount).to.equal(0);
|
|
132
|
+
expect(
|
|
133
|
+
analytics.events.some(
|
|
134
|
+
e => e.action === 'contentBlockerDetectedNotRetrying',
|
|
135
|
+
),
|
|
136
|
+
).to.be.true;
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('calls sleepFn on retry with correct delay', async () => {
|
|
140
|
+
fetchStub.onCall(0).resolves(new Response(null, { status: 500 }));
|
|
141
|
+
fetchStub.onCall(1).resolves(new Response('ok', { status: 200 }));
|
|
142
|
+
|
|
143
|
+
const retrier = new FetchRetrier({
|
|
144
|
+
analyticsHandler: analytics,
|
|
145
|
+
retryCount: 2,
|
|
146
|
+
retryDelay: 1234,
|
|
147
|
+
sleepFn: sleepStub,
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
const res = await retrier.fetchRetry('https://foo.org/retry-with-sleep');
|
|
151
|
+
|
|
152
|
+
expect(res.status).to.equal(200);
|
|
153
|
+
expect(sleepStub.calledOnce).to.be.true;
|
|
154
|
+
expect(sleepStub.calledWith(1234)).to.be.true;
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it('sleeps for each retry attempt', async () => {
|
|
158
|
+
fetchStub.resolves(new Response(null, { status: 500 }));
|
|
159
|
+
|
|
160
|
+
const retrier = new FetchRetrier({
|
|
161
|
+
analyticsHandler: analytics,
|
|
162
|
+
retryCount: 2,
|
|
163
|
+
retryDelay: 300,
|
|
164
|
+
sleepFn: sleepStub,
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
const res = await retrier.fetchRetry('https://foo.org/retry-fail');
|
|
168
|
+
|
|
169
|
+
expect(res.status).to.equal(500);
|
|
170
|
+
expect(sleepStub.callCount).to.equal(2);
|
|
171
|
+
expect(sleepStub.alwaysCalledWith(300)).to.be.true;
|
|
172
|
+
});
|
|
173
|
+
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { expect } from '@open-wc/testing';
|
|
2
|
+
import { IaFetchHandler } from '../src/ia-fetch-handler';
|
|
3
|
+
import { FetchRetrierInterface } from '../src/utils/fetch-retrier';
|
|
4
|
+
|
|
5
|
+
class MockFetchRetrier implements FetchRetrierInterface {
|
|
6
|
+
requestInfo?: RequestInfo;
|
|
7
|
+
init?: RequestInit;
|
|
8
|
+
retries?: number;
|
|
9
|
+
|
|
10
|
+
async fetchRetry(
|
|
11
|
+
requestInfo: RequestInfo,
|
|
12
|
+
init?: RequestInit,
|
|
13
|
+
retries?: number,
|
|
14
|
+
): Promise<Response> {
|
|
15
|
+
this.init = init;
|
|
16
|
+
this.requestInfo = requestInfo;
|
|
17
|
+
this.retries = retries;
|
|
18
|
+
return new Response(JSON.stringify({ boop: 'snoot' }), { status: 200 });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
describe('Fetch Handler', () => {
|
|
23
|
+
describe('fetch', () => {
|
|
24
|
+
it('adds reCache=1 if it is in the current url', async () => {
|
|
25
|
+
const fetchRetrier = new MockFetchRetrier();
|
|
26
|
+
const fetchHandler = new IaFetchHandler({
|
|
27
|
+
fetchRetrier: fetchRetrier,
|
|
28
|
+
searchParams: '?reCache=1',
|
|
29
|
+
});
|
|
30
|
+
await fetchHandler.fetch('https://foo.org/api/v1/snoot');
|
|
31
|
+
expect(fetchRetrier.requestInfo).to.equal(
|
|
32
|
+
'https://foo.org/api/v1/snoot?reCache=1',
|
|
33
|
+
);
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
describe('fetchIAApiResponse', () => {
|
|
38
|
+
it('prepends the IA basehost to the url when making a request', async () => {
|
|
39
|
+
const endpoint = '/foo/service/endpoint.php';
|
|
40
|
+
const fetchRetrier = new MockFetchRetrier();
|
|
41
|
+
const fetchHandler = new IaFetchHandler({
|
|
42
|
+
iaApiBaseUrl: 'www.example.com',
|
|
43
|
+
fetchRetrier: fetchRetrier,
|
|
44
|
+
});
|
|
45
|
+
await fetchHandler.fetchIAApiResponse(endpoint);
|
|
46
|
+
expect(fetchRetrier.requestInfo).to.equal(
|
|
47
|
+
'www.example.com/foo/service/endpoint.php',
|
|
48
|
+
);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
describe('fetchApiResponse', () => {
|
|
53
|
+
it('adds credentials: include if requested', async () => {
|
|
54
|
+
const endpoint = '/foo/service/endpoint.php';
|
|
55
|
+
const fetchRetrier = new MockFetchRetrier();
|
|
56
|
+
const fetchHandler = new IaFetchHandler({
|
|
57
|
+
iaApiBaseUrl: 'www.example.com',
|
|
58
|
+
fetchRetrier: fetchRetrier,
|
|
59
|
+
});
|
|
60
|
+
await fetchHandler.fetchApiResponse(endpoint, {
|
|
61
|
+
includeCredentials: true,
|
|
62
|
+
});
|
|
63
|
+
expect(fetchRetrier.init).to.deep.equal({ credentials: 'include' });
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { AnalyticsEvent } from '@internetarchive/analytics-manager';
|
|
2
|
+
import type { AnalyticsHandlerInterface } from '@internetarchive/analytics-manager';
|
|
3
|
+
|
|
4
|
+
export type MockAnalyticsEvent = AnalyticsEvent & {
|
|
5
|
+
bucketType: '1%' | '100%';
|
|
6
|
+
additionalEventParams?: object;
|
|
7
|
+
};
|
|
8
|
+
export class MockAnalyticsHandler implements AnalyticsHandlerInterface {
|
|
9
|
+
events: MockAnalyticsEvent[] = [];
|
|
10
|
+
|
|
11
|
+
sendPing(values: Record<string, any>): void {}
|
|
12
|
+
sendEvent(event: MockAnalyticsEvent): void {
|
|
13
|
+
const thisEvent = Object.assign({}, event, { bucketType: '1%' });
|
|
14
|
+
this.events.push(thisEvent);
|
|
15
|
+
}
|
|
16
|
+
send_event(
|
|
17
|
+
category: string,
|
|
18
|
+
action: string,
|
|
19
|
+
label?: string,
|
|
20
|
+
additionalEventParams?: object,
|
|
21
|
+
): void {
|
|
22
|
+
this.events.push({
|
|
23
|
+
category,
|
|
24
|
+
action,
|
|
25
|
+
label,
|
|
26
|
+
bucketType: '1%',
|
|
27
|
+
additionalEventParams: { ...additionalEventParams },
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
sendEventNoSampling(event: AnalyticsEvent): void {
|
|
31
|
+
this.events.push({
|
|
32
|
+
...event,
|
|
33
|
+
bucketType: '100%',
|
|
34
|
+
} as unknown as MockAnalyticsEvent);
|
|
35
|
+
}
|
|
36
|
+
trackIaxParameter(location: string): void {}
|
|
37
|
+
trackPageView(options?: {
|
|
38
|
+
mediaType?: string;
|
|
39
|
+
mediaLanguage?: string;
|
|
40
|
+
primaryCollection?: string;
|
|
41
|
+
page?: string;
|
|
42
|
+
}): void {}
|
|
43
|
+
}
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "es2018",
|
|
4
|
+
"module": "esnext",
|
|
5
|
+
"moduleResolution": "node",
|
|
6
|
+
"noEmitOnError": true,
|
|
7
|
+
"lib": ["es2017", "dom"],
|
|
8
|
+
"strict": true,
|
|
9
|
+
"esModuleInterop": false,
|
|
10
|
+
"allowSyntheticDefaultImports": true,
|
|
11
|
+
"experimentalDecorators": true,
|
|
12
|
+
"importHelpers": true,
|
|
13
|
+
"outDir": "dist",
|
|
14
|
+
"sourceMap": true,
|
|
15
|
+
"inlineSources": true,
|
|
16
|
+
"rootDir": "./",
|
|
17
|
+
"declaration": true,
|
|
18
|
+
"plugins": [
|
|
19
|
+
{
|
|
20
|
+
"name": "ts-lit-plugin"
|
|
21
|
+
}
|
|
22
|
+
],
|
|
23
|
+
"paths": {
|
|
24
|
+
// workaround for: https://github.com/vitest-dev/vitest/issues/4567
|
|
25
|
+
"rollup/parseAst": [
|
|
26
|
+
"./node_modules/rollup/dist/parseAst"
|
|
27
|
+
]
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
"include": ["**/*.ts"]
|
|
31
|
+
}
|
package/vite.config.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { defineConfig } from 'vite';
|
|
2
|
+
import { resolve } from 'path';
|
|
3
|
+
|
|
4
|
+
// https://vitejs.dev/config/
|
|
5
|
+
export default defineConfig({
|
|
6
|
+
base: './',
|
|
7
|
+
root: resolve(__dirname, './demo'),
|
|
8
|
+
build: {
|
|
9
|
+
/**
|
|
10
|
+
* This is the directory where the built files will be placed
|
|
11
|
+
* that we upload to GitHub Pages.
|
|
12
|
+
*/
|
|
13
|
+
outDir: '../ghpages/demo',
|
|
14
|
+
emptyOutDir: true,
|
|
15
|
+
manifest: true,
|
|
16
|
+
rollupOptions: {
|
|
17
|
+
input: {
|
|
18
|
+
main: resolve(__dirname, 'demo/index.html'),
|
|
19
|
+
},
|
|
20
|
+
output: {
|
|
21
|
+
entryFileNames: 'app-root.js',
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
},
|
|
25
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// import { hmrPlugin, presets } from '@open-wc/dev-server-hmr';
|
|
2
|
+
|
|
3
|
+
/** Use Hot Module replacement by adding --hmr to the start command */
|
|
4
|
+
const hmr = process.argv.includes('--hmr');
|
|
5
|
+
|
|
6
|
+
export default /** @type {import('@web/dev-server').DevServerConfig} */ ({
|
|
7
|
+
nodeResolve: true,
|
|
8
|
+
open: '/demo',
|
|
9
|
+
watch: !hmr,
|
|
10
|
+
sslCert: './ssl/server.crt',
|
|
11
|
+
sslKey: './ssl/server.key',
|
|
12
|
+
hostname: 'local.archive.org',
|
|
13
|
+
http2: true,
|
|
14
|
+
|
|
15
|
+
/** Compile JS for older browsers. Requires @web/dev-server-esbuild plugin */
|
|
16
|
+
// esbuildTarget: 'auto'
|
|
17
|
+
|
|
18
|
+
/** Set appIndex to enable SPA routing */
|
|
19
|
+
// appIndex: 'demo/index.html',
|
|
20
|
+
|
|
21
|
+
/** Configure bare import resolve plugin */
|
|
22
|
+
// nodeResolve: {
|
|
23
|
+
// exportConditions: ['browser', 'development']
|
|
24
|
+
// },
|
|
25
|
+
|
|
26
|
+
plugins: [
|
|
27
|
+
/** Use Hot Module Replacement by uncommenting. Requires @open-wc/dev-server-hmr plugin */
|
|
28
|
+
// hmr && hmrPlugin({ exclude: ['**/*/node_modules/**/*'], presets: [presets.litElement] }),
|
|
29
|
+
],
|
|
30
|
+
|
|
31
|
+
// See documentation for all available options
|
|
32
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
// import { playwrightLauncher } from '@web/test-runner-playwright';
|
|
2
|
+
|
|
3
|
+
const filteredLogs = ['Running in dev mode', 'lit-html is in dev mode'];
|
|
4
|
+
|
|
5
|
+
export default /** @type {import("@web/test-runner").TestRunnerConfig} */ ({
|
|
6
|
+
/** Test files to run */
|
|
7
|
+
files: 'dist/test/**/*.test.js',
|
|
8
|
+
|
|
9
|
+
/** Resolve bare module imports */
|
|
10
|
+
nodeResolve: {
|
|
11
|
+
exportConditions: ['browser', 'development'],
|
|
12
|
+
},
|
|
13
|
+
|
|
14
|
+
/** Filter out lit dev mode logs */
|
|
15
|
+
filterBrowserLogs(log) {
|
|
16
|
+
for (const arg of log.args) {
|
|
17
|
+
if (typeof arg === 'string' && filteredLogs.some(l => arg.includes(l))) {
|
|
18
|
+
return false;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return true;
|
|
22
|
+
},
|
|
23
|
+
|
|
24
|
+
/** Compile JS for older browsers. Requires @web/dev-server-esbuild plugin */
|
|
25
|
+
// esbuildTarget: 'auto',
|
|
26
|
+
|
|
27
|
+
/** Amount of browsers to run concurrently */
|
|
28
|
+
// concurrentBrowsers: 2,
|
|
29
|
+
|
|
30
|
+
/** Amount of test files per browser to test concurrently */
|
|
31
|
+
// concurrency: 1,
|
|
32
|
+
|
|
33
|
+
/** Browsers to run tests on */
|
|
34
|
+
// browsers: [
|
|
35
|
+
// playwrightLauncher({ product: 'chromium' }),
|
|
36
|
+
// playwrightLauncher({ product: 'firefox' }),
|
|
37
|
+
// playwrightLauncher({ product: 'webkit' }),
|
|
38
|
+
// ],
|
|
39
|
+
|
|
40
|
+
// See documentation for all available options
|
|
41
|
+
});
|