@bluefields/fetcher 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/dist/asset-block-measure.d.ts +14 -0
- package/dist/asset-block-measure.js +82 -0
- package/dist/asset-block-measure.js.map +1 -0
- package/dist/browser-pool.d.ts +65 -0
- package/dist/browser-pool.js +161 -0
- package/dist/browser-pool.js.map +1 -0
- package/dist/cost.d.ts +51 -0
- package/dist/cost.js +54 -0
- package/dist/cost.js.map +1 -0
- package/dist/fetcher-brightdata.d.ts +52 -0
- package/dist/fetcher-brightdata.js +135 -0
- package/dist/fetcher-brightdata.js.map +1 -0
- package/dist/fetcher-escalating.d.ts +100 -0
- package/dist/fetcher-escalating.js +489 -0
- package/dist/fetcher-escalating.js.map +1 -0
- package/dist/fetcher-http.d.ts +81 -0
- package/dist/fetcher-http.js +395 -0
- package/dist/fetcher-http.js.map +1 -0
- package/dist/fetcher-patchright.d.ts +212 -0
- package/dist/fetcher-patchright.js +505 -0
- package/dist/fetcher-patchright.js.map +1 -0
- package/dist/fetcher-robots-gate.d.ts +60 -0
- package/dist/fetcher-robots-gate.js +87 -0
- package/dist/fetcher-robots-gate.js.map +1 -0
- package/dist/index.d.ts +93 -0
- package/dist/index.js +219 -0
- package/dist/index.js.map +1 -0
- package/dist/internal-options.d.ts +59 -0
- package/dist/internal-options.js +16 -0
- package/dist/internal-options.js.map +1 -0
- package/dist/perf-harness.d.ts +105 -0
- package/dist/perf-harness.js +176 -0
- package/dist/perf-harness.js.map +1 -0
- package/dist/storage-memory.d.ts +18 -0
- package/dist/storage-memory.js +45 -0
- package/dist/storage-memory.js.map +1 -0
- package/dist/storage-r2.d.ts +44 -0
- package/dist/storage-r2.js +209 -0
- package/dist/storage-r2.js.map +1 -0
- package/dist/types.d.ts +231 -0
- package/dist/types.js +22 -0
- package/dist/types.js.map +1 -0
- package/package.json +54 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
/**
|
|
3
|
+
* Deterministic fixture perf harness for the escalating fetcher (perf round).
|
|
4
|
+
*
|
|
5
|
+
* The mission requires a before/after p50/p95/cost/coverage number PER LEVER on a
|
|
6
|
+
* FIXTURE corpus (never live sites). This library scripts each tier's latency +
|
|
7
|
+
* outcome per fixture URL, drives the real `createEscalatingFetcherAdapter` against
|
|
8
|
+
* those mocks, and reports the distribution. It is deliberately dependency-free and
|
|
9
|
+
* vitest-free: the CALLER injects the clock (a fake-timer test drives `now` +
|
|
10
|
+
* `advanceAndFlush`), so the whole run is deterministic and fast — no real waiting,
|
|
11
|
+
* no network. The runnable entrypoint is `fetcher-escalating.perf.test.ts`
|
|
12
|
+
* (FETCH_PERF=1-gated, inert in the normal suite); `perf-harness.test.ts` guards the
|
|
13
|
+
* metrics math in CI.
|
|
14
|
+
*
|
|
15
|
+
* Fetcher-layer only: it exercises tier selection + timing, nothing else.
|
|
16
|
+
*/
|
|
17
|
+
import { classifyFetchResult, createEscalatingFetcherAdapter } from './fetcher-escalating.js';
|
|
18
|
+
/** Billed credits per tier (mirrors billing: tier-0 HTTP=1, tier-1 browser=3, tier-2 unlocker=8). */
|
|
19
|
+
export const TIER_CREDITS = { 0: 1, 1: 3, 2: 8 };
|
|
20
|
+
/** fetcherId each tier's mock reports — matches the real adapters so we can map result→tier. */
|
|
21
|
+
export const TIER_FETCHER_ID = {
|
|
22
|
+
0: 'http-v1',
|
|
23
|
+
1: 'patchright-v1',
|
|
24
|
+
2: 'brightdata-v1',
|
|
25
|
+
};
|
|
26
|
+
function isAbsent(s) {
|
|
27
|
+
return 'kind' in s && s.kind === 'absent';
|
|
28
|
+
}
|
|
29
|
+
function isHang(s) {
|
|
30
|
+
return 'kind' in s && s.kind === 'hang';
|
|
31
|
+
}
|
|
32
|
+
function makeResult(url, tier, status, html) {
|
|
33
|
+
return {
|
|
34
|
+
html,
|
|
35
|
+
responseStatus: status,
|
|
36
|
+
responseHeaders: {},
|
|
37
|
+
fetchedAt: new Date(0),
|
|
38
|
+
fetchDurationMs: 1,
|
|
39
|
+
fetcherId: TIER_FETCHER_ID[tier] ?? `tier-${tier}-v1`,
|
|
40
|
+
proxyEgressIp: '0.0.0.0',
|
|
41
|
+
userAgent: 'perf-harness',
|
|
42
|
+
rawHtmlSizeBytes: Buffer.byteLength(html, 'utf8'),
|
|
43
|
+
screenshot: null,
|
|
44
|
+
finalUrl: url,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A mock tier adapter honoring the abort seam the escalating fetcher uses: a hung
|
|
49
|
+
* tier stays pending until aborted (then rejects, like a real abandoned tier); a
|
|
50
|
+
* scripted tier resolves after `latencyMs` via setTimeout (fake-clock driven). Returns
|
|
51
|
+
* null for an absent tier so the ladder omits it.
|
|
52
|
+
*/
|
|
53
|
+
export function mockTierAdapter(tier, script) {
|
|
54
|
+
if (isAbsent(script))
|
|
55
|
+
return null;
|
|
56
|
+
return {
|
|
57
|
+
fetch(url, options) {
|
|
58
|
+
const signal = options?.__abortSignal;
|
|
59
|
+
if (isHang(script)) {
|
|
60
|
+
return new Promise((_resolve, reject) => {
|
|
61
|
+
if (signal) {
|
|
62
|
+
if (signal.aborted)
|
|
63
|
+
reject(new Error('aborted'));
|
|
64
|
+
else
|
|
65
|
+
signal.addEventListener('abort', () => reject(new Error('aborted')), { once: true });
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
const s = script;
|
|
70
|
+
return new Promise((resolve, reject) => {
|
|
71
|
+
const t = setTimeout(() => resolve(makeResult(url, tier, s.status, s.html)), s.latencyMs);
|
|
72
|
+
if (signal) {
|
|
73
|
+
const onAbort = () => {
|
|
74
|
+
clearTimeout(t);
|
|
75
|
+
reject(new Error('aborted'));
|
|
76
|
+
};
|
|
77
|
+
if (signal.aborted)
|
|
78
|
+
onAbort();
|
|
79
|
+
else
|
|
80
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
/** Nearest-rank percentile (no interpolation) — same convention as the SLA harness. */
|
|
87
|
+
export function percentile(sorted, q) {
|
|
88
|
+
if (sorted.length === 0)
|
|
89
|
+
return Number.NaN;
|
|
90
|
+
const idx = Math.min(sorted.length - 1, Math.floor(q * sorted.length));
|
|
91
|
+
return sorted[idx];
|
|
92
|
+
}
|
|
93
|
+
function tierFromFetcherId(id) {
|
|
94
|
+
for (const [tier, fid] of Object.entries(TIER_FETCHER_ID)) {
|
|
95
|
+
if (fid === id)
|
|
96
|
+
return Number(tier);
|
|
97
|
+
}
|
|
98
|
+
return -1;
|
|
99
|
+
}
|
|
100
|
+
function summarize(requests) {
|
|
101
|
+
const lat = requests.map((r) => r.elapsedMs).sort((a, b) => a - b);
|
|
102
|
+
const covered = requests.filter((r) => r.covered);
|
|
103
|
+
const totalCredits = requests.reduce((a, r) => a + r.credits, 0);
|
|
104
|
+
const perTier = {};
|
|
105
|
+
for (const r of requests)
|
|
106
|
+
perTier[r.tier] = (perTier[r.tier] ?? 0) + 1;
|
|
107
|
+
return {
|
|
108
|
+
n: requests.length,
|
|
109
|
+
p50: percentile(lat, 0.5),
|
|
110
|
+
p95: percentile(lat, 0.95),
|
|
111
|
+
max: lat.length ? lat[lat.length - 1] : Number.NaN,
|
|
112
|
+
successRate: requests.length ? covered.length / requests.length : 0,
|
|
113
|
+
creditsPerSuccess: covered.length ? totalCredits / covered.length : Number.NaN,
|
|
114
|
+
meanCredits: requests.length ? totalCredits / requests.length : Number.NaN,
|
|
115
|
+
perTier,
|
|
116
|
+
requests,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Run the fixture corpus through the real escalating fetcher under the caller's fake
|
|
121
|
+
* clock, returning the latency/cost/coverage distribution. For each fixture it builds
|
|
122
|
+
* `createEscalatingFetcherAdapter` with the scripted mock tiers + the injected
|
|
123
|
+
* `nowMsFn`, starts the fetch, and drives `advanceAndFlush(stepMs)` until it settles —
|
|
124
|
+
* measuring end-to-end elapsed on the fake clock (quantized to stepMs).
|
|
125
|
+
*/
|
|
126
|
+
export async function runCorpus(fixtures, o) {
|
|
127
|
+
const requests = [];
|
|
128
|
+
const iterations = o.iterations ?? 1;
|
|
129
|
+
const step = o.stepMs ?? 25;
|
|
130
|
+
const cap = o.capMs ?? 120_000;
|
|
131
|
+
for (let it = 0; it < iterations; it++) {
|
|
132
|
+
for (const fx of fixtures) {
|
|
133
|
+
const esc = createEscalatingFetcherAdapter({
|
|
134
|
+
tier0: mockTierAdapter(0, fx.tier0) ?? undefined,
|
|
135
|
+
tier1: fx.tier1 ? mockTierAdapter(1, fx.tier1) : null,
|
|
136
|
+
tier2: fx.tier2 ? mockTierAdapter(2, fx.tier2) : null,
|
|
137
|
+
nowMsFn: o.now,
|
|
138
|
+
...o.escalatingOpts,
|
|
139
|
+
});
|
|
140
|
+
const start = o.now();
|
|
141
|
+
let settled = false;
|
|
142
|
+
const fetchP = esc.fetch(fx.url).then((r) => r, () => null);
|
|
143
|
+
void fetchP.finally(() => {
|
|
144
|
+
settled = true;
|
|
145
|
+
});
|
|
146
|
+
let advanced = 0;
|
|
147
|
+
while (!settled && advanced < cap) {
|
|
148
|
+
await o.advanceAndFlush(step);
|
|
149
|
+
advanced += step;
|
|
150
|
+
}
|
|
151
|
+
const result = await fetchP;
|
|
152
|
+
const elapsedMs = o.now() - start;
|
|
153
|
+
const tier = result ? tierFromFetcherId(result.fetcherId) : -1;
|
|
154
|
+
const outcome = result ? classifyFetchResult(result) : 'error';
|
|
155
|
+
requests.push({
|
|
156
|
+
url: fx.url,
|
|
157
|
+
label: fx.label,
|
|
158
|
+
elapsedMs,
|
|
159
|
+
tier,
|
|
160
|
+
credits: TIER_CREDITS[tier] ?? 0,
|
|
161
|
+
outcome,
|
|
162
|
+
covered: outcome === 'success',
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return summarize(requests);
|
|
167
|
+
}
|
|
168
|
+
/** Pretty one-line summary for a config's report (for the perf entrypoint's console output). */
|
|
169
|
+
export function formatReport(name, r) {
|
|
170
|
+
const tiers = Object.entries(r.perTier)
|
|
171
|
+
.sort(([a], [b]) => Number(a) - Number(b))
|
|
172
|
+
.map(([t, c]) => `t${t}:${c}`)
|
|
173
|
+
.join(' ');
|
|
174
|
+
return `${name.padEnd(22)} p50=${String(Math.round(r.p50)).padStart(6)}ms p95=${String(Math.round(r.p95)).padStart(6)}ms max=${String(Math.round(r.max)).padStart(6)}ms cov=${(r.successRate * 100).toFixed(1)}% cr/succ=${r.creditsPerSuccess.toFixed(2)} [${tiers}]`;
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=perf-harness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"perf-harness.js","sourceRoot":"","sources":["../src/perf-harness.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,mBAAmB,EAAE,8BAA8B,EAAE,MAAM,yBAAyB,CAAC;AAK9F,qGAAqG;AACrG,MAAM,CAAC,MAAM,YAAY,GAAqC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;AACnF,gGAAgG;AAChG,MAAM,CAAC,MAAM,eAAe,GAAqC;IAC/D,CAAC,EAAE,SAAS;IACZ,CAAC,EAAE,eAAe;IAClB,CAAC,EAAE,eAAe;CACnB,CAAC;AAoBF,SAAS,QAAQ,CAAC,CAAa;IAC7B,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC5C,CAAC;AACD,SAAS,MAAM,CAAC,CAAa;IAC3B,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,IAAY,EAAE,MAAc,EAAE,IAAY;IACzE,OAAO;QACL,IAAI;QACJ,cAAc,EAAE,MAAM;QACtB,eAAe,EAAE,EAAE;QACnB,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACtB,eAAe,EAAE,CAAC;QAClB,SAAS,EAAE,eAAe,CAAC,IAAI,CAAC,IAAI,QAAQ,IAAI,KAAK;QACrD,aAAa,EAAE,SAAS;QACxB,SAAS,EAAE,cAAc;QACzB,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC;QACjD,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,GAAG;KACd,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,MAAkB;IAC9D,IAAI,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IAClC,OAAO;QACL,KAAK,CAAC,GAAW,EAAE,OAAO;YACxB,MAAM,MAAM,GAAI,OAA4C,EAAE,aAAa,CAAC;YAC5E,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnB,OAAO,IAAI,OAAO,CAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;oBACnD,IAAI,MAAM,EAAE,CAAC;wBACX,IAAI,MAAM,CAAC,OAAO;4BAAE,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;;4BAE/C,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;oBACzF,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,GAAG,MAA6D,CAAC;YACxE,OAAO,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAClD,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;gBAC1F,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,OAAO,GAAG,GAAS,EAAE;wBACzB,YAAY,CAAC,CAAC,CAAC,CAAC;wBAChB,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;oBAC/B,CAAC,CAAC;oBACF,IAAI,MAAM,CAAC,OAAO;wBAAE,OAAO,EAAE,CAAC;;wBACzB,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjE,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;KACF,CAAC;AACJ,CAAC;AA8BD,uFAAuF;AACvF,MAAM,UAAU,UAAU,CAAC,MAAgB,EAAE,CAAS;IACpD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACvE,OAAO,MAAM,CAAC,GAAG,CAAE,CAAC;AACtB,CAAC;AAED,SAAS,iBAAiB,CAAC,EAAU;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC1D,IAAI,GAAG,KAAK,EAAE;YAAE,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,QAAsB;IACvC,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACjE,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACvE,OAAO;QACL,CAAC,EAAE,QAAQ,CAAC,MAAM;QAClB,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;QACzB,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC;QAC1B,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG;QACnD,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnE,iBAAiB,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG;QAC9E,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG;QAC1E,OAAO;QACP,QAAQ;KACT,CAAC;AACJ,CAAC;AAiBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAuB,EAAE,CAAa;IACpE,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,MAAM,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC;IAC5B,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC;IAC/B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;QACvC,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,8BAA8B,CAAC;gBACzC,KAAK,EAAE,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,SAAS;gBAChD,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrD,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI;gBACrD,OAAO,EAAE,CAAC,CAAC,GAAG;gBACd,GAAG,CAAC,CAAC,cAAc;aACpB,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,MAAM,MAAM,GAAgC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAChE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EACR,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;YACF,KAAK,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE;gBACvB,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC,CAAC,CAAC;YACH,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,OAAO,CAAC,OAAO,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;gBAClC,MAAM,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;gBAC9B,QAAQ,IAAI,IAAI,CAAC;YACnB,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;YAC5B,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC;YAClC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAC/D,QAAQ,CAAC,IAAI,CAAC;gBACZ,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,KAAK,EAAE,EAAE,CAAC,KAAK;gBACf,SAAS;gBACT,IAAI;gBACJ,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChC,OAAO;gBACP,OAAO,EAAE,OAAO,KAAK,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,CAAe;IACxD,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;SACpC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;SACzC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;SAC7B,IAAI,CAAC,GAAG,CAAC,CAAC;IACb,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,GAAG,CAAC;AACzQ,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-memory snapshot storage (chunk 8).
|
|
3
|
+
*
|
|
4
|
+
* Default storage adapter when R2 isn't configured. Map-backed; content-
|
|
5
|
+
* addressed via sha256. Useful for tests and for running the fetcher
|
|
6
|
+
* end-to-end locally without an R2 bucket.
|
|
7
|
+
*
|
|
8
|
+
* NOT persistent across process restarts. Production should swap in the
|
|
9
|
+
* R2 adapter (`storage-r2.ts`) once R2_* env vars are set.
|
|
10
|
+
*/
|
|
11
|
+
import { type StorageAdapter } from './types.js';
|
|
12
|
+
export interface InMemoryStorageOptions {
|
|
13
|
+
/** Optional pre-seeded entries (keyed by content hash). */
|
|
14
|
+
seed?: Record<string, string>;
|
|
15
|
+
}
|
|
16
|
+
export declare function createInMemoryStorageAdapter(opts?: InMemoryStorageOptions): StorageAdapter & {
|
|
17
|
+
dump(): Record<string, string>;
|
|
18
|
+
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
/**
|
|
3
|
+
* In-memory snapshot storage (chunk 8).
|
|
4
|
+
*
|
|
5
|
+
* Default storage adapter when R2 isn't configured. Map-backed; content-
|
|
6
|
+
* addressed via sha256. Useful for tests and for running the fetcher
|
|
7
|
+
* end-to-end locally without an R2 bucket.
|
|
8
|
+
*
|
|
9
|
+
* NOT persistent across process restarts. Production should swap in the
|
|
10
|
+
* R2 adapter (`storage-r2.ts`) once R2_* env vars are set.
|
|
11
|
+
*/
|
|
12
|
+
import { createHash } from 'node:crypto';
|
|
13
|
+
import { buildR2Key } from './types.js';
|
|
14
|
+
export function createInMemoryStorageAdapter(opts = {}) {
|
|
15
|
+
const store = new Map();
|
|
16
|
+
if (opts.seed) {
|
|
17
|
+
for (const [k, v] of Object.entries(opts.seed))
|
|
18
|
+
store.set(k, v);
|
|
19
|
+
}
|
|
20
|
+
return {
|
|
21
|
+
async storeSnapshot(content) {
|
|
22
|
+
const contentHash = createHash('sha256').update(content).digest('hex');
|
|
23
|
+
// Idempotent: dedup by hash. Same content → same key.
|
|
24
|
+
if (!store.has(contentHash)) {
|
|
25
|
+
store.set(contentHash, content);
|
|
26
|
+
}
|
|
27
|
+
return { contentHash, r2Key: buildR2Key(contentHash) };
|
|
28
|
+
},
|
|
29
|
+
async getSnapshot(contentHash) {
|
|
30
|
+
const v = store.get(contentHash);
|
|
31
|
+
if (v === undefined) {
|
|
32
|
+
throw new Error(`snapshot not found: ${contentHash}`);
|
|
33
|
+
}
|
|
34
|
+
return v;
|
|
35
|
+
},
|
|
36
|
+
async hasSnapshot(contentHash) {
|
|
37
|
+
return store.has(contentHash);
|
|
38
|
+
},
|
|
39
|
+
/** Test helper: dump current contents. */
|
|
40
|
+
dump() {
|
|
41
|
+
return Object.fromEntries(store);
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=storage-memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage-memory.js","sourceRoot":"","sources":["../src/storage-memory.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAA4C,UAAU,EAAE,MAAM,YAAY,CAAC;AAOlF,MAAM,UAAU,4BAA4B,CAC1C,OAA+B,EAAE;IAEjC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAClE,CAAC;IAED,OAAO;QACL,KAAK,CAAC,aAAa,CAAC,OAAe;YACjC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,sDAAsD;YACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAC5B,KAAK,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YAClC,CAAC;YACD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QACzD,CAAC;QAED,KAAK,CAAC,WAAW,CAAC,WAAmB;YACnC,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACjC,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAC;YACxD,CAAC;YACD,OAAO,CAAC,CAAC;QACX,CAAC;QAED,KAAK,CAAC,WAAW,CAAC,WAAmB;YACnC,OAAO,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAChC,CAAC;QAED,0CAA0C;QAC1C,IAAI;YACF,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACnC,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* R2 storage adapter (chunk 8) — env-gated, S3-compatible HTTP client.
|
|
3
|
+
*
|
|
4
|
+
* Talks to R2 directly via its S3-compatible HTTP API. Avoids dragging
|
|
5
|
+
* in @aws-sdk/client-s3 (~5MB of deps) for the small surface we need
|
|
6
|
+
* (PUT, GET, HEAD). SigV4 signing is implemented inline below.
|
|
7
|
+
*
|
|
8
|
+
* Activated when R2_ACCESS_KEY + R2_SECRET_KEY + R2_BUCKET are set, plus
|
|
9
|
+
* either R2_ACCOUNT_ID (Cloudflare R2) or R2_ENDPOINT (MinIO / any
|
|
10
|
+
* S3-compatible store). Otherwise `tryCreateR2StorageAdapter` returns null
|
|
11
|
+
* and the factory falls back to the in-memory adapter.
|
|
12
|
+
*/
|
|
13
|
+
import { type StorageAdapter } from './types.js';
|
|
14
|
+
/** Required for Cloudflare R2 (default endpoint derived from account id). */
|
|
15
|
+
export declare const R2_REQUIRED_ENV: readonly ["R2_ACCOUNT_ID", "R2_ACCESS_KEY", "R2_SECRET_KEY", "R2_BUCKET"];
|
|
16
|
+
/** Required when using a custom S3 endpoint (e.g. MinIO in Docker Compose). */
|
|
17
|
+
export declare const R2_ENDPOINT_REQUIRED_ENV: readonly ["R2_ACCESS_KEY", "R2_SECRET_KEY", "R2_BUCKET"];
|
|
18
|
+
export interface R2Config {
|
|
19
|
+
accountId: string;
|
|
20
|
+
accessKey: string;
|
|
21
|
+
secretKey: string;
|
|
22
|
+
bucket: string;
|
|
23
|
+
/**
|
|
24
|
+
* Custom S3 API base URL (e.g. `http://minio:9000`). When omitted, uses
|
|
25
|
+
* `https://{accountId}.r2.cloudflarestorage.com`.
|
|
26
|
+
*/
|
|
27
|
+
endpoint?: string;
|
|
28
|
+
/** Path-style object URLs — required for MinIO in Docker. */
|
|
29
|
+
forcePathStyle?: boolean;
|
|
30
|
+
/** SigV4 region. Cloudflare R2 uses `auto`; MinIO typically `us-east-1`. */
|
|
31
|
+
region?: string;
|
|
32
|
+
/** Test seam — defaults to global fetch. */
|
|
33
|
+
fetchImpl?: typeof fetch;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Returns the R2 adapter when env vars are set, otherwise null. Callers
|
|
37
|
+
* should fall back to the in-memory adapter on null.
|
|
38
|
+
*/
|
|
39
|
+
export declare function tryCreateR2StorageAdapter(env?: NodeJS.ProcessEnv, fetchImpl?: typeof fetch): StorageAdapter | null;
|
|
40
|
+
/**
|
|
41
|
+
* Factory for direct (non-env-gated) callers — useful in tests that
|
|
42
|
+
* stub `fetchImpl` to verify the request shape without hitting R2.
|
|
43
|
+
*/
|
|
44
|
+
export declare function createR2StorageAdapter(config: R2Config): StorageAdapter;
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
2
|
+
/**
|
|
3
|
+
* R2 storage adapter (chunk 8) — env-gated, S3-compatible HTTP client.
|
|
4
|
+
*
|
|
5
|
+
* Talks to R2 directly via its S3-compatible HTTP API. Avoids dragging
|
|
6
|
+
* in @aws-sdk/client-s3 (~5MB of deps) for the small surface we need
|
|
7
|
+
* (PUT, GET, HEAD). SigV4 signing is implemented inline below.
|
|
8
|
+
*
|
|
9
|
+
* Activated when R2_ACCESS_KEY + R2_SECRET_KEY + R2_BUCKET are set, plus
|
|
10
|
+
* either R2_ACCOUNT_ID (Cloudflare R2) or R2_ENDPOINT (MinIO / any
|
|
11
|
+
* S3-compatible store). Otherwise `tryCreateR2StorageAdapter` returns null
|
|
12
|
+
* and the factory falls back to the in-memory adapter.
|
|
13
|
+
*/
|
|
14
|
+
import { createHash, createHmac } from 'node:crypto';
|
|
15
|
+
import { buildR2Key } from './types.js';
|
|
16
|
+
/** Required for Cloudflare R2 (default endpoint derived from account id). */
|
|
17
|
+
export const R2_REQUIRED_ENV = [
|
|
18
|
+
'R2_ACCOUNT_ID',
|
|
19
|
+
'R2_ACCESS_KEY',
|
|
20
|
+
'R2_SECRET_KEY',
|
|
21
|
+
'R2_BUCKET',
|
|
22
|
+
];
|
|
23
|
+
/** Required when using a custom S3 endpoint (e.g. MinIO in Docker Compose). */
|
|
24
|
+
export const R2_ENDPOINT_REQUIRED_ENV = ['R2_ACCESS_KEY', 'R2_SECRET_KEY', 'R2_BUCKET'];
|
|
25
|
+
/**
|
|
26
|
+
* Returns the R2 adapter when env vars are set, otherwise null. Callers
|
|
27
|
+
* should fall back to the in-memory adapter on null.
|
|
28
|
+
*/
|
|
29
|
+
export function tryCreateR2StorageAdapter(env = process.env, fetchImpl) {
|
|
30
|
+
const endpoint = env.R2_ENDPOINT?.trim();
|
|
31
|
+
if (endpoint) {
|
|
32
|
+
for (const key of R2_ENDPOINT_REQUIRED_ENV) {
|
|
33
|
+
if (!env[key] || env[key]?.length === 0)
|
|
34
|
+
return null;
|
|
35
|
+
}
|
|
36
|
+
return createR2StorageAdapter({
|
|
37
|
+
accountId: env.R2_ACCOUNT_ID?.trim() || 'minio',
|
|
38
|
+
accessKey: env.R2_ACCESS_KEY,
|
|
39
|
+
secretKey: env.R2_SECRET_KEY,
|
|
40
|
+
bucket: env.R2_BUCKET,
|
|
41
|
+
endpoint,
|
|
42
|
+
forcePathStyle: env.R2_FORCE_PATH_STYLE === 'true',
|
|
43
|
+
region: env.R2_REGION?.trim() || 'us-east-1',
|
|
44
|
+
fetchImpl,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
for (const key of R2_REQUIRED_ENV) {
|
|
48
|
+
if (!env[key] || env[key]?.length === 0)
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
return createR2StorageAdapter({
|
|
52
|
+
accountId: env.R2_ACCOUNT_ID,
|
|
53
|
+
accessKey: env.R2_ACCESS_KEY,
|
|
54
|
+
secretKey: env.R2_SECRET_KEY,
|
|
55
|
+
bucket: env.R2_BUCKET,
|
|
56
|
+
fetchImpl,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
function resolveStorageEndpoint(config) {
|
|
60
|
+
if (config.endpoint) {
|
|
61
|
+
return {
|
|
62
|
+
baseUrl: config.endpoint.replace(/\/$/, ''),
|
|
63
|
+
region: config.region ?? 'us-east-1',
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
baseUrl: `https://${config.accountId}.r2.cloudflarestorage.com`,
|
|
68
|
+
region: config.region ?? 'auto',
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
function objectUrl(baseUrl, bucket, key, forcePathStyle) {
|
|
72
|
+
if (forcePathStyle) {
|
|
73
|
+
return `${baseUrl}/${bucket}/${key}`;
|
|
74
|
+
}
|
|
75
|
+
const host = new URL(baseUrl).host;
|
|
76
|
+
const protocol = new URL(baseUrl).protocol;
|
|
77
|
+
return `${protocol}//${bucket}.${host}/${key}`;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Factory for direct (non-env-gated) callers — useful in tests that
|
|
81
|
+
* stub `fetchImpl` to verify the request shape without hitting R2.
|
|
82
|
+
*/
|
|
83
|
+
export function createR2StorageAdapter(config) {
|
|
84
|
+
const fetchImpl = config.fetchImpl ?? fetch;
|
|
85
|
+
const { baseUrl, region } = resolveStorageEndpoint(config);
|
|
86
|
+
const forcePathStyle = config.forcePathStyle ?? false;
|
|
87
|
+
const urlForKey = (key) => objectUrl(baseUrl, config.bucket, key, forcePathStyle);
|
|
88
|
+
return {
|
|
89
|
+
async storeSnapshot(content) {
|
|
90
|
+
const contentHash = createHash('sha256').update(content).digest('hex');
|
|
91
|
+
const r2Key = buildR2Key(contentHash);
|
|
92
|
+
// Idempotent: HEAD first; skip the PUT on hit.
|
|
93
|
+
const already = await this.hasSnapshot(contentHash);
|
|
94
|
+
if (!already) {
|
|
95
|
+
const url = urlForKey(r2Key);
|
|
96
|
+
const res = await signedRequest({
|
|
97
|
+
method: 'PUT',
|
|
98
|
+
url,
|
|
99
|
+
region,
|
|
100
|
+
service: 's3',
|
|
101
|
+
accessKey: config.accessKey,
|
|
102
|
+
secretKey: config.secretKey,
|
|
103
|
+
body: content,
|
|
104
|
+
headers: { 'content-type': 'text/html; charset=utf-8' },
|
|
105
|
+
fetchImpl,
|
|
106
|
+
});
|
|
107
|
+
if (!res.ok) {
|
|
108
|
+
throw new Error(`R2 PUT failed: ${res.status} ${await res.text()}`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return { contentHash, r2Key };
|
|
112
|
+
},
|
|
113
|
+
async getSnapshot(contentHash) {
|
|
114
|
+
const url = urlForKey(buildR2Key(contentHash));
|
|
115
|
+
const res = await signedRequest({
|
|
116
|
+
method: 'GET',
|
|
117
|
+
url,
|
|
118
|
+
region,
|
|
119
|
+
service: 's3',
|
|
120
|
+
accessKey: config.accessKey,
|
|
121
|
+
secretKey: config.secretKey,
|
|
122
|
+
fetchImpl,
|
|
123
|
+
});
|
|
124
|
+
if (res.status === 404)
|
|
125
|
+
throw new Error(`snapshot not found: ${contentHash}`);
|
|
126
|
+
if (!res.ok)
|
|
127
|
+
throw new Error(`R2 GET failed: ${res.status}`);
|
|
128
|
+
return await res.text();
|
|
129
|
+
},
|
|
130
|
+
async hasSnapshot(contentHash) {
|
|
131
|
+
const url = urlForKey(buildR2Key(contentHash));
|
|
132
|
+
const res = await signedRequest({
|
|
133
|
+
method: 'HEAD',
|
|
134
|
+
url,
|
|
135
|
+
region,
|
|
136
|
+
service: 's3',
|
|
137
|
+
accessKey: config.accessKey,
|
|
138
|
+
secretKey: config.secretKey,
|
|
139
|
+
fetchImpl,
|
|
140
|
+
});
|
|
141
|
+
return res.ok;
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
// ── AWS SigV4 ────────────────────────────────────────────────────────────
|
|
146
|
+
// Minimal SigV4 implementation for S3-compatible PUT/GET/HEAD. Verified
|
|
147
|
+
// against the AWS-published examples. Doesn't handle multipart or chunked
|
|
148
|
+
// encoding — fine for our small (<10MB) HTML snapshots.
|
|
149
|
+
/** Bound an R2 request so a hung object store can't stall a snapshot write. */
|
|
150
|
+
const R2_REQUEST_TIMEOUT_MS = 30_000;
|
|
151
|
+
async function signedRequest(args) {
|
|
152
|
+
const url = new URL(args.url);
|
|
153
|
+
const body = args.body ?? '';
|
|
154
|
+
const bodyHash = createHash('sha256').update(body).digest('hex');
|
|
155
|
+
const now = new Date();
|
|
156
|
+
const amzDate = formatAmzDate(now);
|
|
157
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
158
|
+
const headers = {
|
|
159
|
+
...(args.headers ?? {}),
|
|
160
|
+
host: url.host,
|
|
161
|
+
'x-amz-content-sha256': bodyHash,
|
|
162
|
+
'x-amz-date': amzDate,
|
|
163
|
+
};
|
|
164
|
+
// Canonical request
|
|
165
|
+
const sortedHeaderNames = Object.keys(headers)
|
|
166
|
+
.map((h) => h.toLowerCase())
|
|
167
|
+
.sort();
|
|
168
|
+
const canonicalHeaders = sortedHeaderNames.map((h) => `${h}:${headers[h]?.trim()}\n`).join('');
|
|
169
|
+
const signedHeaders = sortedHeaderNames.join(';');
|
|
170
|
+
const canonicalRequest = [
|
|
171
|
+
args.method,
|
|
172
|
+
url.pathname,
|
|
173
|
+
url.searchParams.toString(), // canonical query
|
|
174
|
+
canonicalHeaders,
|
|
175
|
+
signedHeaders,
|
|
176
|
+
bodyHash,
|
|
177
|
+
].join('\n');
|
|
178
|
+
// String to sign
|
|
179
|
+
const credentialScope = `${dateStamp}/${args.region}/${args.service}/aws4_request`;
|
|
180
|
+
const canonicalRequestHash = createHash('sha256').update(canonicalRequest).digest('hex');
|
|
181
|
+
const stringToSign = ['AWS4-HMAC-SHA256', amzDate, credentialScope, canonicalRequestHash].join('\n');
|
|
182
|
+
// Signing key
|
|
183
|
+
const kDate = createHmac('sha256', `AWS4${args.secretKey}`).update(dateStamp).digest();
|
|
184
|
+
const kRegion = createHmac('sha256', kDate).update(args.region).digest();
|
|
185
|
+
const kService = createHmac('sha256', kRegion).update(args.service).digest();
|
|
186
|
+
const kSigning = createHmac('sha256', kService).update('aws4_request').digest();
|
|
187
|
+
const signature = createHmac('sha256', kSigning).update(stringToSign).digest('hex');
|
|
188
|
+
const authorization = `AWS4-HMAC-SHA256 Credential=${args.accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
|
189
|
+
const controller = new AbortController();
|
|
190
|
+
const timeout = setTimeout(() => controller.abort(), args.timeoutMs ?? R2_REQUEST_TIMEOUT_MS);
|
|
191
|
+
try {
|
|
192
|
+
return await args.fetchImpl(args.url, {
|
|
193
|
+
method: args.method,
|
|
194
|
+
headers: { ...headers, authorization },
|
|
195
|
+
body: args.method === 'PUT' ? body : undefined,
|
|
196
|
+
signal: controller.signal,
|
|
197
|
+
});
|
|
198
|
+
}
|
|
199
|
+
finally {
|
|
200
|
+
clearTimeout(timeout);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
function formatAmzDate(d) {
|
|
204
|
+
// ISO 8601 basic format: YYYYMMDDTHHMMSSZ
|
|
205
|
+
const pad = (n) => n.toString().padStart(2, '0');
|
|
206
|
+
return (`${d.getUTCFullYear()}${pad(d.getUTCMonth() + 1)}${pad(d.getUTCDate())}T` +
|
|
207
|
+
`${pad(d.getUTCHours())}${pad(d.getUTCMinutes())}${pad(d.getUTCSeconds())}Z`);
|
|
208
|
+
}
|
|
209
|
+
//# sourceMappingURL=storage-r2.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"storage-r2.js","sourceRoot":"","sources":["../src/storage-r2.ts"],"names":[],"mappings":"AAAA,6CAA6C;AAC7C;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,EAA4C,UAAU,EAAE,MAAM,YAAY,CAAC;AAElF,6EAA6E;AAC7E,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,eAAe;IACf,eAAe;IACf,eAAe;IACf,WAAW;CACH,CAAC;AAEX,+EAA+E;AAC/E,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,eAAe,EAAE,eAAe,EAAE,WAAW,CAAU,CAAC;AAoBjG;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CACvC,MAAyB,OAAO,CAAC,GAAG,EACpC,SAAwB;IAExB,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;IACzC,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,GAAG,IAAI,wBAAwB,EAAE,CAAC;YAC3C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;QACvD,CAAC;QACD,OAAO,sBAAsB,CAAC;YAC5B,SAAS,EAAE,GAAG,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,OAAO;YAC/C,SAAS,EAAE,GAAG,CAAC,aAAc;YAC7B,SAAS,EAAE,GAAG,CAAC,aAAc;YAC7B,MAAM,EAAE,GAAG,CAAC,SAAU;YACtB,QAAQ;YACR,cAAc,EAAE,GAAG,CAAC,mBAAmB,KAAK,MAAM;YAClD,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,WAAW;YAC5C,SAAS;SACV,CAAC,CAAC;IACL,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;IACvD,CAAC;IACD,OAAO,sBAAsB,CAAC;QAC5B,SAAS,EAAE,GAAG,CAAC,aAAc;QAC7B,SAAS,EAAE,GAAG,CAAC,aAAc;QAC7B,SAAS,EAAE,GAAG,CAAC,aAAc;QAC7B,MAAM,EAAE,GAAG,CAAC,SAAU;QACtB,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAAC,MAAgB;IAC9C,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,OAAO;YACL,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;YAC3C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,WAAW;SACrC,CAAC;IACJ,CAAC;IACD,OAAO;QACL,OAAO,EAAE,WAAW,MAAM,CAAC,SAAS,2BAA2B;QAC/D,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,MAAM;KAChC,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,OAAe,EAAE,MAAc,EAAE,GAAW,EAAE,cAAuB;IACtF,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,GAAG,OAAO,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;IACvC,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC;IAC3C,OAAO,GAAG,QAAQ,KAAK,MAAM,IAAI,IAAI,IAAI,GAAG,EAAE,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,MAAgB;IACrD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,KAAK,CAAC;IAC5C,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC3D,MAAM,cAAc,GAAG,MAAM,CAAC,cAAc,IAAI,KAAK,CAAC;IACtD,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,cAAc,CAAC,CAAC;IAE1F,OAAO;QACL,KAAK,CAAC,aAAa,CAAC,OAAe;YACjC,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,KAAK,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;YAEtC,+CAA+C;YAC/C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC;YACpD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;gBAC7B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC;oBAC9B,MAAM,EAAE,KAAK;oBACb,GAAG;oBACH,MAAM;oBACN,OAAO,EAAE,IAAI;oBACb,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE,EAAE,cAAc,EAAE,0BAA0B,EAAE;oBACvD,SAAS;iBACV,CAAC,CAAC;gBACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,IAAI,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;YACD,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QAChC,CAAC;QAED,KAAK,CAAC,WAAW,CAAC,WAAmB;YACnC,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC;gBAC9B,MAAM,EAAE,KAAK;gBACb,GAAG;gBACH,MAAM;gBACN,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS;aACV,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,WAAW,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7D,OAAO,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QAED,KAAK,CAAC,WAAW,CAAC,WAAmB;YACnC,MAAM,GAAG,GAAG,SAAS,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC,CAAC;YAC/C,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC;gBAC9B,MAAM,EAAE,MAAM;gBACd,GAAG;gBACH,MAAM;gBACN,OAAO,EAAE,IAAI;gBACb,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,SAAS;aACV,CAAC,CAAC;YACH,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,4EAA4E;AAC5E,wEAAwE;AACxE,0EAA0E;AAC1E,wDAAwD;AAExD,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,MAAM,CAAC;AAgBrC,KAAK,UAAU,aAAa,CAAC,IAAuB;IAClD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjE,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAEtC,MAAM,OAAO,GAA2B;QACtC,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QACvB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,sBAAsB,EAAE,QAAQ;QAChC,YAAY,EAAE,OAAO;KACtB,CAAC;IAEF,oBAAoB;IACpB,MAAM,iBAAiB,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;SAC3C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;SAC3B,IAAI,EAAE,CAAC;IACV,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,aAAa,GAAG,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAClD,MAAM,gBAAgB,GAAG;QACvB,IAAI,CAAC,MAAM;QACX,GAAG,CAAC,QAAQ;QACZ,GAAG,CAAC,YAAY,CAAC,QAAQ,EAAE,EAAE,kBAAkB;QAC/C,gBAAgB;QAChB,aAAa;QACb,QAAQ;KACT,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,iBAAiB;IACjB,MAAM,eAAe,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,eAAe,CAAC;IACnF,MAAM,oBAAoB,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzF,MAAM,YAAY,GAAG,CAAC,kBAAkB,EAAE,OAAO,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC,IAAI,CAC5F,IAAI,CACL,CAAC;IAEF,cAAc;IACd,MAAM,KAAK,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;IACvF,MAAM,OAAO,GAAG,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC;IACzE,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC;IAC7E,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC;IAChF,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEpF,MAAM,aAAa,GAAG,+BAA+B,IAAI,CAAC,SAAS,IAAI,eAAe,mBAAmB,aAAa,eAAe,SAAS,EAAE,CAAC;IAEjJ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,SAAS,IAAI,qBAAqB,CAAC,CAAC;IAC9F,IAAI,CAAC;QACH,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YACpC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,aAAa,EAAE;YACtC,IAAI,EAAE,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC9C,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,CAAO;IAC5B,0CAA0C;IAC1C,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACzD,OAAO,CACL,GAAG,CAAC,CAAC,cAAc,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,GAAG;QACzE,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,aAAa,EAAE,CAAC,GAAG,CAC7E,CAAC;AACJ,CAAC"}
|