@hostr/upload 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +298 -0
- package/dist/api/client.d.ts +71 -0
- package/dist/browser/index.d.ts +50 -0
- package/dist/browser.js +1061 -0
- package/dist/core/engine.d.ts +150 -0
- package/dist/core/events.d.ts +33 -0
- package/dist/core/retry.d.ts +20 -0
- package/dist/core/session.d.ts +28 -0
- package/dist/core/types.d.ts +44 -0
- package/dist/core/upload.d.ts +32 -0
- package/dist/hash/composite.d.ts +2 -0
- package/dist/hash/hasher.d.ts +2 -0
- package/dist/hash/hex.d.ts +2 -0
- package/dist/hash/md5.d.ts +24 -0
- package/dist/hash/sha256.d.ts +25 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +1012 -0
- package/dist/node/hasher.d.ts +2 -0
- package/dist/node/index.d.ts +43 -0
- package/dist/node.js +1089 -0
- package/dist/react-native/background-plan.d.ts +57 -0
- package/dist/react-native/background.d.ts +126 -0
- package/dist/react-native/fs.d.ts +28 -0
- package/dist/react-native/index.d.ts +69 -0
- package/dist/react-native/native-upload.d.ts +76 -0
- package/dist/react-native.js +896 -0
- package/package.json +48 -0
package/dist/node.js
ADDED
|
@@ -0,0 +1,1089 @@
|
|
|
1
|
+
// src/node/index.ts
|
|
2
|
+
import { open, mkdir, readFile, writeFile, unlink } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
|
|
6
|
+
// src/core/session.ts
|
|
7
|
+
var storageKey = (fileId) => `hostr.upload.v1:${fileId}`;
|
|
8
|
+
async function loadSession(store, fileId) {
|
|
9
|
+
try {
|
|
10
|
+
const raw = await store.get(storageKey(fileId));
|
|
11
|
+
if (raw == null) return null;
|
|
12
|
+
const parsed = JSON.parse(raw);
|
|
13
|
+
if (parsed?.v !== 1) return null;
|
|
14
|
+
return parsed;
|
|
15
|
+
} catch {
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
async function saveSession(store, s) {
|
|
20
|
+
await store.set(storageKey(s.fileId), JSON.stringify(s));
|
|
21
|
+
}
|
|
22
|
+
async function clearSession(store, fileId) {
|
|
23
|
+
await store.delete(storageKey(fileId));
|
|
24
|
+
}
|
|
25
|
+
function partSizeOf(s, n) {
|
|
26
|
+
if (n === s.expectedParts) {
|
|
27
|
+
return s.declaredSize - (s.expectedParts - 1) * s.partSize;
|
|
28
|
+
}
|
|
29
|
+
return s.partSize;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// src/api/client.ts
|
|
33
|
+
var ApiError = class extends Error {
|
|
34
|
+
status;
|
|
35
|
+
code;
|
|
36
|
+
constructor(status, code) {
|
|
37
|
+
super(code ? `hostr api error ${status}: ${code}` : `hostr api error ${status}`);
|
|
38
|
+
this.name = "ApiError";
|
|
39
|
+
this.status = status;
|
|
40
|
+
this.code = code;
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
var NetworkError = class extends Error {
|
|
44
|
+
constructor(message = "network error") {
|
|
45
|
+
super(message);
|
|
46
|
+
this.name = "NetworkError";
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
var trimEndpoint = (endpoint) => endpoint.replace(/\/$/, "");
|
|
50
|
+
async function req(fetchFn, endpoint, bearer, path, init = {}) {
|
|
51
|
+
const url = `${trimEndpoint(endpoint)}${path}`;
|
|
52
|
+
const headers = { authorization: `Bearer ${bearer}` };
|
|
53
|
+
let body;
|
|
54
|
+
if (init.body !== void 0) {
|
|
55
|
+
headers["content-type"] = "application/json";
|
|
56
|
+
body = JSON.stringify(init.body);
|
|
57
|
+
}
|
|
58
|
+
let res;
|
|
59
|
+
try {
|
|
60
|
+
res = await fetchFn(url, { method: init.method ?? "GET", headers, body });
|
|
61
|
+
} catch {
|
|
62
|
+
throw new NetworkError();
|
|
63
|
+
}
|
|
64
|
+
const text = await res.text();
|
|
65
|
+
const data = text ? JSON.parse(text) : {};
|
|
66
|
+
if (!res.ok) {
|
|
67
|
+
const code = data && typeof data === "object" && typeof data.error === "string" ? data.error : void 0;
|
|
68
|
+
throw new ApiError(res.status, code);
|
|
69
|
+
}
|
|
70
|
+
return data;
|
|
71
|
+
}
|
|
72
|
+
var HostrApi = class {
|
|
73
|
+
#endpoint;
|
|
74
|
+
#bearer;
|
|
75
|
+
#fetch;
|
|
76
|
+
constructor(endpoint, bearer, fetchFn = fetch) {
|
|
77
|
+
this.#endpoint = endpoint;
|
|
78
|
+
this.#bearer = bearer;
|
|
79
|
+
this.#fetch = fetchFn;
|
|
80
|
+
}
|
|
81
|
+
setBearer(bearer) {
|
|
82
|
+
this.#bearer = bearer;
|
|
83
|
+
}
|
|
84
|
+
getUpload(assetId) {
|
|
85
|
+
return req(this.#fetch, this.#endpoint, this.#bearer, `/v1/uploads/${assetId}`);
|
|
86
|
+
}
|
|
87
|
+
presignParts(assetId, partNumbers, expiresIn) {
|
|
88
|
+
return req(this.#fetch, this.#endpoint, this.#bearer, `/v1/uploads/${assetId}/parts`, {
|
|
89
|
+
method: "POST",
|
|
90
|
+
body: expiresIn === void 0 ? { partNumbers } : { partNumbers, expiresIn }
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
complete(assetId, body) {
|
|
94
|
+
return req(this.#fetch, this.#endpoint, this.#bearer, `/v1/uploads/${assetId}/complete`, {
|
|
95
|
+
method: "POST",
|
|
96
|
+
body
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
async abort(assetId) {
|
|
100
|
+
await req(this.#fetch, this.#endpoint, this.#bearer, `/v1/uploads/${assetId}`, { method: "DELETE" });
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
async function createUploadSession(opts, fetchFn = fetch) {
|
|
104
|
+
const { endpoint, apiKey, ...body } = opts;
|
|
105
|
+
return req(fetchFn, endpoint, apiKey, "/v1/uploads", { method: "POST", body });
|
|
106
|
+
}
|
|
107
|
+
function mintUploadToken(opts, fetchFn = fetch) {
|
|
108
|
+
const { endpoint, apiKey, assetId, tokenTtlSec } = opts;
|
|
109
|
+
return req(fetchFn, endpoint, apiKey, `/v1/uploads/${assetId}/token`, {
|
|
110
|
+
method: "POST",
|
|
111
|
+
body: tokenTtlSec === void 0 ? {} : { tokenTtlSec }
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// src/core/retry.ts
|
|
116
|
+
var RETRY_BUDGET = 6;
|
|
117
|
+
function classifyPut(res, err) {
|
|
118
|
+
if (err instanceof NetworkError) return "network";
|
|
119
|
+
if (err != null) return "fatal";
|
|
120
|
+
if (res === null) return "network";
|
|
121
|
+
const { status } = res;
|
|
122
|
+
if (status === 403) return "expired";
|
|
123
|
+
if (status === 429 || status >= 500 && status < 600) return "transient";
|
|
124
|
+
return "fatal";
|
|
125
|
+
}
|
|
126
|
+
function classifyApi(err) {
|
|
127
|
+
if (err instanceof NetworkError) return "network";
|
|
128
|
+
if (err instanceof ApiError) {
|
|
129
|
+
if (err.status === 429 || err.status >= 500 && err.status < 600) return "transient";
|
|
130
|
+
return "fatal";
|
|
131
|
+
}
|
|
132
|
+
return "transient";
|
|
133
|
+
}
|
|
134
|
+
function backoffMs(attempt, rng = Math.random, capMs = 6e4) {
|
|
135
|
+
const base = 500 * 2 ** (attempt - 1);
|
|
136
|
+
const jittered = base * (1 + 0.5 * rng());
|
|
137
|
+
return Math.min(capMs, jittered);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// src/core/events.ts
|
|
141
|
+
var Emitter = class {
|
|
142
|
+
listeners = /* @__PURE__ */ new Map();
|
|
143
|
+
on(ev, cb) {
|
|
144
|
+
let set = this.listeners.get(ev);
|
|
145
|
+
if (!set) {
|
|
146
|
+
set = /* @__PURE__ */ new Set();
|
|
147
|
+
this.listeners.set(ev, set);
|
|
148
|
+
}
|
|
149
|
+
const wrapped = cb;
|
|
150
|
+
set.add(wrapped);
|
|
151
|
+
return () => {
|
|
152
|
+
set.delete(wrapped);
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
emit(ev, v) {
|
|
156
|
+
const set = this.listeners.get(ev);
|
|
157
|
+
if (!set) return;
|
|
158
|
+
for (const cb of set) {
|
|
159
|
+
try {
|
|
160
|
+
cb(v);
|
|
161
|
+
} catch {
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
function computeProgress(i) {
|
|
167
|
+
const elapsedSec = (i.nowMs - i.startedAtMs) / 1e3;
|
|
168
|
+
const confirmedThisRun = i.confirmed - i.confirmedAtStart;
|
|
169
|
+
const throughputBps = elapsedSec >= 1 && confirmedThisRun > 0 ? confirmedThisRun / elapsedSec : null;
|
|
170
|
+
const bytesSent = Math.min(i.confirmed + i.inflight, i.total);
|
|
171
|
+
const etaSec = throughputBps === null ? null : (i.total - bytesSent) / throughputBps;
|
|
172
|
+
const progress = {
|
|
173
|
+
bytesSent,
|
|
174
|
+
bytesConfirmed: i.confirmed,
|
|
175
|
+
totalBytes: i.total,
|
|
176
|
+
throughputBps,
|
|
177
|
+
etaSec,
|
|
178
|
+
partsDone: i.partsDone,
|
|
179
|
+
partsTotal: i.partsTotal
|
|
180
|
+
};
|
|
181
|
+
if (i.partStates !== void 0) progress.partStates = i.partStates;
|
|
182
|
+
return progress;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// src/hash/hex.ts
|
|
186
|
+
var HEX = "0123456789abcdef";
|
|
187
|
+
function bytesToHex(b) {
|
|
188
|
+
let s = "";
|
|
189
|
+
for (let i = 0; i < b.length; i++) s += HEX[b[i] >> 4] + HEX[b[i] & 15];
|
|
190
|
+
return s;
|
|
191
|
+
}
|
|
192
|
+
function hexToBytes(s) {
|
|
193
|
+
const out = new Uint8Array(s.length >> 1);
|
|
194
|
+
for (let i = 0; i < out.length; i++) out[i] = parseInt(s.slice(i * 2, i * 2 + 2), 16);
|
|
195
|
+
return out;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// src/hash/sha256.ts
|
|
199
|
+
var K = new Int32Array(64);
|
|
200
|
+
var H0 = new Int32Array(8);
|
|
201
|
+
{
|
|
202
|
+
const primes = [];
|
|
203
|
+
for (let n = 2; primes.length < 64; n++) if (primes.every((p) => n % p)) primes.push(n);
|
|
204
|
+
for (let i = 0; i < 64; i++) K[i] = Math.floor(primes[i] ** (1 / 3) % 1 * 2 ** 32) | 0;
|
|
205
|
+
for (let i = 0; i < 8; i++) H0[i] = Math.floor(Math.sqrt(primes[i]) % 1 * 2 ** 32) | 0;
|
|
206
|
+
}
|
|
207
|
+
var rotr = (x, n) => x >>> n | x << 32 - n;
|
|
208
|
+
var Sha256 = class _Sha256 {
|
|
209
|
+
h = new Int32Array(H0);
|
|
210
|
+
buf = new Uint8Array(64);
|
|
211
|
+
bufLen = 0;
|
|
212
|
+
len = 0;
|
|
213
|
+
// total bytes; Number is exact to 2^53
|
|
214
|
+
w = new Int32Array(64);
|
|
215
|
+
update(data) {
|
|
216
|
+
this.len += data.length;
|
|
217
|
+
let off = 0;
|
|
218
|
+
if (this.bufLen > 0) {
|
|
219
|
+
const take = Math.min(64 - this.bufLen, data.length);
|
|
220
|
+
this.buf.set(data.subarray(0, take), this.bufLen);
|
|
221
|
+
this.bufLen += take;
|
|
222
|
+
off = take;
|
|
223
|
+
if (this.bufLen === 64) {
|
|
224
|
+
this.compress(this.buf, 0);
|
|
225
|
+
this.bufLen = 0;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
for (; off + 64 <= data.length; off += 64) this.compress(data, off);
|
|
229
|
+
if (off < data.length) {
|
|
230
|
+
this.buf.set(data.subarray(off));
|
|
231
|
+
this.bufLen = data.length - off;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
compress(p, off) {
|
|
235
|
+
const w = this.w;
|
|
236
|
+
for (let i = 0; i < 16; i++)
|
|
237
|
+
w[i] = p[off + 4 * i] << 24 | p[off + 4 * i + 1] << 16 | p[off + 4 * i + 2] << 8 | p[off + 4 * i + 3];
|
|
238
|
+
for (let i = 16; i < 64; i++) {
|
|
239
|
+
const s0 = rotr(w[i - 15], 7) ^ rotr(w[i - 15], 18) ^ w[i - 15] >>> 3;
|
|
240
|
+
const s1 = rotr(w[i - 2], 17) ^ rotr(w[i - 2], 19) ^ w[i - 2] >>> 10;
|
|
241
|
+
w[i] = w[i - 16] + s0 + w[i - 7] + s1 | 0;
|
|
242
|
+
}
|
|
243
|
+
let [a, b, c, d, e, f, g, hh] = [this.h[0], this.h[1], this.h[2], this.h[3], this.h[4], this.h[5], this.h[6], this.h[7]];
|
|
244
|
+
for (let i = 0; i < 64; i++) {
|
|
245
|
+
const S1 = rotr(e, 6) ^ rotr(e, 11) ^ rotr(e, 25);
|
|
246
|
+
const ch = e & f ^ ~e & g;
|
|
247
|
+
const t1 = hh + S1 + ch + K[i] + w[i] | 0;
|
|
248
|
+
const S0 = rotr(a, 2) ^ rotr(a, 13) ^ rotr(a, 22);
|
|
249
|
+
const maj = a & b ^ a & c ^ b & c;
|
|
250
|
+
const t2 = S0 + maj | 0;
|
|
251
|
+
hh = g;
|
|
252
|
+
g = f;
|
|
253
|
+
f = e;
|
|
254
|
+
e = d + t1 | 0;
|
|
255
|
+
d = c;
|
|
256
|
+
c = b;
|
|
257
|
+
b = a;
|
|
258
|
+
a = t1 + t2 | 0;
|
|
259
|
+
}
|
|
260
|
+
this.h[0] = this.h[0] + a | 0;
|
|
261
|
+
this.h[1] = this.h[1] + b | 0;
|
|
262
|
+
this.h[2] = this.h[2] + c | 0;
|
|
263
|
+
this.h[3] = this.h[3] + d | 0;
|
|
264
|
+
this.h[4] = this.h[4] + e | 0;
|
|
265
|
+
this.h[5] = this.h[5] + f | 0;
|
|
266
|
+
this.h[6] = this.h[6] + g | 0;
|
|
267
|
+
this.h[7] = this.h[7] + hh | 0;
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Finalize on a clone so the hash can keep accepting updates.
|
|
271
|
+
*
|
|
272
|
+
* Padding: capture the message length in bits BEFORE touching the clone,
|
|
273
|
+
* then build the entire 0x80 + zero-padding + 64-bit length tail as one
|
|
274
|
+
* buffer sized so the clone's running byte count lands on a block
|
|
275
|
+
* boundary, and hand it to update() in a single call. update()'s own
|
|
276
|
+
* `len` bookkeeping on the clone is irrelevant after this point (the
|
|
277
|
+
* clone is discarded), so there is no need to keep it meaningful.
|
|
278
|
+
*/
|
|
279
|
+
digestBytes() {
|
|
280
|
+
const c = this.clone();
|
|
281
|
+
const bitLen = c.len * 8;
|
|
282
|
+
const zeroCount = (56 - (c.len + 1) % 64 + 64) % 64;
|
|
283
|
+
const pad = new Uint8Array(1 + zeroCount + 8);
|
|
284
|
+
pad[0] = 128;
|
|
285
|
+
const lo = bitLen >>> 0, hi = Math.floor(bitLen / 2 ** 32);
|
|
286
|
+
const tailOff = 1 + zeroCount;
|
|
287
|
+
for (let i = 0; i < 4; i++) {
|
|
288
|
+
pad[tailOff + i] = hi >>> 8 * (3 - i) & 255;
|
|
289
|
+
pad[tailOff + 4 + i] = lo >>> 8 * (3 - i) & 255;
|
|
290
|
+
}
|
|
291
|
+
c.update(pad);
|
|
292
|
+
const out = new Uint8Array(32);
|
|
293
|
+
for (let i = 0; i < 8; i++)
|
|
294
|
+
for (let j = 0; j < 4; j++) out[4 * i + j] = c.h[i] >>> 8 * (3 - j) & 255;
|
|
295
|
+
return out;
|
|
296
|
+
}
|
|
297
|
+
digestHex() {
|
|
298
|
+
return bytesToHex(this.digestBytes());
|
|
299
|
+
}
|
|
300
|
+
exportState() {
|
|
301
|
+
return JSON.stringify({ h: [...this.h], buf: bytesToHex(this.buf.subarray(0, this.bufLen)), len: this.len });
|
|
302
|
+
}
|
|
303
|
+
importState(state) {
|
|
304
|
+
const s = JSON.parse(state);
|
|
305
|
+
this.h = new Int32Array(s.h);
|
|
306
|
+
this.buf.fill(0);
|
|
307
|
+
this.bufLen = s.buf.length / 2;
|
|
308
|
+
this.len = s.len;
|
|
309
|
+
for (let i = 0; i < this.bufLen; i++) this.buf[i] = parseInt(s.buf.slice(i * 2, i * 2 + 2), 16);
|
|
310
|
+
}
|
|
311
|
+
clone() {
|
|
312
|
+
const c = new _Sha256();
|
|
313
|
+
c.h = new Int32Array(this.h);
|
|
314
|
+
c.buf = new Uint8Array(this.buf);
|
|
315
|
+
c.bufLen = this.bufLen;
|
|
316
|
+
c.len = this.len;
|
|
317
|
+
return c;
|
|
318
|
+
}
|
|
319
|
+
};
|
|
320
|
+
|
|
321
|
+
// src/hash/md5.ts
|
|
322
|
+
var K2 = new Int32Array(64);
|
|
323
|
+
for (let i = 0; i < 64; i++) K2[i] = Math.floor(Math.abs(Math.sin(i + 1)) * 2 ** 32) | 0;
|
|
324
|
+
var S = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21];
|
|
325
|
+
var rotl = (x, n) => x << n | x >>> 32 - n;
|
|
326
|
+
var Md5 = class _Md5 {
|
|
327
|
+
h = new Int32Array([1732584193, 4023233417 | 0, 2562383102 | 0, 271733878]);
|
|
328
|
+
buf = new Uint8Array(64);
|
|
329
|
+
bufLen = 0;
|
|
330
|
+
len = 0;
|
|
331
|
+
// total bytes; Number is exact to 2^53
|
|
332
|
+
update(data) {
|
|
333
|
+
this.len += data.length;
|
|
334
|
+
let off = 0;
|
|
335
|
+
if (this.bufLen > 0) {
|
|
336
|
+
const take = Math.min(64 - this.bufLen, data.length);
|
|
337
|
+
this.buf.set(data.subarray(0, take), this.bufLen);
|
|
338
|
+
this.bufLen += take;
|
|
339
|
+
off = take;
|
|
340
|
+
if (this.bufLen === 64) {
|
|
341
|
+
this.compress(this.buf, 0);
|
|
342
|
+
this.bufLen = 0;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
for (; off + 64 <= data.length; off += 64) this.compress(data, off);
|
|
346
|
+
if (off < data.length) {
|
|
347
|
+
this.buf.set(data.subarray(off));
|
|
348
|
+
this.bufLen = data.length - off;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
compress(p, off) {
|
|
352
|
+
const m = new Int32Array(16);
|
|
353
|
+
for (let i = 0; i < 16; i++)
|
|
354
|
+
m[i] = p[off + 4 * i] | p[off + 4 * i + 1] << 8 | p[off + 4 * i + 2] << 16 | p[off + 4 * i + 3] << 24;
|
|
355
|
+
let a = this.h[0], b = this.h[1], c = this.h[2], d = this.h[3];
|
|
356
|
+
for (let i = 0; i < 64; i++) {
|
|
357
|
+
let f, g;
|
|
358
|
+
if (i < 16) {
|
|
359
|
+
f = b & c | ~b & d;
|
|
360
|
+
g = i;
|
|
361
|
+
} else if (i < 32) {
|
|
362
|
+
f = d & b | ~d & c;
|
|
363
|
+
g = (5 * i + 1) % 16;
|
|
364
|
+
} else if (i < 48) {
|
|
365
|
+
f = b ^ c ^ d;
|
|
366
|
+
g = (3 * i + 5) % 16;
|
|
367
|
+
} else {
|
|
368
|
+
f = c ^ (b | ~d);
|
|
369
|
+
g = 7 * i % 16;
|
|
370
|
+
}
|
|
371
|
+
const t = d;
|
|
372
|
+
d = c;
|
|
373
|
+
c = b;
|
|
374
|
+
b = b + rotl(a + f + K2[i] + m[g] | 0, S[(i >> 4) * 4 + (i & 3)]) | 0;
|
|
375
|
+
a = t;
|
|
376
|
+
}
|
|
377
|
+
this.h[0] = this.h[0] + a | 0;
|
|
378
|
+
this.h[1] = this.h[1] + b | 0;
|
|
379
|
+
this.h[2] = this.h[2] + c | 0;
|
|
380
|
+
this.h[3] = this.h[3] + d | 0;
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* Finalize on a clone so the hash can keep accepting updates.
|
|
384
|
+
*
|
|
385
|
+
* Padding: capture the message length in bits BEFORE touching the clone,
|
|
386
|
+
* then build the entire 0x80 + zero-padding + 64-bit length tail as one
|
|
387
|
+
* buffer sized so the clone's running byte count lands on a block
|
|
388
|
+
* boundary, and hand it to update() in a single call. update()'s own
|
|
389
|
+
* `len` bookkeeping on the clone is irrelevant after this point (the
|
|
390
|
+
* clone is discarded), so there is no need to keep it meaningful.
|
|
391
|
+
*/
|
|
392
|
+
digestBytes() {
|
|
393
|
+
const c = this.clone();
|
|
394
|
+
const bitLen = c.len * 8;
|
|
395
|
+
const zeroCount = (56 - (c.len + 1) % 64 + 64) % 64;
|
|
396
|
+
const pad = new Uint8Array(1 + zeroCount + 8);
|
|
397
|
+
pad[0] = 128;
|
|
398
|
+
const lo = bitLen >>> 0, hi = Math.floor(bitLen / 2 ** 32);
|
|
399
|
+
const tailOff = 1 + zeroCount;
|
|
400
|
+
for (let i = 0; i < 4; i++) {
|
|
401
|
+
pad[tailOff + i] = lo >>> 8 * i & 255;
|
|
402
|
+
pad[tailOff + 4 + i] = hi >>> 8 * i & 255;
|
|
403
|
+
}
|
|
404
|
+
c.update(pad);
|
|
405
|
+
const out = new Uint8Array(16);
|
|
406
|
+
for (let i = 0; i < 4; i++)
|
|
407
|
+
for (let j = 0; j < 4; j++) out[4 * i + j] = c.h[i] >>> 8 * j & 255;
|
|
408
|
+
return out;
|
|
409
|
+
}
|
|
410
|
+
digestHex() {
|
|
411
|
+
return bytesToHex(this.digestBytes());
|
|
412
|
+
}
|
|
413
|
+
exportState() {
|
|
414
|
+
return JSON.stringify({ h: [...this.h], buf: bytesToHex(this.buf.subarray(0, this.bufLen)), len: this.len });
|
|
415
|
+
}
|
|
416
|
+
importState(state) {
|
|
417
|
+
const s = JSON.parse(state);
|
|
418
|
+
this.h = new Int32Array(s.h);
|
|
419
|
+
this.buf.fill(0);
|
|
420
|
+
this.bufLen = s.buf.length / 2;
|
|
421
|
+
this.len = s.len;
|
|
422
|
+
for (let i = 0; i < this.bufLen; i++) this.buf[i] = parseInt(s.buf.slice(i * 2, i * 2 + 2), 16);
|
|
423
|
+
}
|
|
424
|
+
clone() {
|
|
425
|
+
const c = new _Md5();
|
|
426
|
+
c.h = new Int32Array(this.h);
|
|
427
|
+
c.buf = new Uint8Array(this.buf);
|
|
428
|
+
c.bufLen = this.bufLen;
|
|
429
|
+
c.len = this.len;
|
|
430
|
+
return c;
|
|
431
|
+
}
|
|
432
|
+
};
|
|
433
|
+
|
|
434
|
+
// src/hash/composite.ts
|
|
435
|
+
function compositeEtag(md5Hexes) {
|
|
436
|
+
const h = new Md5();
|
|
437
|
+
for (const hex of md5Hexes) h.update(hexToBytes(hex));
|
|
438
|
+
return `${h.digestHex()}-${md5Hexes.length}`;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// src/core/engine.ts
|
|
442
|
+
var SYSTEMIC_TRANSPORT_FAILURE_THRESHOLD = 3;
|
|
443
|
+
var IntegrityError = class extends Error {
|
|
444
|
+
constructor(message) {
|
|
445
|
+
super(message);
|
|
446
|
+
this.name = "IntegrityError";
|
|
447
|
+
}
|
|
448
|
+
};
|
|
449
|
+
var Stopped = class extends Error {
|
|
450
|
+
constructor(message) {
|
|
451
|
+
super(message);
|
|
452
|
+
this.name = "Stopped";
|
|
453
|
+
}
|
|
454
|
+
};
|
|
455
|
+
var stripQuotes = (s) => s.replace(/^"|"$/g, "");
|
|
456
|
+
var TERMINAL = ["complete", "failed", "aborted"];
|
|
457
|
+
var UploadEngine = class {
|
|
458
|
+
done;
|
|
459
|
+
resolveDone;
|
|
460
|
+
rejectDone;
|
|
461
|
+
emitter = new Emitter();
|
|
462
|
+
presigned = /* @__PURE__ */ new Map();
|
|
463
|
+
/** Presign fetches since the last confirmed part. Healthy uploads confirm
|
|
464
|
+
* constantly, so this stays at zero and costs nothing; in a sustained
|
|
465
|
+
* failure loop (every PUT dying without landing a part) it grows, and
|
|
466
|
+
* presign() spaces further /parts fetches on the same backoff curve the
|
|
467
|
+
* 'expired' storm uses — the API call rate is what trips rate limits, so
|
|
468
|
+
* it gets its own governor, independent of per-part attempt cadence and
|
|
469
|
+
* worker count. */
|
|
470
|
+
presignsSinceConfirm = 0;
|
|
471
|
+
/** The in-flight /parts fetch, shared so concurrent cache-missing workers
|
|
472
|
+
* ride one round-trip and re-check the cache instead of each firing an
|
|
473
|
+
* overlapping batch of their own (same memo shape as hashLanePromise). */
|
|
474
|
+
presignInflight = null;
|
|
475
|
+
claimed = /* @__PURE__ */ new Set();
|
|
476
|
+
inflightSent = /* @__PURE__ */ new Map();
|
|
477
|
+
stopped = false;
|
|
478
|
+
running = false;
|
|
479
|
+
restartPending = false;
|
|
480
|
+
startedAt = 0;
|
|
481
|
+
confirmedAtStart = 0;
|
|
482
|
+
lastEmit = 0;
|
|
483
|
+
deps;
|
|
484
|
+
opts;
|
|
485
|
+
session;
|
|
486
|
+
/** The in-flight sha256 lane, tracked so a pause→resume never starts a
|
|
487
|
+
* second concurrent lane over the same file. Cleared once settled so a
|
|
488
|
+
* later resume can start a fresh one from the persisted midstate. */
|
|
489
|
+
hashLanePromise = null;
|
|
490
|
+
/** Engine-wide (not per-part): a CORS-blocked bucket fails every worker's
|
|
491
|
+
* PUT the same way, so the streak accumulates across parts. Deliberately
|
|
492
|
+
* not bumped by apiCall's network stalls — the API answering while every
|
|
493
|
+
* R2 PUT dies is exactly the bucket-CORS signature. */
|
|
494
|
+
consecutiveNetworkFailures = 0;
|
|
495
|
+
diagnosticFired = false;
|
|
496
|
+
/** Resolvers for anyone currently parked in waitOnline(), woken the
|
|
497
|
+
* instant pause()/abort()/fail() halts the engine — without this a
|
|
498
|
+
* stalled engine could only be interrupted by the network itself
|
|
499
|
+
* coming back online. */
|
|
500
|
+
stopWaiters = /* @__PURE__ */ new Set();
|
|
501
|
+
constructor(deps, opts, session) {
|
|
502
|
+
this.deps = deps;
|
|
503
|
+
this.opts = opts;
|
|
504
|
+
this.session = session;
|
|
505
|
+
this.done = new Promise((res, rej) => {
|
|
506
|
+
this.resolveDone = res;
|
|
507
|
+
this.rejectDone = rej;
|
|
508
|
+
});
|
|
509
|
+
this.done.catch(() => {
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
on(ev, cb) {
|
|
513
|
+
return this.emitter.on(ev, cb);
|
|
514
|
+
}
|
|
515
|
+
get state() {
|
|
516
|
+
return this.session.status;
|
|
517
|
+
}
|
|
518
|
+
start() {
|
|
519
|
+
if (TERMINAL.includes(this.session.status)) return;
|
|
520
|
+
if (this.running) {
|
|
521
|
+
this.restartPending = true;
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
524
|
+
this.stopped = false;
|
|
525
|
+
this.running = true;
|
|
526
|
+
this.run().catch((e) => {
|
|
527
|
+
if (!(e instanceof Stopped)) this.fail(e instanceof Error ? e : new Error(String(e)));
|
|
528
|
+
}).finally(() => {
|
|
529
|
+
this.running = false;
|
|
530
|
+
if (this.restartPending) {
|
|
531
|
+
this.restartPending = false;
|
|
532
|
+
this.start();
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
}
|
|
536
|
+
pause() {
|
|
537
|
+
this.halt();
|
|
538
|
+
this.restartPending = false;
|
|
539
|
+
this.setStatus("paused");
|
|
540
|
+
}
|
|
541
|
+
resume() {
|
|
542
|
+
this.start();
|
|
543
|
+
}
|
|
544
|
+
async abort() {
|
|
545
|
+
this.halt();
|
|
546
|
+
this.setStatus("aborted");
|
|
547
|
+
await this.deps.api.abort(this.session.assetId).catch(() => {
|
|
548
|
+
});
|
|
549
|
+
await clearSession(this.deps.store, this.session.fileId);
|
|
550
|
+
}
|
|
551
|
+
/** Stop dispatch everywhere at once: new part claims, waitOnline's
|
|
552
|
+
* parked callers, and apiCall's network-retry loop all observe this on
|
|
553
|
+
* their next check. */
|
|
554
|
+
halt() {
|
|
555
|
+
this.stopped = true;
|
|
556
|
+
for (const w of this.stopWaiters) w();
|
|
557
|
+
this.stopWaiters.clear();
|
|
558
|
+
}
|
|
559
|
+
setStatus(s) {
|
|
560
|
+
if (this.session.status === s) return;
|
|
561
|
+
this.session.status = s;
|
|
562
|
+
this.emitter.emit("statechange", s);
|
|
563
|
+
}
|
|
564
|
+
fail(e) {
|
|
565
|
+
this.halt();
|
|
566
|
+
this.setStatus("failed");
|
|
567
|
+
this.emitter.emit("error", e);
|
|
568
|
+
this.rejectDone(e);
|
|
569
|
+
}
|
|
570
|
+
async run() {
|
|
571
|
+
const remote = await this.apiCall(() => this.deps.api.getUpload(this.session.assetId));
|
|
572
|
+
if (remote.status !== "uploading") {
|
|
573
|
+
if (["uploaded", "queued", "processing", "ready"].includes(remote.status)) return this.finish();
|
|
574
|
+
throw new Error(`upload is ${remote.status}, cannot continue`);
|
|
575
|
+
}
|
|
576
|
+
await this.reconcile(remote.parts);
|
|
577
|
+
if (this.stopped) return;
|
|
578
|
+
this.setStatus("running");
|
|
579
|
+
this.startedAt = this.deps.clock.now();
|
|
580
|
+
this.confirmedAtStart = this.bytesConfirmed();
|
|
581
|
+
const hashLane = this.getHashLane();
|
|
582
|
+
await Promise.all(Array.from({ length: this.opts.concurrency }, () => this.worker()));
|
|
583
|
+
if (this.stopped) return;
|
|
584
|
+
this.session.sha256 = await hashLane;
|
|
585
|
+
if (this.stopped) return;
|
|
586
|
+
await saveSession(this.deps.store, this.session);
|
|
587
|
+
await this.finish();
|
|
588
|
+
}
|
|
589
|
+
async finish() {
|
|
590
|
+
this.setStatus("completing");
|
|
591
|
+
await saveSession(this.deps.store, this.session);
|
|
592
|
+
const res = await this.apiCall(() => this.deps.api.complete(
|
|
593
|
+
this.session.assetId,
|
|
594
|
+
this.session.sha256 ? { sha256: this.session.sha256 } : {}
|
|
595
|
+
));
|
|
596
|
+
const expected = this.expectedComposite();
|
|
597
|
+
if (expected && res.etag && res.etag !== expected)
|
|
598
|
+
throw new IntegrityError(`R2 assembled ${res.etag}; client computed ${expected}`);
|
|
599
|
+
await clearSession(this.deps.store, this.session.fileId);
|
|
600
|
+
await this.deps.file.close?.();
|
|
601
|
+
this.setStatus("complete");
|
|
602
|
+
const info = {
|
|
603
|
+
assetId: this.session.assetId,
|
|
604
|
+
etag: res.etag ?? expected ?? "",
|
|
605
|
+
sha256: this.session.sha256,
|
|
606
|
+
bytes: res.bytes ?? this.session.declaredSize
|
|
607
|
+
};
|
|
608
|
+
this.emitter.emit("complete", info);
|
|
609
|
+
this.resolveDone(info);
|
|
610
|
+
}
|
|
611
|
+
expectedComposite() {
|
|
612
|
+
const md5s = [];
|
|
613
|
+
for (let n = 1; n <= this.session.expectedParts; n++) {
|
|
614
|
+
const m = this.session.done[n];
|
|
615
|
+
if (!m) return null;
|
|
616
|
+
md5s.push(m);
|
|
617
|
+
}
|
|
618
|
+
return compositeEtag(md5s);
|
|
619
|
+
}
|
|
620
|
+
/** Reconcile local bookkeeping against what R2 actually holds before resuming.
|
|
621
|
+
* Local claims R2 can't confirm are stale and forgotten; parts whose stored
|
|
622
|
+
* md5 matches R2's etag are kept as-is; a mismatch means the landed bytes
|
|
623
|
+
* aren't verified as ours, so it's dropped for re-upload. For parts R2 holds
|
|
624
|
+
* but local `done` has no record of (state lost — reinstall, cleared
|
|
625
|
+
* storage), the local byte range is re-read and re-hashed; it's only
|
|
626
|
+
* adopted as done when that hash matches R2's etag, never trusted blind. */
|
|
627
|
+
async reconcile(remote) {
|
|
628
|
+
const remoteByN = new Map(remote.map((p) => [p.partNumber, p]));
|
|
629
|
+
for (const n of Object.keys(this.session.done).map(Number))
|
|
630
|
+
if (!remoteByN.has(n)) delete this.session.done[n];
|
|
631
|
+
for (const p of remote) {
|
|
632
|
+
const have = this.session.done[p.partNumber];
|
|
633
|
+
if (have === p.etag) continue;
|
|
634
|
+
if (have) {
|
|
635
|
+
delete this.session.done[p.partNumber];
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
const offset = (p.partNumber - 1) * this.session.partSize;
|
|
639
|
+
const body = await this.deps.file.read(offset, partSizeOf(this.session, p.partNumber));
|
|
640
|
+
const h = this.deps.hasher.md5();
|
|
641
|
+
h.update(body);
|
|
642
|
+
const md5 = h.digestHex();
|
|
643
|
+
if (md5 === p.etag) this.session.done[p.partNumber] = md5;
|
|
644
|
+
}
|
|
645
|
+
if (this.session.status === "aborted") return;
|
|
646
|
+
await saveSession(this.deps.store, this.session);
|
|
647
|
+
}
|
|
648
|
+
nextPart() {
|
|
649
|
+
for (let n = 1; n <= this.session.expectedParts; n++)
|
|
650
|
+
if (!this.session.done[n] && !this.claimed.has(n)) return n;
|
|
651
|
+
return null;
|
|
652
|
+
}
|
|
653
|
+
async worker() {
|
|
654
|
+
while (!this.stopped) {
|
|
655
|
+
const n = this.nextPart();
|
|
656
|
+
if (n == null) return;
|
|
657
|
+
this.claimed.add(n);
|
|
658
|
+
try {
|
|
659
|
+
await this.uploadPart(n);
|
|
660
|
+
} finally {
|
|
661
|
+
this.claimed.delete(n);
|
|
662
|
+
this.inflightSent.delete(n);
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
async uploadPart(n) {
|
|
667
|
+
let budget = RETRY_BUDGET, attempt = 0, consecutiveExpired = 0;
|
|
668
|
+
let cls;
|
|
669
|
+
for (; ; ) {
|
|
670
|
+
if (this.stopped) return;
|
|
671
|
+
await this.waitOnline();
|
|
672
|
+
if (this.stopped) return;
|
|
673
|
+
let caught;
|
|
674
|
+
try {
|
|
675
|
+
const url = await this.presign(n, cls === "expired");
|
|
676
|
+
const offset = (n - 1) * this.session.partSize;
|
|
677
|
+
const expectedLen = partSizeOf(this.session, n);
|
|
678
|
+
const body = await this.deps.file.read(offset, expectedLen);
|
|
679
|
+
if (body.length !== expectedLen)
|
|
680
|
+
throw new Error(
|
|
681
|
+
`part ${n}: source file changed size mid-upload (expected ${expectedLen} bytes, read ${body.length}) \u2014 the file was modified or truncated after the upload started`
|
|
682
|
+
);
|
|
683
|
+
const h = this.deps.hasher.md5();
|
|
684
|
+
h.update(body);
|
|
685
|
+
const md5 = h.digestHex();
|
|
686
|
+
const res = await this.deps.transport.put(
|
|
687
|
+
url,
|
|
688
|
+
body,
|
|
689
|
+
{ onProgress: (s) => this.onPartProgress(n, s) }
|
|
690
|
+
);
|
|
691
|
+
if (res.status >= 200 && res.status < 300) {
|
|
692
|
+
if (res.etag && stripQuotes(res.etag) === md5) {
|
|
693
|
+
if (this.session.status === "aborted") return;
|
|
694
|
+
this.notePutOutcome("ok");
|
|
695
|
+
this.session.done[n] = md5;
|
|
696
|
+
this.presignsSinceConfirm = 0;
|
|
697
|
+
await saveSession(this.deps.store, this.session);
|
|
698
|
+
this.inflightSent.delete(n);
|
|
699
|
+
this.emitProgress(true);
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
cls = "corrupt";
|
|
703
|
+
} else cls = classifyPut(res, void 0);
|
|
704
|
+
} catch (err) {
|
|
705
|
+
if (err instanceof Stopped) return;
|
|
706
|
+
caught = err instanceof Error ? err : void 0;
|
|
707
|
+
cls = err instanceof ApiError ? classifyApi(err) : classifyPut(null, err);
|
|
708
|
+
}
|
|
709
|
+
this.inflightSent.delete(n);
|
|
710
|
+
this.notePutOutcome(cls);
|
|
711
|
+
if (cls === "fatal") {
|
|
712
|
+
this.halt();
|
|
713
|
+
throw caught ?? new Error(`part ${n}: unrecoverable upload error`);
|
|
714
|
+
}
|
|
715
|
+
if (cls === "expired") {
|
|
716
|
+
this.presigned.delete(n);
|
|
717
|
+
consecutiveExpired++;
|
|
718
|
+
if (consecutiveExpired > 1) await this.deps.clock.sleep(backoffMs(consecutiveExpired, this.deps.rng));
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
consecutiveExpired = 0;
|
|
722
|
+
attempt++;
|
|
723
|
+
if (cls !== "network" && --budget <= 0) {
|
|
724
|
+
this.halt();
|
|
725
|
+
throw new Error(`part ${n}: retries exhausted (${cls})`);
|
|
726
|
+
}
|
|
727
|
+
if (cls === "network") this.setStatus("stalled");
|
|
728
|
+
await this.deps.clock.sleep(backoffMs(attempt, this.deps.rng));
|
|
729
|
+
if (cls === "network" && !this.stopped) this.setStatus("running");
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
async presign(n, afterExpiry = false) {
|
|
733
|
+
const c = this.presigned.get(n);
|
|
734
|
+
if (c && this.presignFresh(c)) return c.url;
|
|
735
|
+
if (this.presignInflight) {
|
|
736
|
+
await this.presignInflight;
|
|
737
|
+
if (this.stopped) throw new Stopped();
|
|
738
|
+
const landed = this.presigned.get(n);
|
|
739
|
+
if (landed && this.presignFresh(landed)) return landed.url;
|
|
740
|
+
}
|
|
741
|
+
if (!afterExpiry && this.presignsSinceConfirm > 0) {
|
|
742
|
+
await this.deps.clock.sleep(backoffMs(this.presignsSinceConfirm, this.deps.rng));
|
|
743
|
+
if (this.stopped) throw new Stopped();
|
|
744
|
+
const landed = this.presigned.get(n);
|
|
745
|
+
if (landed && this.presignFresh(landed)) return landed.url;
|
|
746
|
+
}
|
|
747
|
+
this.presignsSinceConfirm++;
|
|
748
|
+
const want = [n];
|
|
749
|
+
for (let m = 1; m <= this.session.expectedParts && want.length < this.opts.presignBatch; m++) {
|
|
750
|
+
const e = this.presigned.get(m);
|
|
751
|
+
if (m !== n && !this.session.done[m] && !(e && this.presignFresh(e))) want.push(m);
|
|
752
|
+
}
|
|
753
|
+
const pending = this.apiCall(() => this.deps.api.presignParts(this.session.assetId, want, this.opts.presignTtlSec));
|
|
754
|
+
const inflight = pending.then(() => {
|
|
755
|
+
}, () => {
|
|
756
|
+
});
|
|
757
|
+
this.presignInflight = inflight;
|
|
758
|
+
try {
|
|
759
|
+
const res = await pending;
|
|
760
|
+
for (const u of res.urls)
|
|
761
|
+
this.presigned.set(u.partNumber, { url: u.url, expiresAt: this.deps.clock.now() + res.expiresIn * 1e3 });
|
|
762
|
+
return this.presigned.get(n).url;
|
|
763
|
+
} finally {
|
|
764
|
+
if (this.presignInflight === inflight) this.presignInflight = null;
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
/** Fresh = safely usable for a new PUT: at least 60s of validity left, so
|
|
768
|
+
* a part starting now won't have its URL expire mid-flight. */
|
|
769
|
+
presignFresh(c) {
|
|
770
|
+
return c.expiresAt - 6e4 > this.deps.clock.now();
|
|
771
|
+
}
|
|
772
|
+
/** API calls: infinite patience for network loss, one token refresh on 401, else throw.
|
|
773
|
+
* Throws Stopped (never surfaced) if pause()/abort() interrupts a stuck retry. */
|
|
774
|
+
async apiCall(fn) {
|
|
775
|
+
const attempts = /* @__PURE__ */ new Map();
|
|
776
|
+
let refreshed = false;
|
|
777
|
+
const bump = (cls) => {
|
|
778
|
+
const n = (attempts.get(cls) ?? 0) + 1;
|
|
779
|
+
attempts.set(cls, n);
|
|
780
|
+
return n;
|
|
781
|
+
};
|
|
782
|
+
for (; ; ) {
|
|
783
|
+
try {
|
|
784
|
+
return await fn();
|
|
785
|
+
} catch (e) {
|
|
786
|
+
const cls = classifyApi(e);
|
|
787
|
+
if (cls === "network") {
|
|
788
|
+
this.setStatus("stalled");
|
|
789
|
+
const attempt = bump("network");
|
|
790
|
+
await this.waitOnline();
|
|
791
|
+
if (this.stopped) throw new Stopped();
|
|
792
|
+
await this.deps.clock.sleep(backoffMs(attempt, this.deps.rng));
|
|
793
|
+
if (this.stopped) throw new Stopped();
|
|
794
|
+
continue;
|
|
795
|
+
}
|
|
796
|
+
if (e instanceof ApiError && e.status === 401 && this.opts.onTokenExpired && !refreshed) {
|
|
797
|
+
refreshed = true;
|
|
798
|
+
const t = await this.opts.onTokenExpired(this.session.assetId);
|
|
799
|
+
this.session.uploadToken = t;
|
|
800
|
+
this.deps.api.setBearer(t);
|
|
801
|
+
await saveSession(this.deps.store, this.session);
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
if (cls === "transient") {
|
|
805
|
+
const attempt = bump("transient");
|
|
806
|
+
if (attempt <= RETRY_BUDGET) {
|
|
807
|
+
await this.deps.clock.sleep(backoffMs(attempt, this.deps.rng));
|
|
808
|
+
continue;
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
throw e;
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
async waitOnline() {
|
|
816
|
+
const net = this.deps.network;
|
|
817
|
+
if (!net || net.isOnline()) return;
|
|
818
|
+
if (this.stopped) return;
|
|
819
|
+
this.setStatus("stalled");
|
|
820
|
+
await new Promise((res) => {
|
|
821
|
+
let off = () => {
|
|
822
|
+
};
|
|
823
|
+
const finish = () => {
|
|
824
|
+
off();
|
|
825
|
+
this.stopWaiters.delete(finish);
|
|
826
|
+
res();
|
|
827
|
+
};
|
|
828
|
+
off = net.onChange((on) => {
|
|
829
|
+
if (on) finish();
|
|
830
|
+
});
|
|
831
|
+
this.stopWaiters.add(finish);
|
|
832
|
+
});
|
|
833
|
+
if (!this.stopped) this.setStatus("running");
|
|
834
|
+
}
|
|
835
|
+
/** Lazily starts the sha256 lane and remembers the in-flight promise so a
|
|
836
|
+
* fast pause→resume can never spawn a second concurrent lane over the
|
|
837
|
+
* same file. The promise carries its own no-op .catch so it can never
|
|
838
|
+
* become an unhandledRejection while nothing is actively awaiting it
|
|
839
|
+
* (e.g. during a paused window); a real failure still surfaces normally
|
|
840
|
+
* to whoever does await it, and from there through run() to fail(). */
|
|
841
|
+
getHashLane() {
|
|
842
|
+
if (this.opts.checksum !== "sha256" || this.session.sha256) return Promise.resolve(this.session.sha256);
|
|
843
|
+
if (!this.hashLanePromise) {
|
|
844
|
+
const p = this.hashLane().finally(() => {
|
|
845
|
+
if (this.hashLanePromise === p) this.hashLanePromise = null;
|
|
846
|
+
});
|
|
847
|
+
this.hashLanePromise = p;
|
|
848
|
+
this.hashLanePromise.catch(() => {
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
return this.hashLanePromise;
|
|
852
|
+
}
|
|
853
|
+
async hashLane() {
|
|
854
|
+
const CHUNK = 1 << 20;
|
|
855
|
+
const h = new Sha256();
|
|
856
|
+
let off = 0;
|
|
857
|
+
if (this.session.hash) {
|
|
858
|
+
h.importState(this.session.hash.state);
|
|
859
|
+
off = this.session.hash.offset;
|
|
860
|
+
}
|
|
861
|
+
while (off < this.deps.file.size) {
|
|
862
|
+
if (this.stopped) {
|
|
863
|
+
this.hashLanePromise = null;
|
|
864
|
+
if (this.session.status === "aborted") return void 0;
|
|
865
|
+
this.session.hash = { offset: off, state: h.exportState() };
|
|
866
|
+
await saveSession(this.deps.store, this.session);
|
|
867
|
+
return void 0;
|
|
868
|
+
}
|
|
869
|
+
const b = await this.deps.file.read(off, Math.min(CHUNK, this.deps.file.size - off));
|
|
870
|
+
h.update(b);
|
|
871
|
+
off += b.length;
|
|
872
|
+
if (off % (8 * CHUNK) === 0) this.session.hash = { offset: off, state: h.exportState() };
|
|
873
|
+
}
|
|
874
|
+
this.session.hash = void 0;
|
|
875
|
+
return h.digestHex();
|
|
876
|
+
}
|
|
877
|
+
/** Feed every part-PUT outcome to the systemic-transport-failure detector.
|
|
878
|
+
* Any outcome that isn't network-class — success, 403, 5xx, even corrupt
|
|
879
|
+
* bytes — proves a request reached the server, so the streak resets. */
|
|
880
|
+
notePutOutcome(cls) {
|
|
881
|
+
if (cls !== "network") {
|
|
882
|
+
this.consecutiveNetworkFailures = 0;
|
|
883
|
+
this.diagnosticFired = false;
|
|
884
|
+
return;
|
|
885
|
+
}
|
|
886
|
+
this.consecutiveNetworkFailures++;
|
|
887
|
+
if (this.diagnosticFired) return;
|
|
888
|
+
if (this.consecutiveNetworkFailures < SYSTEMIC_TRANSPORT_FAILURE_THRESHOLD) return;
|
|
889
|
+
if (this.bytesConfirmed() > 0) return;
|
|
890
|
+
this.diagnosticFired = true;
|
|
891
|
+
this.emitter.emit("diagnostic", {
|
|
892
|
+
code: "systemic-transport-failure",
|
|
893
|
+
consecutiveNetworkFailures: this.consecutiveNetworkFailures,
|
|
894
|
+
bytesEverConfirmed: this.bytesConfirmed(),
|
|
895
|
+
hint: 'every part PUT is failing at the network layer and no bytes have ever been confirmed for this session \u2014 if this is a browser, check the source bucket CORS rules (preflight PUT from this origin, ExposeHeaders ETag); see README "Bucket CORS is a hard prerequisite for browser uploads"'
|
|
896
|
+
});
|
|
897
|
+
}
|
|
898
|
+
bytesConfirmed() {
|
|
899
|
+
let sum = 0;
|
|
900
|
+
for (const n of Object.keys(this.session.done)) sum += partSizeOf(this.session, Number(n));
|
|
901
|
+
return sum;
|
|
902
|
+
}
|
|
903
|
+
onPartProgress(n, sent) {
|
|
904
|
+
this.inflightSent.set(n, sent);
|
|
905
|
+
this.emitProgress(false);
|
|
906
|
+
}
|
|
907
|
+
emitProgress(force) {
|
|
908
|
+
const now = this.deps.clock.now();
|
|
909
|
+
if (!force && now - this.lastEmit < 200) return;
|
|
910
|
+
this.lastEmit = now;
|
|
911
|
+
let inflight = 0;
|
|
912
|
+
for (const v of this.inflightSent.values()) inflight += v;
|
|
913
|
+
const partStates = {};
|
|
914
|
+
for (const n of this.claimed) partStates[n] = "inflight";
|
|
915
|
+
for (const n of Object.keys(this.session.done)) partStates[Number(n)] = "done";
|
|
916
|
+
this.emitter.emit("progress", computeProgress({
|
|
917
|
+
confirmed: this.bytesConfirmed(),
|
|
918
|
+
inflight,
|
|
919
|
+
total: this.session.declaredSize,
|
|
920
|
+
partsDone: Object.keys(this.session.done).length,
|
|
921
|
+
partsTotal: this.session.expectedParts,
|
|
922
|
+
startedAtMs: this.startedAt,
|
|
923
|
+
nowMs: now,
|
|
924
|
+
confirmedAtStart: this.confirmedAtStart,
|
|
925
|
+
partStates
|
|
926
|
+
}));
|
|
927
|
+
}
|
|
928
|
+
};
|
|
929
|
+
|
|
930
|
+
// src/core/upload.ts
|
|
931
|
+
var realClock = { now: () => Date.now(), sleep: (ms) => new Promise((res) => setTimeout(res, ms)) };
|
|
932
|
+
async function buildUpload(opts) {
|
|
933
|
+
const clock = opts.clock ?? realClock;
|
|
934
|
+
const fetchFn = opts.fetchFn ?? fetch;
|
|
935
|
+
const api = new HostrApi(opts.endpoint, opts.uploadToken, fetchFn);
|
|
936
|
+
let session = await loadSession(opts.store, opts.file.id);
|
|
937
|
+
if (session) {
|
|
938
|
+
session.uploadToken = opts.uploadToken;
|
|
939
|
+
await saveSession(opts.store, session);
|
|
940
|
+
} else {
|
|
941
|
+
const remote = await api.getUpload(opts.assetId);
|
|
942
|
+
session = {
|
|
943
|
+
v: 1,
|
|
944
|
+
assetId: opts.assetId,
|
|
945
|
+
endpoint: opts.endpoint,
|
|
946
|
+
uploadToken: opts.uploadToken,
|
|
947
|
+
key: remote.key,
|
|
948
|
+
partSize: remote.partSize,
|
|
949
|
+
expectedParts: remote.expectedParts,
|
|
950
|
+
declaredSize: opts.file.size,
|
|
951
|
+
fileId: opts.file.id,
|
|
952
|
+
done: {},
|
|
953
|
+
status: "idle",
|
|
954
|
+
createdAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
955
|
+
};
|
|
956
|
+
await saveSession(opts.store, session);
|
|
957
|
+
}
|
|
958
|
+
const deps = {
|
|
959
|
+
api,
|
|
960
|
+
file: opts.file,
|
|
961
|
+
transport: opts.transport,
|
|
962
|
+
store: opts.store,
|
|
963
|
+
hasher: opts.hasher,
|
|
964
|
+
clock,
|
|
965
|
+
network: opts.network
|
|
966
|
+
};
|
|
967
|
+
const engineOpts = {
|
|
968
|
+
concurrency: opts.concurrency ?? 4,
|
|
969
|
+
checksum: opts.checksum ?? "sha256",
|
|
970
|
+
presignBatch: 16,
|
|
971
|
+
presignTtlSec: opts.presignTtlSec ?? 3600,
|
|
972
|
+
onTokenExpired: opts.onTokenExpired
|
|
973
|
+
};
|
|
974
|
+
return new UploadEngine(deps, engineOpts, session);
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
// src/node/hasher.ts
|
|
978
|
+
import { createHash } from "node:crypto";
|
|
979
|
+
var NodeMd5 = class {
|
|
980
|
+
#hash = createHash("md5");
|
|
981
|
+
#digest = null;
|
|
982
|
+
update(data) {
|
|
983
|
+
if (!this.#hash) throw new Error("NodeMd5: cannot update after digest() has been called");
|
|
984
|
+
this.#hash.update(data);
|
|
985
|
+
}
|
|
986
|
+
digestBytes() {
|
|
987
|
+
if (this.#digest) return this.#digest;
|
|
988
|
+
if (!this.#hash) throw new Error("NodeMd5: cannot digest twice");
|
|
989
|
+
this.#digest = new Uint8Array(this.#hash.digest());
|
|
990
|
+
this.#hash = null;
|
|
991
|
+
return this.#digest;
|
|
992
|
+
}
|
|
993
|
+
digestHex() {
|
|
994
|
+
return Buffer.from(this.digestBytes()).toString("hex");
|
|
995
|
+
}
|
|
996
|
+
};
|
|
997
|
+
var nodeHasher = { md5: () => new NodeMd5() };
|
|
998
|
+
|
|
999
|
+
// src/node/index.ts
|
|
1000
|
+
async function nodeFileSource(path) {
|
|
1001
|
+
const handle = await open(path, "r");
|
|
1002
|
+
const stat = await handle.stat();
|
|
1003
|
+
const size = stat.size;
|
|
1004
|
+
const id = `${path}:${size}:${Math.floor(stat.mtimeMs)}`;
|
|
1005
|
+
return {
|
|
1006
|
+
size,
|
|
1007
|
+
id,
|
|
1008
|
+
async read(offset, length) {
|
|
1009
|
+
const want = Math.max(0, Math.min(length, size - offset));
|
|
1010
|
+
if (want === 0) return new Uint8Array(0);
|
|
1011
|
+
const buf = Buffer.alloc(want);
|
|
1012
|
+
const { bytesRead } = await handle.read(buf, 0, want, offset);
|
|
1013
|
+
return buf.subarray(0, bytesRead);
|
|
1014
|
+
},
|
|
1015
|
+
async close() {
|
|
1016
|
+
await handle.close();
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
var sanitizeKey = (key) => key.replace(/[^A-Za-z0-9._-]/g, "_");
|
|
1021
|
+
function fileStateStore(dir) {
|
|
1022
|
+
const fileFor = (key) => join(dir, sanitizeKey(key));
|
|
1023
|
+
return {
|
|
1024
|
+
async get(key) {
|
|
1025
|
+
try {
|
|
1026
|
+
return await readFile(fileFor(key), "utf8");
|
|
1027
|
+
} catch (err) {
|
|
1028
|
+
if (err?.code === "ENOENT") return null;
|
|
1029
|
+
throw err;
|
|
1030
|
+
}
|
|
1031
|
+
},
|
|
1032
|
+
async set(key, value) {
|
|
1033
|
+
await mkdir(dir, { recursive: true });
|
|
1034
|
+
await writeFile(fileFor(key), value, "utf8");
|
|
1035
|
+
},
|
|
1036
|
+
async delete(key) {
|
|
1037
|
+
try {
|
|
1038
|
+
await unlink(fileFor(key));
|
|
1039
|
+
} catch (err) {
|
|
1040
|
+
if (err?.code !== "ENOENT") throw err;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
function nodeTransport(fetchFn = fetch) {
|
|
1046
|
+
return {
|
|
1047
|
+
async put(url, body, opts) {
|
|
1048
|
+
const res = await fetchFn(url, {
|
|
1049
|
+
method: "PUT",
|
|
1050
|
+
body,
|
|
1051
|
+
signal: opts?.signal
|
|
1052
|
+
});
|
|
1053
|
+
opts?.onProgress?.(body.length);
|
|
1054
|
+
return { status: res.status, etag: res.headers.get("etag") };
|
|
1055
|
+
}
|
|
1056
|
+
};
|
|
1057
|
+
}
|
|
1058
|
+
async function createUpload(opts) {
|
|
1059
|
+
const fetchFn = opts.fetchFn ?? fetch;
|
|
1060
|
+
const file = await nodeFileSource(opts.file);
|
|
1061
|
+
const stateDir = opts.stateDir ?? join(tmpdir(), "hostr-upload");
|
|
1062
|
+
const store = fileStateStore(stateDir);
|
|
1063
|
+
const transport = opts.transport ?? nodeTransport(fetchFn);
|
|
1064
|
+
return buildUpload({
|
|
1065
|
+
endpoint: opts.endpoint,
|
|
1066
|
+
uploadToken: opts.uploadToken,
|
|
1067
|
+
assetId: opts.assetId,
|
|
1068
|
+
file,
|
|
1069
|
+
transport,
|
|
1070
|
+
store,
|
|
1071
|
+
hasher: nodeHasher,
|
|
1072
|
+
fetchFn,
|
|
1073
|
+
concurrency: opts.concurrency,
|
|
1074
|
+
checksum: opts.checksum,
|
|
1075
|
+
onTokenExpired: opts.onTokenExpired
|
|
1076
|
+
});
|
|
1077
|
+
}
|
|
1078
|
+
export {
|
|
1079
|
+
ApiError,
|
|
1080
|
+
IntegrityError,
|
|
1081
|
+
NetworkError,
|
|
1082
|
+
createUpload,
|
|
1083
|
+
createUploadSession,
|
|
1084
|
+
fileStateStore,
|
|
1085
|
+
mintUploadToken,
|
|
1086
|
+
nodeFileSource,
|
|
1087
|
+
nodeHasher,
|
|
1088
|
+
nodeTransport
|
|
1089
|
+
};
|