@uploadfilm/sdk 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -6
- package/dist/copy/types.d.ts +29 -2
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/player.js +1 -1
- package/dist/player.mjs +1 -1
- package/dist/server/media/client.d.ts +34 -1
- package/dist/server/media/protocol.d.ts +242 -4
- package/dist/server/r2/client.d.ts +18 -4
- package/dist/server/storage/chunked.d.ts +6 -2
- package/dist/server/storage/s3.d.ts +51 -0
- package/dist/server/testing/fake-r2.d.ts +22 -2
- package/dist/server/transfer/handler.d.ts +50 -0
- package/dist/server/transfer/protocol.d.ts +53 -4
- package/dist/server-media.js +1 -1
- package/dist/server-media.mjs +1 -1
- package/dist/server-testing.js +1 -1
- package/dist/server-testing.mjs +1 -1
- package/dist/server.js +1 -1
- package/dist/server.mjs +1 -1
- package/dist/transfer/client.d.ts +13 -2
- package/dist/transfer/engine.d.ts +22 -2
- package/dist/transfer/growing/bytes.d.ts +17 -0
- package/dist/transfer/growing/index.d.ts +12 -0
- package/dist/transfer/growing/parts.d.ts +157 -0
- package/dist/transfer/growing/retry.d.ts +38 -0
- package/dist/transfer/growing/store.d.ts +66 -0
- package/dist/transfer/growing/upload.d.ts +126 -0
- package/dist/transfer/index.d.ts +1 -0
- package/dist/transfer/protocol.d.ts +5 -0
- package/dist/transfer/storage.d.ts +28 -3
- package/dist/workers/cli.mjs +115 -0
- package/dist/workers/media/config.mjs +53 -0
- package/dist/workers/media/container/Dockerfile +19 -0
- package/dist/workers/media/container/FFMPEG.md +129 -0
- package/dist/workers/media/container/runner.mjs +2690 -0
- package/dist/workers/media/index.mjs +1011 -0
- package/dist/workers/media/init/config.ts +52 -0
- package/dist/workers/media/init/worker.ts +8 -0
- package/dist/workers/media/init/wrangler.toml +44 -0
- package/dist/workers/media/runner.mjs +2656 -0
- package/dist/workers/media/types/build/entry/config.d.mts +6 -0
- package/dist/workers/media/types/build/entry/index.d.mts +43 -0
- package/dist/workers/media/types/build/entry/program.d.mts +13 -0
- package/dist/workers/media/types/build/entry/runner.d.mts +12 -0
- package/dist/workers/media/types/build/entry/settings.d.mts +40 -0
- package/dist/workers/media/types/src/config.d.mts +135 -0
- package/dist/workers/media/types/src/jobs.d.mts +73 -0
- package/dist/workers/media/types/src/runner/container.d.mts +32 -0
- package/dist/workers/media/types/src/runner/ffmpeg.d.mts +227 -0
- package/dist/workers/media/types/src/runner/job.d.mts +107 -0
- package/dist/workers/media/types/src/runner/node.d.mts +71 -0
- package/dist/workers/media/types/src/runner/plugins.d.mts +45 -0
- package/dist/workers/media/types/src/serve.d.mts +107 -0
- package/dist/workers/media/types/src/steps/audio.d.mts +33 -0
- package/dist/workers/media/types/src/steps/edit.d.mts +62 -0
- package/dist/workers/media/types/src/steps/frameCheck.d.mts +44 -0
- package/dist/workers/media/types/src/steps/gate.d.mts +18 -0
- package/dist/workers/media/types/src/steps/paths.d.mts +1 -0
- package/dist/workers/media/types/src/steps/prep.d.mts +60 -0
- package/dist/workers/media/types/src/steps/render.d.mts +40 -0
- package/dist/workers/media/types/src/steps/runner.d.mts +30 -0
- package/dist/workers/media/types/src/steps/transcribe.d.mts +89 -0
- package/dist/workers/media/types/src/storyboard/index.d.mts +61 -0
- package/dist/workers/media/types/src/transcoder.d.mts +30 -0
- package/dist/workers/media/types/src/worker.d.mts +30 -0
- package/dist/workers/zip/index.mjs +589 -0
- package/dist/workers/zip/init/config.ts +22 -0
- package/dist/workers/zip/init/worker.ts +5 -0
- package/dist/workers/zip/init/wrangler.toml +25 -0
- package/dist/workers/zip/types/build/entry/index.d.mts +5 -0
- package/dist/workers/zip/types/src/config.d.mts +35 -0
- package/dist/workers/zip/types/src/serve.d.mts +86 -0
- package/dist/workers/zip/types/src/worker.d.mts +12 -0
- package/package.json +51 -3
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
// ../zip-worker/build/out/stage/src/serve.ts
|
|
2
|
+
import { crc32 } from "node:zlib";
|
|
3
|
+
import { LayoutError, ZipLayout, archiveEtag } from "@uploadfilm/sdk";
|
|
4
|
+
import { MAX_PLAN_BYTES, PLAN_ID, parseGen, planServable, normEtag, parsePlan, planIdOf } from "@uploadfilm/sdk";
|
|
5
|
+
import { verify } from "@uploadfilm/sdk/server";
|
|
6
|
+
import { crcHex } from "@uploadfilm/sdk/server";
|
|
7
|
+
|
|
8
|
+
// ../zip-worker/build/out/stage/src/config.ts
|
|
9
|
+
function pagesOf(cfg) {
|
|
10
|
+
return {
|
|
11
|
+
back: cfg.pages?.back ?? "Go back to the page you downloaded from and press Download again for a fresh link.",
|
|
12
|
+
tooLarge: cfg.pages?.tooLarge ?? "Download it in parts (folder by folder)."
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
// ../zip-worker/build/out/stage/src/serve.ts
|
|
17
|
+
var READ_TIMEOUT_MS = 5e3;
|
|
18
|
+
var COLD_WAIT_MS = 15e3;
|
|
19
|
+
var Timeout = class extends Error {
|
|
20
|
+
};
|
|
21
|
+
function within(p, ms, what) {
|
|
22
|
+
let t;
|
|
23
|
+
return Promise.race([p, new Promise((_, bad) => t = setTimeout(() => bad(new Timeout(`${what}: no answer in ${ms} ms`)), ms))]).finally(() => clearTimeout(t));
|
|
24
|
+
}
|
|
25
|
+
var MAX_CRC_RANGE = 4 * 1024 ** 3;
|
|
26
|
+
var CRC_BODY_MAX = 2048;
|
|
27
|
+
async function readCapped(req, max) {
|
|
28
|
+
const cl = req.headers.get("content-length");
|
|
29
|
+
if (cl !== null && !(Number(cl) <= max)) return null;
|
|
30
|
+
if (!req.body) return "";
|
|
31
|
+
const rd = req.body.getReader();
|
|
32
|
+
const parts = [];
|
|
33
|
+
let n = 0;
|
|
34
|
+
for (; ; ) {
|
|
35
|
+
const { done, value } = await rd.read();
|
|
36
|
+
if (done) break;
|
|
37
|
+
n += value.length;
|
|
38
|
+
if (n > max) {
|
|
39
|
+
await rd.cancel().catch(() => {
|
|
40
|
+
});
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
parts.push(value);
|
|
44
|
+
}
|
|
45
|
+
const all = new Uint8Array(n);
|
|
46
|
+
let o = 0;
|
|
47
|
+
for (const p of parts) all.set(p, o), o += p.length;
|
|
48
|
+
return new TextDecoder().decode(all);
|
|
49
|
+
}
|
|
50
|
+
var PinError = class extends Error {
|
|
51
|
+
status;
|
|
52
|
+
key;
|
|
53
|
+
constructor(status, key) {
|
|
54
|
+
super(status === 410 ? `gone: ${key}` : `changed: ${key}`);
|
|
55
|
+
this.status = status;
|
|
56
|
+
this.key = key;
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
var CACHE_BYTES = 32 * 1024 * 1024;
|
|
60
|
+
function remember(z, id, l) {
|
|
61
|
+
const cache = z.cache;
|
|
62
|
+
cache.delete(id);
|
|
63
|
+
cache.set(id, l);
|
|
64
|
+
let total = 0;
|
|
65
|
+
for (const v of cache.values()) total += v.bytes;
|
|
66
|
+
for (const k of cache.keys()) {
|
|
67
|
+
if (total <= CACHE_BYTES || cache.size <= 1) break;
|
|
68
|
+
total -= cache.get(k).bytes;
|
|
69
|
+
cache.delete(k);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
var page = (status, title, text, extra = {}) => new Response(
|
|
73
|
+
`<!doctype html><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>${title}</title><body style="font:16px/1.5 system-ui;max-width:36em;margin:4em auto;padding:0 1em"><h1 style="font-size:1.3em">${title}</h1><p>${text}</p>`,
|
|
74
|
+
{ status, headers: { "content-type": "text/html; charset=utf-8", "cache-control": "no-store", "x-robots-tag": "noindex", ...extra } }
|
|
75
|
+
);
|
|
76
|
+
function disposition(name) {
|
|
77
|
+
const ascii = name.replace(/[^\x20-\x7e]/g, "_").replace(/["\\]/g, "_");
|
|
78
|
+
return `attachment; filename="${ascii}"; filename*=UTF-8''${encodeURIComponent(name).replace(/['()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`)}`;
|
|
79
|
+
}
|
|
80
|
+
function parseRange(h, length) {
|
|
81
|
+
if (!h) return null;
|
|
82
|
+
const m = /^\s*bytes\s*=\s*(\d*)\s*-\s*(\d*)\s*$/i.exec(h);
|
|
83
|
+
if (!m) return null;
|
|
84
|
+
const [, a, b] = m;
|
|
85
|
+
if (a === "" && b === "") return null;
|
|
86
|
+
if (a === "") {
|
|
87
|
+
const n = Number(b);
|
|
88
|
+
if (!Number.isSafeInteger(n)) return null;
|
|
89
|
+
if (n === 0) return "unsatisfiable";
|
|
90
|
+
return [Math.max(0, length - n), length];
|
|
91
|
+
}
|
|
92
|
+
const start = Number(a);
|
|
93
|
+
if (!Number.isSafeInteger(start)) return null;
|
|
94
|
+
if (start >= length) return "unsatisfiable";
|
|
95
|
+
let end = b === "" ? length - 1 : Number(b);
|
|
96
|
+
if (!Number.isSafeInteger(end)) end = length - 1;
|
|
97
|
+
if (end < start) return null;
|
|
98
|
+
return [start, Math.min(end, length - 1) + 1];
|
|
99
|
+
}
|
|
100
|
+
var listHas = (h, etag, weakOk) => h.trim() === "*" || h.split(",").some((t) => {
|
|
101
|
+
const s = t.trim();
|
|
102
|
+
return s === etag || weakOk && s === `W/${etag}`;
|
|
103
|
+
});
|
|
104
|
+
async function planCurrent(z, env, share, gen, planId, retries = 3, timeoutMs = READ_TIMEOUT_MS) {
|
|
105
|
+
for (let attempt = 0; ; attempt++) {
|
|
106
|
+
try {
|
|
107
|
+
const text = await within(
|
|
108
|
+
(async () => {
|
|
109
|
+
const o = await env.BUCKET.get((env.KEY_PREFIX ?? "") + z.cfg.keys.genKey(share));
|
|
110
|
+
return o ? new TextDecoder().decode(await o.arrayBuffer()) : null;
|
|
111
|
+
})(),
|
|
112
|
+
timeoutMs,
|
|
113
|
+
"generation marker"
|
|
114
|
+
);
|
|
115
|
+
return planServable(parseGen(text), gen, planId);
|
|
116
|
+
} catch (e) {
|
|
117
|
+
if (attempt >= retries) throw new StorageDown(String(e?.message ?? e));
|
|
118
|
+
await new Promise((r) => setTimeout(r, 200 * 2 ** attempt));
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
var StorageDown = class extends Error {
|
|
123
|
+
};
|
|
124
|
+
async function loadPlan(z, env, share, id, opts = {}) {
|
|
125
|
+
const hit = z.cache.get(id);
|
|
126
|
+
if (hit && hit.share === share) {
|
|
127
|
+
if (!await planCurrent(z, env, share, hit.gen, id, opts.getRetries ?? 3, opts.readTimeoutMs)) throw new PinError(410, z.cfg.keys.planKey(share, id));
|
|
128
|
+
remember(z, id, hit);
|
|
129
|
+
return hit;
|
|
130
|
+
}
|
|
131
|
+
const key = `${share}/${id}`;
|
|
132
|
+
let p = z.loading.get(key);
|
|
133
|
+
if (!p) {
|
|
134
|
+
const turn = within(z.coldQueue, COLD_WAIT_MS, "queue").catch(() => {
|
|
135
|
+
}).then(() => coldLoad(z, env, share, id, opts));
|
|
136
|
+
z.coldQueue = turn.then(
|
|
137
|
+
() => {
|
|
138
|
+
},
|
|
139
|
+
() => {
|
|
140
|
+
}
|
|
141
|
+
);
|
|
142
|
+
p = turn.finally(() => z.loading.delete(key));
|
|
143
|
+
z.loading.set(key, p);
|
|
144
|
+
}
|
|
145
|
+
return p;
|
|
146
|
+
}
|
|
147
|
+
async function coldLoad(z, env, share, id, opts = {}) {
|
|
148
|
+
const planKey = (sh, pid) => z.cfg.keys.planKey(sh, pid);
|
|
149
|
+
const tmo = opts.readTimeoutMs ?? READ_TIMEOUT_MS;
|
|
150
|
+
const o = await within(env.BUCKET.get((env.KEY_PREFIX ?? "") + planKey(share, id)), tmo, "plan").catch((e) => {
|
|
151
|
+
throw e instanceof Timeout ? new StorageDown(e.message) : e;
|
|
152
|
+
});
|
|
153
|
+
if (!o || !o.arrayBuffer) throw new PinError(410, planKey(share, id));
|
|
154
|
+
if (o.size > MAX_PLAN_BYTES) throw new PlanTooBig(String(o.size));
|
|
155
|
+
let bytes = new Uint8Array(
|
|
156
|
+
await within(o.arrayBuffer(), tmo * 4, "plan body").catch((e) => {
|
|
157
|
+
throw e instanceof Timeout ? new StorageDown(e.message) : e;
|
|
158
|
+
})
|
|
159
|
+
);
|
|
160
|
+
if (await planIdOf(bytes) !== id) throw new PinError(410, planKey(share, id));
|
|
161
|
+
let text = new TextDecoder().decode(bytes);
|
|
162
|
+
bytes = null;
|
|
163
|
+
let parsed;
|
|
164
|
+
try {
|
|
165
|
+
parsed = parsePlan(text, share, z.cfg.keys.entryPrefix(share), (k) => z.cfg.keyAllowed(share, k));
|
|
166
|
+
} catch {
|
|
167
|
+
throw new PinError(410, planKey(share, id));
|
|
168
|
+
}
|
|
169
|
+
text = null;
|
|
170
|
+
if (!await planCurrent(z, env, share, parsed.gen, id, opts.getRetries ?? 3, opts.readTimeoutMs)) throw new PinError(410, planKey(share, id));
|
|
171
|
+
let layout;
|
|
172
|
+
try {
|
|
173
|
+
layout = new ZipLayout(parsed.source);
|
|
174
|
+
} catch (e) {
|
|
175
|
+
if (e instanceof LayoutError && /names total/.test(e.message)) throw new PlanTooBig(e.message);
|
|
176
|
+
throw e;
|
|
177
|
+
}
|
|
178
|
+
const { keys, etags } = parsed;
|
|
179
|
+
let strBytes = 0;
|
|
180
|
+
for (let i = 0; i < keys.length; i++) strBytes += keys[i].length + etags[i].length + 40;
|
|
181
|
+
const l = { share, name: parsed.name, created: parsed.created, gen: parsed.gen, keyPrefix: parsed.keyPrefix, keys, etags, layout, etag: archiveEtag(id), bytes: layout.retainedBytes + strBytes };
|
|
182
|
+
remember(z, id, l);
|
|
183
|
+
return l;
|
|
184
|
+
}
|
|
185
|
+
var PlanTooBig = class extends Error {
|
|
186
|
+
};
|
|
187
|
+
function createZipHandler(cfg) {
|
|
188
|
+
const shareId = new RegExp(cfg.shareId.source, cfg.shareId.flags);
|
|
189
|
+
const shareIdOk = (id) => (shareId.lastIndex = 0, shareId.test(id));
|
|
190
|
+
const z = { cfg, shareIdOk, pages: pagesOf(cfg), cache: /* @__PURE__ */ new Map(), loading: /* @__PURE__ */ new Map(), coldQueue: Promise.resolve() };
|
|
191
|
+
return {
|
|
192
|
+
handle: (req, env, opts = {}) => handle(z, req, env, opts),
|
|
193
|
+
clearPlanCache: () => (z.cache.clear(), z.loading.clear()),
|
|
194
|
+
planCacheBytes: () => [...z.cache.values()].reduce((a, v) => a + v.bytes, 0)
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
async function handle(z, req, env, opts = {}) {
|
|
198
|
+
const BACK = z.pages.back;
|
|
199
|
+
const url = new URL(req.url);
|
|
200
|
+
if (url.pathname === "/crc") return req.method === "POST" ? crcRange(z, req, env, opts) : new Response("POST only", { status: 405 });
|
|
201
|
+
const m = /^\/z\/([^/]+)\/([^/]+)\/[^/]+$/.exec(url.pathname);
|
|
202
|
+
if (!m) return new Response("Not found", { status: 404 });
|
|
203
|
+
if (req.method !== "GET" && req.method !== "HEAD") return new Response("GET or HEAD", { status: 405, headers: { allow: "GET, HEAD" } });
|
|
204
|
+
const [, share, planId] = m;
|
|
205
|
+
const e = Number(url.searchParams.get("e"));
|
|
206
|
+
const s = url.searchParams.get("s") ?? "";
|
|
207
|
+
if (!z.shareIdOk(share) || !PLAN_ID.test(planId) || !Number.isSafeInteger(e)) return new Response("Not found", { status: 404 });
|
|
208
|
+
if (!await verify(env.ZIP_SIGNING_KEY, z.cfg.domains.link, [share, planId, e], s)) return page(403, "This download link isn't valid", BACK);
|
|
209
|
+
if ((opts.now?.() ?? Date.now()) / 1e3 > e) return page(403, "This download link has expired", BACK);
|
|
210
|
+
let l;
|
|
211
|
+
try {
|
|
212
|
+
l = await loadPlan(z, env, share, planId, opts);
|
|
213
|
+
} catch (err) {
|
|
214
|
+
if (err instanceof PinError) return page(410, "This download is no longer available", BACK);
|
|
215
|
+
if (err instanceof StorageDown) return page(502, "Storage didn't answer", "Try again in a minute (your browser can resume the download).");
|
|
216
|
+
if (err instanceof PlanTooBig) return page(413, "This download is too large to make as one zip", z.pages.tooLarge);
|
|
217
|
+
return page(500, "This download link is broken", BACK);
|
|
218
|
+
}
|
|
219
|
+
const { layout, etag } = l;
|
|
220
|
+
const common = {
|
|
221
|
+
"accept-ranges": "bytes",
|
|
222
|
+
etag,
|
|
223
|
+
"last-modified": new Date(l.created * 1e3).toUTCString(),
|
|
224
|
+
"cache-control": "private, no-transform",
|
|
225
|
+
"x-robots-tag": "noindex"
|
|
226
|
+
};
|
|
227
|
+
const ifMatch = req.headers.get("if-match");
|
|
228
|
+
if (ifMatch && !listHas(ifMatch, etag, false)) return new Response(null, { status: 412, headers: common });
|
|
229
|
+
const inm = req.headers.get("if-none-match");
|
|
230
|
+
if (inm && listHas(inm, etag, true)) return new Response(null, { status: 304, headers: common });
|
|
231
|
+
const ifRange = req.headers.get("if-range");
|
|
232
|
+
const rangeAllowed = !ifRange || ifRange.trim() === etag;
|
|
233
|
+
const r = rangeAllowed ? parseRange(req.headers.get("range"), layout.length) : null;
|
|
234
|
+
if (r === "unsatisfiable") return new Response(null, { status: 416, headers: { ...common, "content-range": `bytes */${layout.length}` } });
|
|
235
|
+
const [start, end] = r ?? [0, layout.length];
|
|
236
|
+
const headers = {
|
|
237
|
+
...common,
|
|
238
|
+
"content-type": "application/zip",
|
|
239
|
+
"content-disposition": disposition(l.name),
|
|
240
|
+
"content-length": String(end - start),
|
|
241
|
+
...r ? { "content-range": `bytes ${start}-${end - 1}/${layout.length}` } : {}
|
|
242
|
+
};
|
|
243
|
+
const status = r ? 206 : 200;
|
|
244
|
+
if (req.method === "HEAD") return new Response(null, { status, headers });
|
|
245
|
+
if (end === start) return new Response(new Uint8Array(0), { status, headers });
|
|
246
|
+
const pump = new Pump(z, env, l, planId, layout.pieces(start, end), opts);
|
|
247
|
+
try {
|
|
248
|
+
await pump.prime();
|
|
249
|
+
} catch (err) {
|
|
250
|
+
pump.cancel();
|
|
251
|
+
if (err instanceof PinError) {
|
|
252
|
+
return err.status === 412 ? page(412, "A file in this download changed", `Someone replaced a file after this link was made, so this download can't continue without mixing versions. ${BACK}`, { etag }) : page(410, "A file in this download was removed", BACK, { etag });
|
|
253
|
+
}
|
|
254
|
+
return page(502, "Storage didn't answer", "Try again in a minute (your browser can resume the download).");
|
|
255
|
+
}
|
|
256
|
+
const { readable, writable } = (opts.fixedLength ?? ((n) => new TransformStream()))(end - start);
|
|
257
|
+
pump.run(writable).catch(() => {
|
|
258
|
+
});
|
|
259
|
+
return new Response(readable, { status, headers });
|
|
260
|
+
}
|
|
261
|
+
var Pump = class {
|
|
262
|
+
q = [];
|
|
263
|
+
done = false;
|
|
264
|
+
inflight = 0;
|
|
265
|
+
buffered = 0;
|
|
266
|
+
cancelled = false;
|
|
267
|
+
aheadReads;
|
|
268
|
+
aheadBytes;
|
|
269
|
+
small;
|
|
270
|
+
retries;
|
|
271
|
+
z;
|
|
272
|
+
env;
|
|
273
|
+
l;
|
|
274
|
+
it;
|
|
275
|
+
planId;
|
|
276
|
+
planCheckMs;
|
|
277
|
+
bigPiece;
|
|
278
|
+
freshMs;
|
|
279
|
+
/** When the plan was last found current (the request's own check counts). */
|
|
280
|
+
checkedAt = Date.now();
|
|
281
|
+
/** The entry whose big piece the last pre-start check was made for (-1: none; the 30 s timer doesn't count). */
|
|
282
|
+
checkedIndex = -1;
|
|
283
|
+
checkRetries;
|
|
284
|
+
readTimeoutMs;
|
|
285
|
+
constructor(z, env, l, planId, it, opts) {
|
|
286
|
+
this.z = z;
|
|
287
|
+
this.env = env;
|
|
288
|
+
this.l = l;
|
|
289
|
+
this.planId = planId;
|
|
290
|
+
this.planCheckMs = opts.planCheckMs ?? 3e4;
|
|
291
|
+
this.bigPiece = opts.bigPiece ?? 64 * 1024 * 1024;
|
|
292
|
+
this.freshMs = opts.freshMs ?? 1e3;
|
|
293
|
+
this.checkRetries = opts.getRetries ?? 3;
|
|
294
|
+
this.readTimeoutMs = opts.readTimeoutMs ?? READ_TIMEOUT_MS;
|
|
295
|
+
this.it = it;
|
|
296
|
+
this.aheadReads = opts.aheadReads ?? 5;
|
|
297
|
+
this.aheadBytes = opts.aheadBytes ?? 6 * 1024 * 1024;
|
|
298
|
+
this.small = opts.smallPiece ?? 1024 * 1024;
|
|
299
|
+
this.retries = opts.getRetries ?? 3;
|
|
300
|
+
}
|
|
301
|
+
cancel() {
|
|
302
|
+
this.cancelled = true;
|
|
303
|
+
}
|
|
304
|
+
async get(p) {
|
|
305
|
+
const key = this.l.keyPrefix + this.l.keys[p.index];
|
|
306
|
+
const etag = this.l.etags[p.index];
|
|
307
|
+
for (let attempt = 0; ; attempt++) {
|
|
308
|
+
let o;
|
|
309
|
+
try {
|
|
310
|
+
o = await within(this.env.BUCKET.get((this.env.KEY_PREFIX ?? "") + key, { range: { offset: p.from, length: p.to - p.from }, onlyIf: { etagMatches: etag } }), this.readTimeoutMs, key);
|
|
311
|
+
} catch (e) {
|
|
312
|
+
if (attempt < this.retries && !this.cancelled) {
|
|
313
|
+
await new Promise((r) => setTimeout(r, 200 * 2 ** attempt));
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
throw e;
|
|
317
|
+
}
|
|
318
|
+
if (!o) throw new PinError(410, key);
|
|
319
|
+
if (!o.body || normEtag(o.etag) !== etag) throw new PinError(412, key);
|
|
320
|
+
return o;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
/** An empty entry's pin: the object must exist, with the plan's ETag, and still be empty (410 / 412 otherwise). */
|
|
324
|
+
async checkEmpty(index) {
|
|
325
|
+
const key = this.l.keyPrefix + this.l.keys[index];
|
|
326
|
+
const etag = this.l.etags[index];
|
|
327
|
+
for (let attempt = 0; ; attempt++) {
|
|
328
|
+
let o;
|
|
329
|
+
try {
|
|
330
|
+
o = await within(this.env.BUCKET.head((this.env.KEY_PREFIX ?? "") + key), this.readTimeoutMs, key);
|
|
331
|
+
} catch (e) {
|
|
332
|
+
if (attempt < this.retries && !this.cancelled) {
|
|
333
|
+
await new Promise((r) => setTimeout(r, 200 * 2 ** attempt));
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
throw e;
|
|
337
|
+
}
|
|
338
|
+
if (!o) throw new PinError(410, key);
|
|
339
|
+
if (normEtag(o.etag) !== etag || o.size !== 0) throw new PinError(412, key);
|
|
340
|
+
return new Uint8Array(0);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
async readSmall(p) {
|
|
344
|
+
const o = await this.get(p);
|
|
345
|
+
const rd = o.body.getReader();
|
|
346
|
+
const want = p.to - p.from;
|
|
347
|
+
const readAll = async () => {
|
|
348
|
+
const out = new Uint8Array(want);
|
|
349
|
+
let n = 0;
|
|
350
|
+
for (; ; ) {
|
|
351
|
+
const { done, value } = await rd.read();
|
|
352
|
+
if (done) break;
|
|
353
|
+
if (n + value.length > want) throw new Error(`long read of ${this.l.keyPrefix + this.l.keys[p.index]}`);
|
|
354
|
+
out.set(value, n);
|
|
355
|
+
n += value.length;
|
|
356
|
+
}
|
|
357
|
+
return out.subarray(0, n);
|
|
358
|
+
};
|
|
359
|
+
const b = await within(readAll(), this.readTimeoutMs, this.l.keyPrefix + this.l.keys[p.index]).catch((e) => {
|
|
360
|
+
rd.cancel(e).catch(() => {
|
|
361
|
+
});
|
|
362
|
+
throw e;
|
|
363
|
+
});
|
|
364
|
+
if (b.length !== p.to - p.from) throw new Error(`short read of ${this.l.keyPrefix + this.l.keys[p.index]}`);
|
|
365
|
+
return b;
|
|
366
|
+
}
|
|
367
|
+
/** Queue pieces while the read-ahead budget allows. */
|
|
368
|
+
fill() {
|
|
369
|
+
while (!this.done && this.q.length < 256) {
|
|
370
|
+
const peekJob = this.next();
|
|
371
|
+
if (!peekJob) break;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
pending = null;
|
|
375
|
+
/** Big files whose body is open (requested, not yet piped through). */
|
|
376
|
+
largeOpen = 0;
|
|
377
|
+
/** One connection held for a big file: open, or queued (it opens while the read-aheads behind it are still open). */
|
|
378
|
+
bigSlot() {
|
|
379
|
+
return this.largeOpen > 0 || this.q.some((j) => j.kind === "large") ? 1 : 0;
|
|
380
|
+
}
|
|
381
|
+
openLarge(p) {
|
|
382
|
+
this.largeOpen++;
|
|
383
|
+
return this.get(p).catch((e) => {
|
|
384
|
+
this.largeOpen--;
|
|
385
|
+
throw e;
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
next() {
|
|
389
|
+
const p = this.pending ?? this.it.next().value;
|
|
390
|
+
this.pending = null;
|
|
391
|
+
if (!p) {
|
|
392
|
+
this.done = true;
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
if (p.kind === "bytes") {
|
|
396
|
+
this.q.push({ kind: "bytes", data: p.data });
|
|
397
|
+
return true;
|
|
398
|
+
}
|
|
399
|
+
if (p.kind === "check") {
|
|
400
|
+
if (this.inflight + this.bigSlot() >= this.aheadReads) {
|
|
401
|
+
this.pending = p;
|
|
402
|
+
return false;
|
|
403
|
+
}
|
|
404
|
+
this.inflight++;
|
|
405
|
+
const buf2 = this.checkEmpty(p.index).finally(() => this.inflight--);
|
|
406
|
+
buf2.catch(() => {
|
|
407
|
+
});
|
|
408
|
+
this.q.push({ kind: "small", buf: buf2 });
|
|
409
|
+
return true;
|
|
410
|
+
}
|
|
411
|
+
const n = p.to - p.from;
|
|
412
|
+
if (n > this.small) {
|
|
413
|
+
if (this.q.some((j) => j.kind !== "bytes")) {
|
|
414
|
+
this.pending = p;
|
|
415
|
+
return false;
|
|
416
|
+
}
|
|
417
|
+
this.q.push({ kind: "large", p });
|
|
418
|
+
return true;
|
|
419
|
+
}
|
|
420
|
+
if (this.inflight + this.bigSlot() >= this.aheadReads || this.buffered + n > this.aheadBytes && this.buffered > 0) {
|
|
421
|
+
this.pending = p;
|
|
422
|
+
return false;
|
|
423
|
+
}
|
|
424
|
+
this.inflight++;
|
|
425
|
+
this.buffered += n;
|
|
426
|
+
const buf = this.readSmall(p).finally(() => this.inflight--);
|
|
427
|
+
buf.catch(() => {
|
|
428
|
+
});
|
|
429
|
+
this.q.push({ kind: "small", buf });
|
|
430
|
+
return true;
|
|
431
|
+
}
|
|
432
|
+
/** Start the first file read and wait for its answer (so a changed/missing file can still be a 412/410). */
|
|
433
|
+
async prime() {
|
|
434
|
+
this.fill();
|
|
435
|
+
const first = this.q.find((j) => j.kind !== "bytes");
|
|
436
|
+
if (!first) return;
|
|
437
|
+
if (first.kind === "small") await first.buf;
|
|
438
|
+
else first.obj = this.openLarge(first.p), await first.obj;
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Re-check the plan every planCheckMs while streaming. Not current → abort; can't tell (storage errors after a
|
|
442
|
+
* bounded retry) → abort too: fail closed, never finish a download whose validity can't be established (R2-B2).
|
|
443
|
+
*/
|
|
444
|
+
watch(writable, stop) {
|
|
445
|
+
const ms = this.planCheckMs;
|
|
446
|
+
let busy = false;
|
|
447
|
+
const t = setInterval(async () => {
|
|
448
|
+
if (busy || this.cancelled || stop.signal.aborted) return;
|
|
449
|
+
busy = true;
|
|
450
|
+
try {
|
|
451
|
+
if (!await planCurrent(this.z, this.env, this.l.share, this.l.gen, this.planId, this.checkRetries, this.readTimeoutMs)) {
|
|
452
|
+
this.cancel();
|
|
453
|
+
stop.abort(new PinError(410, this.z.cfg.keys.planKey(this.l.share, this.planId)));
|
|
454
|
+
} else this.checkedAt = Date.now();
|
|
455
|
+
} catch (e) {
|
|
456
|
+
this.cancel();
|
|
457
|
+
stop.abort(e);
|
|
458
|
+
} finally {
|
|
459
|
+
busy = false;
|
|
460
|
+
}
|
|
461
|
+
}, ms);
|
|
462
|
+
return () => clearInterval(t);
|
|
463
|
+
}
|
|
464
|
+
async run(writable) {
|
|
465
|
+
const stop = new AbortController();
|
|
466
|
+
const unwatch = this.watch(writable, stop);
|
|
467
|
+
let w = writable.getWriter();
|
|
468
|
+
const halted = () => {
|
|
469
|
+
if (stop.signal.aborted) throw stop.signal.reason;
|
|
470
|
+
};
|
|
471
|
+
try {
|
|
472
|
+
for (; ; ) {
|
|
473
|
+
halted();
|
|
474
|
+
this.fill();
|
|
475
|
+
const j = this.q.shift();
|
|
476
|
+
if (!j) break;
|
|
477
|
+
if (j.kind === "bytes") await w.write(j.data);
|
|
478
|
+
else if (j.kind === "small") {
|
|
479
|
+
const b = await j.buf;
|
|
480
|
+
this.buffered -= b.length;
|
|
481
|
+
if (b.length) await w.write(b);
|
|
482
|
+
} else {
|
|
483
|
+
if (j.p.to - j.p.from > this.bigPiece && (j.p.index !== this.checkedIndex || Date.now() - this.checkedAt >= this.freshMs)) {
|
|
484
|
+
if (!await planCurrent(this.z, this.env, this.l.share, this.l.gen, this.planId, this.checkRetries, this.readTimeoutMs)) throw new PinError(410, this.z.cfg.keys.planKey(this.l.share, this.planId));
|
|
485
|
+
this.checkedAt = Date.now();
|
|
486
|
+
this.checkedIndex = j.p.index;
|
|
487
|
+
}
|
|
488
|
+
const o = await (j.obj ?? this.openLarge(j.p));
|
|
489
|
+
w.releaseLock();
|
|
490
|
+
try {
|
|
491
|
+
await o.body.pipeTo(writable, { preventClose: true, preventAbort: true, signal: stop.signal }).catch((e) => {
|
|
492
|
+
w = writable.getWriter();
|
|
493
|
+
throw e;
|
|
494
|
+
});
|
|
495
|
+
} finally {
|
|
496
|
+
this.largeOpen--;
|
|
497
|
+
}
|
|
498
|
+
w = writable.getWriter();
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
halted();
|
|
502
|
+
await w.close();
|
|
503
|
+
} catch (e) {
|
|
504
|
+
this.cancel();
|
|
505
|
+
try {
|
|
506
|
+
await w.abort(e);
|
|
507
|
+
} catch {
|
|
508
|
+
await writable.abort(e).catch(() => {
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
throw e;
|
|
512
|
+
} finally {
|
|
513
|
+
unwatch();
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
};
|
|
517
|
+
async function crcRange(z, req, env, opts) {
|
|
518
|
+
const raw = await readCapped(req, CRC_BODY_MAX);
|
|
519
|
+
if (raw === null) return Response.json({ error: "body too large" }, { status: 413 });
|
|
520
|
+
let b;
|
|
521
|
+
try {
|
|
522
|
+
b = JSON.parse(raw);
|
|
523
|
+
} catch {
|
|
524
|
+
return Response.json({ error: "bad json" }, { status: 400 });
|
|
525
|
+
}
|
|
526
|
+
if (!b || typeof b !== "object" || Array.isArray(b)) return Response.json({ error: "bad request" }, { status: 400 });
|
|
527
|
+
const { key, etag, offset, length, e, s } = b;
|
|
528
|
+
if (typeof key !== "string" || typeof etag !== "string" || !Number.isSafeInteger(offset) || !Number.isSafeInteger(length) || !Number.isSafeInteger(e) || typeof s !== "string")
|
|
529
|
+
return Response.json({ error: "bad request" }, { status: 400 });
|
|
530
|
+
const off = offset;
|
|
531
|
+
const len = length;
|
|
532
|
+
if (off < 0 || len < 0 || len > MAX_CRC_RANGE) return Response.json({ error: "bad range" }, { status: 400 });
|
|
533
|
+
if (!await verify(env.ZIP_SIGNING_KEY, z.cfg.domains.crc, [key, etag, off, len, e], s)) return Response.json({ error: "bad signature" }, { status: 403 });
|
|
534
|
+
if ((opts.now?.() ?? Date.now()) / 1e3 > e) return Response.json({ error: "expired" }, { status: 403 });
|
|
535
|
+
const share = z.cfg.shareOfKey(key);
|
|
536
|
+
const allowed = share ? z.shareIdOk(share) && z.cfg.keyAllowed(share, key) : false;
|
|
537
|
+
if (!allowed) return Response.json({ error: "key not allowed" }, { status: 403 });
|
|
538
|
+
if (len === 0) return Response.json({ crc32: crcHex(0), length: 0 });
|
|
539
|
+
const t0 = Date.now();
|
|
540
|
+
const tmo = opts.readTimeoutMs ?? READ_TIMEOUT_MS;
|
|
541
|
+
let o;
|
|
542
|
+
try {
|
|
543
|
+
o = await within(env.BUCKET.get((env.KEY_PREFIX ?? "") + key, { range: { offset: off, length: len }, onlyIf: { etagMatches: normEtag(etag) } }), tmo, key);
|
|
544
|
+
} catch {
|
|
545
|
+
return Response.json({ error: "storage did not answer" }, { status: 502 });
|
|
546
|
+
}
|
|
547
|
+
if (!o) return Response.json({ error: "gone" }, { status: 404 });
|
|
548
|
+
if (!o.body) return Response.json({ error: "changed", etag: normEtag(o.etag) }, { status: 412 });
|
|
549
|
+
let c = 0;
|
|
550
|
+
let n = 0;
|
|
551
|
+
const rd = o.body.getReader();
|
|
552
|
+
for (; ; ) {
|
|
553
|
+
let r;
|
|
554
|
+
try {
|
|
555
|
+
r = await within(rd.read(), tmo, key);
|
|
556
|
+
} catch {
|
|
557
|
+
rd.cancel().catch(() => {
|
|
558
|
+
});
|
|
559
|
+
return Response.json({ error: `stalled after ${n}/${len}` }, { status: 502 });
|
|
560
|
+
}
|
|
561
|
+
if (r.done) break;
|
|
562
|
+
const value = r.value;
|
|
563
|
+
c = crc32(value, c);
|
|
564
|
+
n += value.length;
|
|
565
|
+
}
|
|
566
|
+
if (n !== len) return Response.json({ error: `short read ${n}/${len}` }, { status: 502 });
|
|
567
|
+
return Response.json({ crc32: crcHex(c), length: n, size: o.size, ms: Date.now() - t0 });
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
// ../zip-worker/build/out/stage/src/worker.ts
|
|
571
|
+
function createZipWorker(cfg, opts = {}) {
|
|
572
|
+
const handler = "handle" in cfg ? cfg : createZipHandler(cfg);
|
|
573
|
+
return {
|
|
574
|
+
fetch: (req, env) => handler.handle(req, env, { ...opts, fixedLength: (n) => new FixedLengthStream(n) })
|
|
575
|
+
};
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
// ../zip-worker/build/out/stage/build/entry/index.ts
|
|
579
|
+
var PACKAGE_VERSION = true ? "0.2.0" : "0.0.0-dev";
|
|
580
|
+
export {
|
|
581
|
+
CACHE_BYTES,
|
|
582
|
+
PACKAGE_VERSION,
|
|
583
|
+
PinError,
|
|
584
|
+
createZipHandler,
|
|
585
|
+
createZipWorker,
|
|
586
|
+
disposition,
|
|
587
|
+
pagesOf,
|
|
588
|
+
parseRange
|
|
589
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Your product's zip Worker configuration. EDIT every value: the site that writes plans and signs links must use the
|
|
3
|
+
* same key layout and domain tags (import them from one file both share).
|
|
4
|
+
*/
|
|
5
|
+
import type { ZipWorkerConfig } from "@uploadfilm/sdk/workers/zip";
|
|
6
|
+
|
|
7
|
+
/** Collection ids your site issues (anything else is a 404 before any storage read). */
|
|
8
|
+
const COLLECTION_ID = /^[A-Za-z0-9_-]{16,64}$/;
|
|
9
|
+
|
|
10
|
+
export const CONFIG: ZipWorkerConfig = {
|
|
11
|
+
keys: {
|
|
12
|
+
planKey: (c, planId) => `zip/${c}/${planId}.json`, // give zip/ a lifecycle rule longer than your link lifetime
|
|
13
|
+
genKey: (c) => `zip-markers/${c}.json`, // outside zip/: a lost marker would revive withdrawn links
|
|
14
|
+
entryPrefix: (c) => `files/${c}/`,
|
|
15
|
+
},
|
|
16
|
+
domains: { link: "my-product-zip-1", crc: "my-product-crc-1" },
|
|
17
|
+
shareId: COLLECTION_ID,
|
|
18
|
+
// Only what a download may contain (e.g. finished files, never drafts or other collections' keys).
|
|
19
|
+
keyAllowed: (c, key) => key.startsWith(`files/${c}/`) && !key.includes("/../"),
|
|
20
|
+
shareOfKey: (key) => /^files\/([^/]+)\//.exec(key)?.[1] ?? null,
|
|
21
|
+
pages: { back: "Go back to the page you downloaded from and press Download again for a fresh link." },
|
|
22
|
+
};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Your zip download Worker: GET /z/<collection>/<plan>/<name>.zip (signed links from your site). Settings: ./config.ts. */
|
|
2
|
+
import { createZipWorker, type Env } from "@uploadfilm/sdk/workers/zip";
|
|
3
|
+
import { CONFIG } from "./config";
|
|
4
|
+
|
|
5
|
+
export default createZipWorker(CONFIG) satisfies ExportedHandler<Env>;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Your zip download Worker (written by `npx @uploadfilm/sdk init zip-worker`). EDIT the name, the route and the bucket.
|
|
2
|
+
#
|
|
3
|
+
# npx wrangler secret put ZIP_SIGNING_KEY # the same value as your site's; ≥ 32 characters
|
|
4
|
+
# npx wrangler deploy
|
|
5
|
+
name = "my-zip-worker"
|
|
6
|
+
main = "worker.ts"
|
|
7
|
+
compatibility_date = "2026-10-01"
|
|
8
|
+
# node:zlib's crc32 (the CRC endpoint).
|
|
9
|
+
compatibility_flags = ["nodejs_compat"]
|
|
10
|
+
workers_dev = true
|
|
11
|
+
# routes = [{ pattern = "zip.example.com", custom_domain = true }]
|
|
12
|
+
|
|
13
|
+
# One archive is one request: plans of 75,000 entries mean as many R2 reads (subrequests), and a CRC request reads up
|
|
14
|
+
# to 4 GiB. Workers Paid plan limits.
|
|
15
|
+
[limits]
|
|
16
|
+
cpu_ms = 300000
|
|
17
|
+
subrequests = 1000000
|
|
18
|
+
|
|
19
|
+
[[r2_buckets]]
|
|
20
|
+
binding = "BUCKET"
|
|
21
|
+
bucket_name = "my-bucket"
|
|
22
|
+
|
|
23
|
+
[vars]
|
|
24
|
+
# Prepended to every key read: set it when the Worker shares a test bucket; "" in production.
|
|
25
|
+
KEY_PREFIX = ""
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** This package's version. */
|
|
2
|
+
export declare const PACKAGE_VERSION: string;
|
|
3
|
+
export { createZipWorker } from "../../src/worker.mjs";
|
|
4
|
+
export { CACHE_BYTES, PinError, createZipHandler, disposition, parseRange, type BucketLike, type Env, type ObjLike, type Options, type ZipHandler } from "../../src/serve.mjs";
|
|
5
|
+
export { pagesOf, type ZipWorkerConfig } from "../../src/config.mjs";
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What a product tells its zip Worker. Nothing here has a product default: the key layout, the key policy and the
|
|
3
|
+
* signature domains are the product's (the site that writes plans and signs links uses the same values).
|
|
4
|
+
*
|
|
5
|
+
* Loaded by plain Node and workerd: imports by file path, erasable TypeScript only.
|
|
6
|
+
*/
|
|
7
|
+
import type { ZipKeyLayout } from "@uploadfilm/sdk";
|
|
8
|
+
import type { ZipDomains } from "@uploadfilm/sdk/server";
|
|
9
|
+
export interface ZipWorkerConfig {
|
|
10
|
+
/** Where plans and generation markers live, and the key prefix of a collection's files. */
|
|
11
|
+
keys: ZipKeyLayout;
|
|
12
|
+
/** Signature domain tags (link, crc): the same as the site's. */
|
|
13
|
+
domains: ZipDomains;
|
|
14
|
+
/** A valid collection id (the <shareId> of /z/<shareId>/…). Anything else is a 404 before any storage read. */
|
|
15
|
+
shareId: RegExp;
|
|
16
|
+
/**
|
|
17
|
+
* May an archive of collection `share` (or a CRC request) read `key`? Called for every key of a plan when it is
|
|
18
|
+
* loaded, and for every CRC request. The product's key policy: e.g. only finished originals and proxies.
|
|
19
|
+
*/
|
|
20
|
+
keyAllowed(share: string, key: string): boolean;
|
|
21
|
+
/** The collection a CRC request's key belongs to, or null (→ 403). Then shareId and keyAllowed are checked too. */
|
|
22
|
+
shareOfKey(key: string): string | null;
|
|
23
|
+
/** Wording of the error pages a browser shows. */
|
|
24
|
+
pages?: {
|
|
25
|
+
/** What to do after a refused link (default: go back to the page and press Download again). */
|
|
26
|
+
back?: string;
|
|
27
|
+
/** What to do when an archive is too large for one zip (default: download it in parts). */
|
|
28
|
+
tooLarge?: string;
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
/** The error-page wording, defaults filled in. */
|
|
32
|
+
export declare function pagesOf(cfg: ZipWorkerConfig): {
|
|
33
|
+
back: string;
|
|
34
|
+
tooLarge: string;
|
|
35
|
+
};
|