@cedarjs/uploads 7.0.1-next.333 → 7.0.1-next.467
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/authenticator.d.ts +2 -2
- package/dist/authenticator.d.ts.map +1 -1
- package/dist/cleanupStaleUploads.d.ts +4 -3
- package/dist/cleanupStaleUploads.d.ts.map +1 -1
- package/dist/cleanupStaleUploads.js +8 -2
- package/dist/contentTypes.d.ts +74 -0
- package/dist/contentTypes.d.ts.map +1 -0
- package/dist/contentTypes.js +78 -0
- package/dist/deleteFile.d.ts +19 -4
- package/dist/deleteFile.d.ts.map +1 -1
- package/dist/deleteFile.js +49 -8
- package/dist/detectContentType.d.ts +168 -0
- package/dist/detectContentType.d.ts.map +1 -0
- package/dist/detectContentType.js +443 -0
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +2 -1
- package/dist/fastify/plugin.d.ts +30 -2
- package/dist/fastify/plugin.d.ts.map +1 -1
- package/dist/fastify/plugin.js +38 -10
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +35 -0
- package/dist/keys.d.ts +3 -1
- package/dist/keys.d.ts.map +1 -1
- package/dist/keys.js +2 -1
- package/dist/profiles.d.ts +8 -2
- package/dist/profiles.d.ts.map +1 -1
- package/dist/profiles.js +5 -4
- package/dist/providers/s3.d.ts.map +1 -1
- package/dist/providers/s3.js +16 -0
- package/dist/services.d.ts +24 -6
- package/dist/services.d.ts.map +1 -1
- package/dist/services.js +45 -10
- package/dist/types.d.ts +116 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/variants.d.ts +207 -0
- package/dist/variants.d.ts.map +1 -0
- package/dist/variants.js +498 -0
- package/dist/verifyStoredContentType.d.ts +19 -0
- package/dist/verifyStoredContentType.d.ts.map +1 -0
- package/dist/verifyStoredContentType.js +41 -0
- package/dist/webhooks/s3.d.ts +17 -8
- package/dist/webhooks/s3.d.ts.map +1 -1
- package/dist/webhooks/s3.js +22 -3
- package/package.json +18 -18
package/dist/variants.js
ADDED
|
@@ -0,0 +1,498 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
3
|
+
import mime from "mime-types";
|
|
4
|
+
import { UploadError } from "./errors.js";
|
|
5
|
+
import { generateStorageKey } from "./keys.js";
|
|
6
|
+
import { DB_MAX_FILE_SIZE } from "./profiles.js";
|
|
7
|
+
import { toDataUri } from "./serialize.js";
|
|
8
|
+
import { resolveTarget } from "./targets.js";
|
|
9
|
+
const ONE_DAY = 24 * 60 * 60 * 1e3;
|
|
10
|
+
const TEN_MINUTES = 10 * 60 * 1e3;
|
|
11
|
+
const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/i;
|
|
12
|
+
const VARIANT_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
13
|
+
function idOf(source) {
|
|
14
|
+
return typeof source === "string" ? source : source.id;
|
|
15
|
+
}
|
|
16
|
+
function isPrismaError(e, code) {
|
|
17
|
+
return typeof e === "object" && e !== null && "code" in e && e.code === code;
|
|
18
|
+
}
|
|
19
|
+
function defaultFilename(sourceFilename, name, mimeType) {
|
|
20
|
+
const base = sourceFilename.replace(/\.[^./\\]*$/, "") || "file";
|
|
21
|
+
const ext = mimeType ? mime.extension(mimeType.split(";")[0].trim()) : false;
|
|
22
|
+
return ext ? `${base}-${name}.${ext}` : `${base}-${name}`;
|
|
23
|
+
}
|
|
24
|
+
async function readBytes(db, targets, upload) {
|
|
25
|
+
const target = resolveTarget(targets, upload.target);
|
|
26
|
+
if (target.providerType === "db") {
|
|
27
|
+
const row = await db.upload.findUnique({ where: { id: upload.id } });
|
|
28
|
+
if (!row?.data) {
|
|
29
|
+
throw new UploadError(
|
|
30
|
+
"NOT_FOUND",
|
|
31
|
+
`Upload '${upload.id}' has no inline data.`
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
return Buffer.from(row.data);
|
|
35
|
+
}
|
|
36
|
+
if (!upload.storageKey) {
|
|
37
|
+
throw new UploadError(
|
|
38
|
+
"NOT_FOUND",
|
|
39
|
+
`Upload '${upload.id}' has no stored file.`
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
return Buffer.from(await target.read(upload.storageKey));
|
|
43
|
+
}
|
|
44
|
+
async function deleteVariantRow(db, targets, row) {
|
|
45
|
+
let current = row;
|
|
46
|
+
while (current) {
|
|
47
|
+
if (current.storageKey) {
|
|
48
|
+
await resolveTarget(targets, current.target).delete(current.storageKey);
|
|
49
|
+
}
|
|
50
|
+
const { count } = await db.upload.deleteMany({
|
|
51
|
+
where: {
|
|
52
|
+
id: current.id,
|
|
53
|
+
status: current.status,
|
|
54
|
+
tokenId: current.tokenId,
|
|
55
|
+
storageKey: current.storageKey
|
|
56
|
+
}
|
|
57
|
+
});
|
|
58
|
+
if (count === 1) {
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
current = await db.upload.findUnique({
|
|
62
|
+
where: { id: current.id },
|
|
63
|
+
omit: { data: true }
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
async function deleteVariants(source, { db, targets }) {
|
|
68
|
+
const rows = await db.upload.findMany({
|
|
69
|
+
where: { sourceId: idOf(source) },
|
|
70
|
+
omit: { data: true }
|
|
71
|
+
});
|
|
72
|
+
for (const row of rows) {
|
|
73
|
+
await deleteVariantRow(db, targets, row);
|
|
74
|
+
}
|
|
75
|
+
return rows.length;
|
|
76
|
+
}
|
|
77
|
+
function defineUploadVariants(options) {
|
|
78
|
+
const {
|
|
79
|
+
db,
|
|
80
|
+
targets,
|
|
81
|
+
target: defaultTarget,
|
|
82
|
+
retryFailedAfter = ONE_DAY,
|
|
83
|
+
claimTimeout = TEN_MINUTES,
|
|
84
|
+
pollInterval = 250,
|
|
85
|
+
onError = (error, { sourceId, variant }) => console.error(
|
|
86
|
+
`[cedar uploads] could not make variant '${variant}' of ${sourceId}:`,
|
|
87
|
+
error
|
|
88
|
+
)
|
|
89
|
+
} = options;
|
|
90
|
+
const definitions = /* @__PURE__ */ new Map();
|
|
91
|
+
for (const [name, value] of Object.entries(options.variants)) {
|
|
92
|
+
if (!VARIANT_NAME.test(name)) {
|
|
93
|
+
throw new UploadError(
|
|
94
|
+
"CONFIGURATION",
|
|
95
|
+
`Invalid variant name '${name}'. Use letters, digits, '.', '_' and '-', starting with a letter or digit.`
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
const definition = typeof value === "function" ? { generate: value } : value;
|
|
99
|
+
const target = definition.target ?? defaultTarget;
|
|
100
|
+
if (!Object.hasOwn(targets, target.name) || targets[target.name] !== target) {
|
|
101
|
+
throw new UploadError(
|
|
102
|
+
"CONFIGURATION",
|
|
103
|
+
`Variant '${name}' uses a storage target that isn't in \`targets\`.`
|
|
104
|
+
);
|
|
105
|
+
}
|
|
106
|
+
definitions.set(name, { generate: definition.generate, target });
|
|
107
|
+
}
|
|
108
|
+
const names = [...definitions.keys()];
|
|
109
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
110
|
+
const waitingFor = /* @__PURE__ */ new Map();
|
|
111
|
+
const keyOf = (sourceId, name) => `${sourceId}\0${name}`;
|
|
112
|
+
const isWaitingFor = (from, to) => {
|
|
113
|
+
const seen = /* @__PURE__ */ new Set();
|
|
114
|
+
const stack = [from];
|
|
115
|
+
for (let key = stack.pop(); key !== void 0; key = stack.pop()) {
|
|
116
|
+
if (key === to) {
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
if (!seen.has(key)) {
|
|
120
|
+
seen.add(key);
|
|
121
|
+
stack.push(...waitingFor.get(key) ?? []);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return false;
|
|
125
|
+
};
|
|
126
|
+
const obtainFor = async (waiter, sourceId, name, chain) => {
|
|
127
|
+
const key = keyOf(sourceId, name);
|
|
128
|
+
if (!chain.includes(name) && inFlight.has(key) && isWaitingFor(key, waiter)) {
|
|
129
|
+
throw new UploadError(
|
|
130
|
+
"CONFIGURATION",
|
|
131
|
+
`Variant '${name}' and the variant reading it depend on each other.`
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
const waits = waitingFor.get(waiter) ?? [];
|
|
135
|
+
waitingFor.set(waiter, [...waits, key]);
|
|
136
|
+
try {
|
|
137
|
+
return await obtain(sourceId, name, chain);
|
|
138
|
+
} finally {
|
|
139
|
+
const rest = waitingFor.get(waiter) ?? [];
|
|
140
|
+
rest.splice(rest.indexOf(key), 1);
|
|
141
|
+
if (rest.length === 0) {
|
|
142
|
+
waitingFor.delete(waiter);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
const definitionOf = (name) => {
|
|
147
|
+
const definition = definitions.get(name);
|
|
148
|
+
if (!definition) {
|
|
149
|
+
throw new UploadError(
|
|
150
|
+
"CONFIGURATION",
|
|
151
|
+
`Unknown upload variant '${name}'. Configured variants: ${names.join(", ") || "(none)"}.`
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
return definition;
|
|
155
|
+
};
|
|
156
|
+
const loadSource = async (sourceId) => {
|
|
157
|
+
const source = await db.upload.findUnique({
|
|
158
|
+
where: { id: sourceId },
|
|
159
|
+
omit: { data: true }
|
|
160
|
+
});
|
|
161
|
+
if (source?.status !== "completed") {
|
|
162
|
+
throw new UploadError(
|
|
163
|
+
"NOT_FOUND",
|
|
164
|
+
`Upload '${sourceId}' does not exist or is not completed, so it cannot have variants.`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
if (source.sourceId) {
|
|
168
|
+
throw new UploadError(
|
|
169
|
+
"NOT_SUPPORTED",
|
|
170
|
+
`Upload '${sourceId}' is itself a variant. Variants are made from original uploads only; use \`readVariant()\` in a generator to make one variant from another.`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
return source;
|
|
174
|
+
};
|
|
175
|
+
const findRow = (sourceId, name) => db.upload.findFirst({
|
|
176
|
+
where: { sourceId, variant: name },
|
|
177
|
+
omit: { data: true }
|
|
178
|
+
});
|
|
179
|
+
const make = async (source, rowId, claim, name, chain) => {
|
|
180
|
+
const definition = definitionOf(name);
|
|
181
|
+
const target = definition.target;
|
|
182
|
+
const ours = { id: rowId, status: "pending", tokenId: claim };
|
|
183
|
+
const fail = async (error) => {
|
|
184
|
+
if (error !== void 0) {
|
|
185
|
+
onError(error, { sourceId: source.id, variant: name });
|
|
186
|
+
}
|
|
187
|
+
await db.upload.updateMany({ where: ours, data: { status: "failed" } });
|
|
188
|
+
return null;
|
|
189
|
+
};
|
|
190
|
+
let file;
|
|
191
|
+
try {
|
|
192
|
+
file = await definition.generate({
|
|
193
|
+
source,
|
|
194
|
+
variant: name,
|
|
195
|
+
readSource: () => readBytes(db, targets, source),
|
|
196
|
+
readVariant: async (other) => {
|
|
197
|
+
const made = await obtainFor(
|
|
198
|
+
keyOf(source.id, name),
|
|
199
|
+
source.id,
|
|
200
|
+
other,
|
|
201
|
+
[...chain, name]
|
|
202
|
+
);
|
|
203
|
+
return made ? readBytes(db, targets, made) : null;
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
} catch (e) {
|
|
207
|
+
return fail(e);
|
|
208
|
+
}
|
|
209
|
+
if (!file) {
|
|
210
|
+
return fail();
|
|
211
|
+
}
|
|
212
|
+
if (!MIME_TYPE.test(file.mimeType)) {
|
|
213
|
+
return fail(
|
|
214
|
+
new UploadError(
|
|
215
|
+
"MIME_TYPE_NOT_ALLOWED",
|
|
216
|
+
`Variant '${name}' returned '${file.mimeType}', which is not a valid MIME type.`
|
|
217
|
+
)
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
const isDbTarget = target.providerType === "db";
|
|
221
|
+
if (isDbTarget && file.data.byteLength > DB_MAX_FILE_SIZE) {
|
|
222
|
+
return fail(
|
|
223
|
+
new UploadError(
|
|
224
|
+
"FILE_TOO_LARGE",
|
|
225
|
+
`Variant '${name}' is ${file.data.byteLength} bytes, which exceeds the ${DB_MAX_FILE_SIZE} byte limit for target '${target.name}'.`
|
|
226
|
+
)
|
|
227
|
+
);
|
|
228
|
+
}
|
|
229
|
+
const storageKey = isDbTarget ? null : generateStorageKey(file.mimeType);
|
|
230
|
+
if (storageKey) {
|
|
231
|
+
try {
|
|
232
|
+
await target.write(storageKey, file.data, {
|
|
233
|
+
contentType: file.mimeType
|
|
234
|
+
});
|
|
235
|
+
} catch (e) {
|
|
236
|
+
await target.delete(storageKey).catch(() => void 0);
|
|
237
|
+
await db.upload.deleteMany({ where: ours });
|
|
238
|
+
throw e;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
const { count } = await db.upload.updateMany({
|
|
242
|
+
where: ours,
|
|
243
|
+
data: {
|
|
244
|
+
status: "completed",
|
|
245
|
+
// A retry or a takeover writes to the variant's current target,
|
|
246
|
+
// which is not necessarily the one the row was created for
|
|
247
|
+
target: target.name,
|
|
248
|
+
storageKey,
|
|
249
|
+
size: BigInt(file.data.byteLength),
|
|
250
|
+
mimeType: file.mimeType,
|
|
251
|
+
filename: file.filename ?? defaultFilename(source.filename, name, file.mimeType),
|
|
252
|
+
// Copying gives Prisma the plain-ArrayBuffer-backed array its
|
|
253
|
+
// `Bytes` input requires
|
|
254
|
+
data: isDbTarget ? new Uint8Array(file.data) : null,
|
|
255
|
+
tokenId: null
|
|
256
|
+
}
|
|
257
|
+
}).catch(async (e) => {
|
|
258
|
+
if (storageKey) {
|
|
259
|
+
await target.delete(storageKey).catch(() => void 0);
|
|
260
|
+
}
|
|
261
|
+
await db.upload.deleteMany({ where: ours }).catch(() => void 0);
|
|
262
|
+
throw e;
|
|
263
|
+
});
|
|
264
|
+
if (count !== 1) {
|
|
265
|
+
if (storageKey) {
|
|
266
|
+
await target.delete(storageKey).catch(() => void 0);
|
|
267
|
+
}
|
|
268
|
+
const current = await findRow(source.id, name);
|
|
269
|
+
return current?.status === "completed" ? current : null;
|
|
270
|
+
}
|
|
271
|
+
return db.upload.findUnique({ where: { id: rowId }, omit: { data: true } });
|
|
272
|
+
};
|
|
273
|
+
const obtainUncached = async (sourceId, name, chain) => {
|
|
274
|
+
const definition = definitionOf(name);
|
|
275
|
+
const source = await loadSource(sourceId);
|
|
276
|
+
for (; ; ) {
|
|
277
|
+
const row = await findRow(sourceId, name);
|
|
278
|
+
const now = Date.now();
|
|
279
|
+
const claim = randomUUID();
|
|
280
|
+
if (!row) {
|
|
281
|
+
let created;
|
|
282
|
+
try {
|
|
283
|
+
created = await db.upload.create({
|
|
284
|
+
data: {
|
|
285
|
+
sourceId,
|
|
286
|
+
variant: name,
|
|
287
|
+
target: definition.target.name,
|
|
288
|
+
status: "pending",
|
|
289
|
+
filename: defaultFilename(source.filename, name),
|
|
290
|
+
mimeType: "application/octet-stream",
|
|
291
|
+
size: 0n,
|
|
292
|
+
tokenId: claim,
|
|
293
|
+
userId: source.userId,
|
|
294
|
+
organizationId: source.organizationId
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
} catch (e) {
|
|
298
|
+
if (isPrismaError(e, "P2002")) {
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
301
|
+
if (isPrismaError(e, "P2003")) {
|
|
302
|
+
throw new UploadError(
|
|
303
|
+
"NOT_FOUND",
|
|
304
|
+
`Upload '${sourceId}' was deleted while its variant '${name}' was being made.`,
|
|
305
|
+
e
|
|
306
|
+
);
|
|
307
|
+
}
|
|
308
|
+
throw e;
|
|
309
|
+
}
|
|
310
|
+
return make(source, created.id, claim, name, chain);
|
|
311
|
+
}
|
|
312
|
+
if (row.status === "completed") {
|
|
313
|
+
return row;
|
|
314
|
+
}
|
|
315
|
+
if (row.status === "failed") {
|
|
316
|
+
const cutoff2 = new Date(now - retryFailedAfter);
|
|
317
|
+
if (row.updatedAt >= cutoff2) {
|
|
318
|
+
return null;
|
|
319
|
+
}
|
|
320
|
+
const { count } = await db.upload.updateMany({
|
|
321
|
+
where: { id: row.id, status: "failed", updatedAt: { lt: cutoff2 } },
|
|
322
|
+
data: { status: "pending", tokenId: claim }
|
|
323
|
+
});
|
|
324
|
+
if (count === 1) {
|
|
325
|
+
return make(source, row.id, claim, name, chain);
|
|
326
|
+
}
|
|
327
|
+
continue;
|
|
328
|
+
}
|
|
329
|
+
const cutoff = new Date(now - claimTimeout);
|
|
330
|
+
if (row.updatedAt < cutoff) {
|
|
331
|
+
const { count } = await db.upload.updateMany({
|
|
332
|
+
where: {
|
|
333
|
+
id: row.id,
|
|
334
|
+
status: "pending",
|
|
335
|
+
tokenId: row.tokenId,
|
|
336
|
+
updatedAt: { lt: cutoff }
|
|
337
|
+
},
|
|
338
|
+
data: { tokenId: claim }
|
|
339
|
+
});
|
|
340
|
+
if (count === 1) {
|
|
341
|
+
return make(source, row.id, claim, name, chain);
|
|
342
|
+
}
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
345
|
+
await sleep(pollInterval);
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
const obtain = (sourceId, name, chain) => {
|
|
349
|
+
if (chain.includes(name)) {
|
|
350
|
+
return Promise.reject(
|
|
351
|
+
new UploadError(
|
|
352
|
+
"CONFIGURATION",
|
|
353
|
+
`Variant '${name}' depends on itself: ${[...chain, name].join(" -> ")}.`
|
|
354
|
+
)
|
|
355
|
+
);
|
|
356
|
+
}
|
|
357
|
+
const key = keyOf(sourceId, name);
|
|
358
|
+
const running = inFlight.get(key);
|
|
359
|
+
if (running) {
|
|
360
|
+
return running;
|
|
361
|
+
}
|
|
362
|
+
const attempt = obtainUncached(sourceId, name, chain);
|
|
363
|
+
const forget = () => {
|
|
364
|
+
if (inFlight.get(key) === attempt) {
|
|
365
|
+
inFlight.delete(key);
|
|
366
|
+
}
|
|
367
|
+
};
|
|
368
|
+
inFlight.set(key, attempt);
|
|
369
|
+
attempt.then(forget, forget);
|
|
370
|
+
return attempt;
|
|
371
|
+
};
|
|
372
|
+
const getVariant = (source, name) => obtain(idOf(source), name, []);
|
|
373
|
+
const findVariant = async (source, name) => {
|
|
374
|
+
definitionOf(name);
|
|
375
|
+
return db.upload.findFirst({
|
|
376
|
+
where: { sourceId: idOf(source), variant: name, status: "completed" },
|
|
377
|
+
omit: { data: true }
|
|
378
|
+
});
|
|
379
|
+
};
|
|
380
|
+
const findVariants = async (sourceIds, name) => {
|
|
381
|
+
definitionOf(name);
|
|
382
|
+
const rows = sourceIds.length ? await db.upload.findMany({
|
|
383
|
+
where: {
|
|
384
|
+
sourceId: { in: sourceIds },
|
|
385
|
+
variant: name,
|
|
386
|
+
status: "completed"
|
|
387
|
+
},
|
|
388
|
+
omit: { data: true }
|
|
389
|
+
}) : [];
|
|
390
|
+
const bySource = /* @__PURE__ */ new Map();
|
|
391
|
+
for (const row of rows) {
|
|
392
|
+
if (row.sourceId) {
|
|
393
|
+
bySource.set(row.sourceId, row);
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
return bySource;
|
|
397
|
+
};
|
|
398
|
+
const readVariant = async (source, name) => {
|
|
399
|
+
const upload = await getVariant(source, name);
|
|
400
|
+
if (!upload) {
|
|
401
|
+
return null;
|
|
402
|
+
}
|
|
403
|
+
return { upload, data: await readBytes(db, targets, upload) };
|
|
404
|
+
};
|
|
405
|
+
const getVariantUrl = async (source, name, urlOptions = {}) => {
|
|
406
|
+
const upload = await getVariant(source, name);
|
|
407
|
+
if (!upload) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
const target = resolveTarget(targets, upload.target);
|
|
411
|
+
if (target.providerType === "db") {
|
|
412
|
+
return toDataUri(upload.mimeType, await readBytes(db, targets, upload));
|
|
413
|
+
}
|
|
414
|
+
if (!upload.storageKey) {
|
|
415
|
+
return null;
|
|
416
|
+
}
|
|
417
|
+
return target.getSignedReadUrl(upload.storageKey, urlOptions);
|
|
418
|
+
};
|
|
419
|
+
const regenerateVariants = async (name, {
|
|
420
|
+
remake = true,
|
|
421
|
+
batchSize = 100
|
|
422
|
+
} = {}) => {
|
|
423
|
+
definitionOf(name);
|
|
424
|
+
const result = {
|
|
425
|
+
deleted: 0,
|
|
426
|
+
made: 0,
|
|
427
|
+
failed: 0,
|
|
428
|
+
errors: 0
|
|
429
|
+
};
|
|
430
|
+
const newest = await db.upload.findFirst({
|
|
431
|
+
where: { variant: name },
|
|
432
|
+
omit: { data: true },
|
|
433
|
+
orderBy: { createdAt: "desc" }
|
|
434
|
+
});
|
|
435
|
+
if (!newest) {
|
|
436
|
+
return result;
|
|
437
|
+
}
|
|
438
|
+
const cutoff = newest.createdAt;
|
|
439
|
+
const exclude = [];
|
|
440
|
+
for (; ; ) {
|
|
441
|
+
const rows = await db.upload.findMany({
|
|
442
|
+
where: {
|
|
443
|
+
variant: name,
|
|
444
|
+
createdAt: { lte: cutoff },
|
|
445
|
+
...exclude.length > 0 && { id: { notIn: exclude } }
|
|
446
|
+
},
|
|
447
|
+
omit: { data: true },
|
|
448
|
+
orderBy: { createdAt: "asc" },
|
|
449
|
+
take: batchSize
|
|
450
|
+
});
|
|
451
|
+
if (rows.length === 0) {
|
|
452
|
+
return result;
|
|
453
|
+
}
|
|
454
|
+
for (const row of rows) {
|
|
455
|
+
const sourceId = row.sourceId ?? "";
|
|
456
|
+
try {
|
|
457
|
+
await deleteVariantRow(db, targets, row);
|
|
458
|
+
result.deleted += 1;
|
|
459
|
+
} catch (e) {
|
|
460
|
+
result.errors += 1;
|
|
461
|
+
exclude.push(row.id);
|
|
462
|
+
onError(e, { sourceId, variant: name });
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (remake) {
|
|
466
|
+
await inFlight.get(keyOf(sourceId, name))?.catch(() => null);
|
|
467
|
+
const made = await getVariant(sourceId, name).catch((e) => {
|
|
468
|
+
onError(e, { sourceId, variant: name });
|
|
469
|
+
return null;
|
|
470
|
+
});
|
|
471
|
+
if (made) {
|
|
472
|
+
result.made += 1;
|
|
473
|
+
} else {
|
|
474
|
+
result.failed += 1;
|
|
475
|
+
}
|
|
476
|
+
const current = await findRow(sourceId, name);
|
|
477
|
+
if (current && current.createdAt <= cutoff) {
|
|
478
|
+
exclude.push(current.id);
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
};
|
|
484
|
+
return {
|
|
485
|
+
names,
|
|
486
|
+
getVariant,
|
|
487
|
+
findVariant,
|
|
488
|
+
findVariants,
|
|
489
|
+
readVariant,
|
|
490
|
+
getVariantUrl,
|
|
491
|
+
regenerateVariants,
|
|
492
|
+
deleteVariants: (source) => deleteVariants(source, { db, targets })
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
export {
|
|
496
|
+
defineUploadVariants,
|
|
497
|
+
deleteVariants
|
|
498
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { StorageProvider } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Checks an object that was uploaded straight to storage against the type
|
|
4
|
+
* the client declared for it when it asked for the upload URL. The declared
|
|
5
|
+
* type was checked against the upload profile then, and is the
|
|
6
|
+
* `Content-Type` the object was stored with, so the bytes must be that type
|
|
7
|
+
* (or an alias of it). Reads the first `DETECT_CONTENT_TYPE_BYTES`, plus the
|
|
8
|
+
* last `DETECT_CONTENT_TYPE_TAIL_BYTES` when the head is a ZIP, plus the
|
|
9
|
+
* start of the ZIP's central directory when that begins before the tail
|
|
10
|
+
* (at most `MAX_ZIP_DIRECTORY_BYTES`).
|
|
11
|
+
*
|
|
12
|
+
* Returns the canonical type to record, or `null` when the bytes aren't the
|
|
13
|
+
* declared type.
|
|
14
|
+
*/
|
|
15
|
+
export declare function verifyStoredContentType(target: StorageProvider, key: string, { size, declaredType }: {
|
|
16
|
+
size: number;
|
|
17
|
+
declaredType: string;
|
|
18
|
+
}): Promise<string | null>;
|
|
19
|
+
//# sourceMappingURL=verifyStoredContentType.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"verifyStoredContentType.d.ts","sourceRoot":"","sources":["../src/verifyStoredContentType.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAmBjD;;;;;;;;;;;;GAYG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,eAAe,EACvB,GAAG,EAAE,MAAM,EACX,EAAE,IAAI,EAAE,YAAY,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,GAC7D,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAsCxB"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import {
|
|
2
|
+
acceptedContentType,
|
|
3
|
+
DETECT_CONTENT_TYPE_BYTES,
|
|
4
|
+
DETECT_CONTENT_TYPE_TAIL_BYTES,
|
|
5
|
+
findZipCentralDirectory
|
|
6
|
+
} from "./detectContentType.js";
|
|
7
|
+
const ZIP_LOCAL_HEADER = [80, 75, 3, 4];
|
|
8
|
+
async function readRange(target, key, offset, length) {
|
|
9
|
+
if (target.readRange) {
|
|
10
|
+
return target.readRange(key, { offset, length });
|
|
11
|
+
}
|
|
12
|
+
return (await target.read(key)).subarray(offset, offset + length);
|
|
13
|
+
}
|
|
14
|
+
async function verifyStoredContentType(target, key, { size, declaredType }) {
|
|
15
|
+
const headLength = Math.min(size, DETECT_CONTENT_TYPE_BYTES);
|
|
16
|
+
const head = await readRange(target, key, 0, headLength);
|
|
17
|
+
let tail;
|
|
18
|
+
const isZip = ZIP_LOCAL_HEADER.every((byte, i) => head[i] === byte);
|
|
19
|
+
if (isZip && size > head.length) {
|
|
20
|
+
const tailLength = Math.min(size, DETECT_CONTENT_TYPE_TAIL_BYTES);
|
|
21
|
+
const tailStart = size - tailLength;
|
|
22
|
+
tail = await readRange(target, key, tailStart, tailLength);
|
|
23
|
+
const directory = findZipCentralDirectory(tail);
|
|
24
|
+
if (directory && directory.offset < tailStart && directory.offset + directory.size > tailStart) {
|
|
25
|
+
const missing = await readRange(
|
|
26
|
+
target,
|
|
27
|
+
key,
|
|
28
|
+
directory.offset,
|
|
29
|
+
tailStart - directory.offset
|
|
30
|
+
);
|
|
31
|
+
const joined = new Uint8Array(missing.length + tail.length);
|
|
32
|
+
joined.set(missing);
|
|
33
|
+
joined.set(tail, missing.length);
|
|
34
|
+
tail = joined;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return acceptedContentType(head, [declaredType], declaredType, { tail });
|
|
38
|
+
}
|
|
39
|
+
export {
|
|
40
|
+
verifyStoredContentType
|
|
41
|
+
};
|
package/dist/webhooks/s3.d.ts
CHANGED
|
@@ -17,21 +17,30 @@ export interface S3Event {
|
|
|
17
17
|
Records: S3EventRecord[];
|
|
18
18
|
}
|
|
19
19
|
export type S3EventOutcome = 'completed' | 'failed' | 'ignored' | 'not-found' | 'no-target';
|
|
20
|
+
export interface ProcessS3EventRecordOptions {
|
|
21
|
+
db: UploadDatabase;
|
|
22
|
+
targets: StorageTargets;
|
|
23
|
+
/**
|
|
24
|
+
* Checks the object's bytes against the type declared when the upload URL
|
|
25
|
+
* was issued, as `confirmUpload` does. Defaults to `true`.
|
|
26
|
+
*/
|
|
27
|
+
verifyContentType?: boolean;
|
|
28
|
+
}
|
|
20
29
|
/**
|
|
21
30
|
* Settles the `pending` row for one `s3:ObjectCreated:*` record. The row is
|
|
22
31
|
* matched on target plus storage key, the object's size is compared with
|
|
23
|
-
* the size authorized at presign time
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
32
|
+
* the size authorized at presign time and, with `verifyContentType` (the
|
|
33
|
+
* default), its bytes with the type declared then, and the row is flipped
|
|
34
|
+
* `pending` to `completed` (or claimed `failed` and the object deleted on a
|
|
35
|
+
* mismatch). Both transitions are conditional, so a duplicate notification
|
|
36
|
+
* or a row the cleanup job already claimed is left alone.
|
|
27
37
|
*/
|
|
28
|
-
export declare function processS3EventRecord(record: S3EventRecord, { db, targets }:
|
|
29
|
-
db: UploadDatabase;
|
|
30
|
-
targets: StorageTargets;
|
|
31
|
-
}): Promise<S3EventOutcome>;
|
|
38
|
+
export declare function processS3EventRecord(record: S3EventRecord, { db, targets, verifyContentType }: ProcessS3EventRecordOptions): Promise<S3EventOutcome>;
|
|
32
39
|
export interface S3WebhookOptions {
|
|
33
40
|
db: UploadDatabase;
|
|
34
41
|
targets: StorageTargets;
|
|
42
|
+
/** See `ProcessS3EventRecordOptions`. Defaults to `true`. */
|
|
43
|
+
verifyContentType?: boolean;
|
|
35
44
|
/** The trusted SNS topic. Messages from any other topic are rejected. */
|
|
36
45
|
topicArn: string;
|
|
37
46
|
fetchCertificate?: FetchCertificate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../src/webhooks/s3.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;
|
|
1
|
+
{"version":3,"file":"s3.d.ts","sourceRoot":"","sources":["../../src/webhooks/s3.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AASjE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAA;AAEhD,uEAAuE;AACvE,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,EAAE,EAAE;QACF,MAAM,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;QACxB,MAAM,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KACvC,CAAA;CACF;AAED,MAAM,WAAW,OAAO;IACtB,OAAO,EAAE,aAAa,EAAE,CAAA;CACzB;AAsBD,MAAM,MAAM,cAAc,GACxB,WAAW,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,WAAW,CAAA;AAkChE,MAAM,WAAW,2BAA2B;IAC1C,EAAE,EAAE,cAAc,CAAA;IAClB,OAAO,EAAE,cAAc,CAAA;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAA;CAC5B;AAED;;;;;;;;GAQG;AACH,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,aAAa,EACrB,EAAE,EAAE,EAAE,OAAO,EAAE,iBAAwB,EAAE,EAAE,2BAA2B,GACrE,OAAO,CAAC,cAAc,CAAC,CA2EzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,cAAc,CAAA;IAClB,OAAO,EAAE,cAAc,CAAA;IACvB,6DAA6D;IAC7D,iBAAiB,CAAC,EAAE,OAAO,CAAA;IAC3B,yEAAyE;IACzE,QAAQ,EAAE,MAAM,CAAA;IAChB,gBAAgB,CAAC,EAAE,gBAAgB,CAAA;IACnC,iEAAiE;IACjE,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;CACrB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,cAAc,GAAG,0BAA0B,GAAG,yBAAyB,CAAA;IAC7E,QAAQ,EAAE,cAAc,EAAE,CAAA;CAC3B;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,eAAe,CAAC,CA+D1B"}
|
package/dist/webhooks/s3.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { UploadError } from "../errors.js";
|
|
2
|
+
import { verifyStoredContentType } from "../verifyStoredContentType.js";
|
|
2
3
|
import {
|
|
3
4
|
assertTrustedSnsUrl,
|
|
4
5
|
isSnsMessage,
|
|
@@ -28,7 +29,7 @@ function matchTarget(targets, bucket, key) {
|
|
|
28
29
|
}
|
|
29
30
|
return best ? { name: best.name, storageKey: key.slice(best.keyPrefix.length) } : null;
|
|
30
31
|
}
|
|
31
|
-
async function processS3EventRecord(record, { db, targets }) {
|
|
32
|
+
async function processS3EventRecord(record, { db, targets, verifyContentType = true }) {
|
|
32
33
|
if (!record.eventName.startsWith("ObjectCreated")) {
|
|
33
34
|
return "ignored";
|
|
34
35
|
}
|
|
@@ -51,7 +52,7 @@ async function processS3EventRecord(record, { db, targets }) {
|
|
|
51
52
|
const target = targets[match.name];
|
|
52
53
|
const reportedSize = record.s3.object.size;
|
|
53
54
|
const actualSize = typeof reportedSize === "number" ? reportedSize : await target.getObjectSize(match.storageKey);
|
|
54
|
-
|
|
55
|
+
const fail = async () => {
|
|
55
56
|
const { count: count2 } = await db.upload.updateMany({
|
|
56
57
|
where: { id: upload.id, status: "pending" },
|
|
57
58
|
data: { status: "failed" }
|
|
@@ -60,10 +61,28 @@ async function processS3EventRecord(record, { db, targets }) {
|
|
|
60
61
|
await target.delete(match.storageKey);
|
|
61
62
|
}
|
|
62
63
|
return "failed";
|
|
64
|
+
};
|
|
65
|
+
if (actualSize === null || BigInt(actualSize) !== upload.size) {
|
|
66
|
+
return fail();
|
|
67
|
+
}
|
|
68
|
+
let mimeType = upload.mimeType;
|
|
69
|
+
if (verifyContentType) {
|
|
70
|
+
const verified = await verifyStoredContentType(target, match.storageKey, {
|
|
71
|
+
size: actualSize,
|
|
72
|
+
declaredType: upload.mimeType
|
|
73
|
+
});
|
|
74
|
+
if (!verified) {
|
|
75
|
+
return fail();
|
|
76
|
+
}
|
|
77
|
+
mimeType = verified;
|
|
63
78
|
}
|
|
64
79
|
const { count } = await db.upload.updateMany({
|
|
65
80
|
where: { id: upload.id, status: "pending" },
|
|
66
|
-
data: {
|
|
81
|
+
data: {
|
|
82
|
+
status: "completed",
|
|
83
|
+
size: BigInt(actualSize),
|
|
84
|
+
...mimeType !== upload.mimeType ? { mimeType } : {}
|
|
85
|
+
}
|
|
67
86
|
});
|
|
68
87
|
return count === 1 ? "completed" : "ignored";
|
|
69
88
|
}
|