cloudrop 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/README.md +34 -4
- package/dist/adapters/drizzle.d.ts +1 -1
- package/dist/adapters/drizzle.js +3 -3
- package/dist/adapters/drizzle.js.map +1 -1
- package/dist/adapters/prisma.d.ts +1 -1
- package/dist/adapters/prisma.js +1 -1
- package/dist/adapters/sql.d.ts +4 -1
- package/dist/adapters/sql.js +8 -1
- package/dist/adapters/sql.js.map +1 -1
- package/dist/cli.js +2045 -0
- package/dist/{client-Dj6ON4MN.d.ts → client-CP_ajLF4.d.ts} +43 -25
- package/dist/{client-Y1zUEbCj.js → client-DxP3uv8V.js} +172 -44
- package/dist/client-DxP3uv8V.js.map +1 -0
- package/dist/client.d.ts +3 -3
- package/dist/client.js +1 -1
- package/dist/errors-CUxMmFAe.js.map +1 -1
- package/dist/index.d.ts +92 -7
- package/dist/index.js +432 -152
- package/dist/index.js.map +1 -1
- package/dist/next.d.ts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/react.d.ts +18 -4
- package/dist/react.js +23 -3
- package/dist/react.js.map +1 -1
- package/dist/{route-CspI1Ypt.js → route-C8-YjzdY.js} +3 -2
- package/dist/{route-CspI1Ypt.js.map → route-C8-YjzdY.js.map} +1 -1
- package/dist/{store-BfCJejGh.d.ts → store-B539nY3J.d.ts} +41 -3
- package/dist/testing.d.ts +81 -0
- package/dist/testing.js +1243 -0
- package/dist/testing.js.map +1 -0
- package/dist/{transactional-DnvnGsVH.js → transactional-CiNwMiDb.js} +215 -105
- package/dist/transactional-CiNwMiDb.js.map +1 -0
- package/dist/{types-BrAx3dA4.d.ts → types-CFfMw-n_.d.ts} +1 -1
- package/docs/README.md +30 -0
- package/docs/bucket-setup.md +226 -0
- package/docs/client.md +386 -0
- package/docs/command.md +129 -0
- package/docs/deployment.md +169 -0
- package/docs/errors.md +150 -0
- package/docs/hono.md +276 -0
- package/docs/react.md +508 -0
- package/docs/resume.md +253 -0
- package/docs/server.md +603 -0
- package/docs/store.md +804 -0
- package/docs/testing-your-app.md +328 -0
- package/docs/troubleshooting.md +176 -0
- package/package.json +16 -8
- package/dist/client-Y1zUEbCj.js.map +0 -1
- package/dist/transactional-DnvnGsVH.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as quotaExceeded, i as ok, n as configError, r as fail, t as CloudropError } from "./errors-CUxMmFAe.js";
|
|
2
2
|
import { n as isPlainObject, t as invalidInput } from "./input-DTC476_I.js";
|
|
3
|
-
import { a as route, c as unauthorized, i as refusalError, n as forbidden, r as isRefusal, s as storeOfRoute, t as allowAnyone } from "./route-
|
|
3
|
+
import { a as route, c as unauthorized, i as refusalError, n as forbidden, r as isRefusal, s as storeOfRoute, t as allowAnyone } from "./route-C8-YjzdY.js";
|
|
4
4
|
import { n as sizeOfPart, t as countParts } from "./part-size-CvMoobfC.js";
|
|
5
5
|
import "./types-BZUOaz_B.js";
|
|
6
6
|
// src/storage/sigv4.ts
|
|
@@ -62,7 +62,7 @@ function encodeKeyPath(key) {
|
|
|
62
62
|
return key.split("/").map(encodeRfc3986).join("/");
|
|
63
63
|
}
|
|
64
64
|
function toAmzDate(date) {
|
|
65
|
-
return date.toISOString().replace(/[-:]
|
|
65
|
+
return date.toISOString().replace(/[-:]|\.\d{3}/g, "");
|
|
66
66
|
}
|
|
67
67
|
function canonicalPath(url) {
|
|
68
68
|
return url.pathname.split("/").map((segment) => encodeRfc3986(decodeURIComponent(segment))).join("/");
|
|
@@ -497,8 +497,8 @@ const PLAIN_EXTENSION = /^[a-z0-9]{1,16}$/;
|
|
|
497
497
|
const MIME_TYPE = /^[a-z0-9][a-z0-9!#$&^_.+-]{0,63}\/[a-z0-9][a-z0-9!#$&^_.+-]{0,63}$/;
|
|
498
498
|
const ACCEPT_ENTRY = /^(\*\/\*|[a-z0-9][a-z0-9!#$&^_.+-]{0,63}\/(\*|[a-z0-9][a-z0-9!#$&^_.+-]{0,63}))$/;
|
|
499
499
|
function normalizeType(type) {
|
|
500
|
-
const
|
|
501
|
-
return MIME_TYPE.test(
|
|
500
|
+
const essence = type === "" ? "application/octet-stream" : (type.split(";", 1)[0] ?? "").trim().toLowerCase();
|
|
501
|
+
return MIME_TYPE.test(essence) ? essence : null;
|
|
502
502
|
}
|
|
503
503
|
function isAcceptEntry(entry) {
|
|
504
504
|
return ACCEPT_ENTRY.test(entry);
|
|
@@ -558,6 +558,10 @@ function readMaxSize(where, definition, storage) {
|
|
|
558
558
|
return maxSize;
|
|
559
559
|
}
|
|
560
560
|
function readMaxFiles(where, definition) {
|
|
561
|
+
if (definition.replace === true) {
|
|
562
|
+
if (definition.maxFiles !== void 0 && definition.maxFiles !== 1) throw configError(`${where}: replace keeps one file per owner, so maxFiles must be 1. Leave maxFiles out.`);
|
|
563
|
+
return 1;
|
|
564
|
+
}
|
|
561
565
|
const maxFiles = definition.maxFiles ?? DEFAULT_MAX_FILES;
|
|
562
566
|
if (!Number.isInteger(maxFiles) || maxFiles < 1 || maxFiles > MAX_FILES_LIMIT) throw configError(`${where}: maxFiles must be a whole number from 1 to ${MAX_FILES_LIMIT}.`);
|
|
563
567
|
return maxFiles;
|
|
@@ -584,6 +588,7 @@ function readRoute(name, route, storage) {
|
|
|
584
588
|
const definition = route?.definition;
|
|
585
589
|
if (typeof definition !== "object" || definition === null) throw configError(`${where} is not a route. Wrap its options in route({ ... }).`);
|
|
586
590
|
if (!isFunction(definition.authorize)) throw configError(`${where} needs authorize. Use authorize: allowAnyone for a route anyone may upload to.`);
|
|
591
|
+
if (definition.replace !== void 0 && typeof definition.replace !== "boolean") throw configError(`${where}: replace must be true or false.`);
|
|
587
592
|
const accept = readAccept(where, definition);
|
|
588
593
|
const maxSize = readMaxSize(where, definition, storage);
|
|
589
594
|
const maxFiles = readMaxFiles(where, definition);
|
|
@@ -594,7 +599,8 @@ function readRoute(name, route, storage) {
|
|
|
594
599
|
accept,
|
|
595
600
|
maxSize,
|
|
596
601
|
maxFiles,
|
|
597
|
-
visibility: readVisibility(where, definition, storage)
|
|
602
|
+
visibility: readVisibility(where, definition, storage),
|
|
603
|
+
replace: definition.replace === true
|
|
598
604
|
};
|
|
599
605
|
}
|
|
600
606
|
// src/server/store-config.ts
|
|
@@ -609,6 +615,7 @@ const STORE_METHODS = [
|
|
|
609
615
|
"end",
|
|
610
616
|
"revoke",
|
|
611
617
|
"stale",
|
|
618
|
+
"list",
|
|
612
619
|
"cleaned",
|
|
613
620
|
"usage",
|
|
614
621
|
"reconcile",
|
|
@@ -622,8 +629,13 @@ function readStoreObject(given) {
|
|
|
622
629
|
if (store.transaction !== void 0 && !isFunction(store.transaction)) throw configError("store.transaction must be a function, or left out for a store without transactions.");
|
|
623
630
|
return given;
|
|
624
631
|
}
|
|
632
|
+
function checkReplace(name, definition, store) {
|
|
633
|
+
if (!store) throw configError(`Route "${name}": replace needs a database store, which deletes the previous file when the next one completes. Pass store to createCloudrop, or remove replace.`);
|
|
634
|
+
if (!definition.owner) throw configError(`Route "${name}": replace keeps one file per owner, so it needs owner. Add owner to the route, or remove replace.`);
|
|
635
|
+
}
|
|
625
636
|
function checkRoutes(routes, store) {
|
|
626
|
-
for (const { name, definition } of routes) {
|
|
637
|
+
for (const { name, definition, replace } of routes) {
|
|
638
|
+
if (replace) checkReplace(name, definition, store);
|
|
627
639
|
const needsStore = ["owner", "afterComplete"].find((callback) => definition[callback]);
|
|
628
640
|
if (needsStore && !store) throw configError(`Route "${name}": ${needsStore} needs a database store. Pass store to createCloudrop, or remove ${needsStore}.`);
|
|
629
641
|
const routeStore = storeOfRoute(definition);
|
|
@@ -883,9 +895,11 @@ function allOk(results) {
|
|
|
883
895
|
}
|
|
884
896
|
// src/server/presign.ts
|
|
885
897
|
const TOKEN_LIFETIME_SECONDS = 86400;
|
|
898
|
+
const LAST_URL_MARGIN_SECONDS = 900;
|
|
886
899
|
const nowSeconds = () => Math.floor(Date.now() / 1e3);
|
|
887
900
|
function urlLifetime(token) {
|
|
888
|
-
const
|
|
901
|
+
const signUntil = token.exp - LAST_URL_MARGIN_SECONDS;
|
|
902
|
+
const seconds = Math.min(900, signUntil - Math.ceil(Date.now() / 1e3));
|
|
889
903
|
return seconds > 0 ? seconds : null;
|
|
890
904
|
}
|
|
891
905
|
function toWire(signed) {
|
|
@@ -945,10 +959,41 @@ function fileCalls(store, logger, token) {
|
|
|
945
959
|
get: () => call("get", () => store.get(id)),
|
|
946
960
|
claim: (claim, leaseMs) => call("claim", () => store.claim(id, claim, /* @__PURE__ */ new Date(), leaseMs)),
|
|
947
961
|
release: (claim) => call("release", () => store.release(id, claim, /* @__PURE__ */ new Date())),
|
|
948
|
-
complete: (claim, result) => call("complete", () => store.complete(id, claim, result, /* @__PURE__ */ new Date())),
|
|
962
|
+
complete: (claim, result, replace) => call("complete", () => store.complete(id, claim, result, /* @__PURE__ */ new Date(), { replace })),
|
|
949
963
|
end: (from, to) => call("end", () => store.end(id, from, to, /* @__PURE__ */ new Date()))
|
|
950
964
|
};
|
|
951
965
|
}
|
|
966
|
+
const URL_LIFETIME_MS = 9e5;
|
|
967
|
+
const MIN_GRACE_MS = URL_LIFETIME_MS;
|
|
968
|
+
function readGrace(graceMs, what) {
|
|
969
|
+
const value = graceMs ?? 36e5;
|
|
970
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value >= MIN_GRACE_MS) return ok(value);
|
|
971
|
+
return fail(invalidInput(`graceMs must be at least ${MIN_GRACE_MS} (15 minutes), ${what}.`));
|
|
972
|
+
}
|
|
973
|
+
async function removeLeftovers(config, record) {
|
|
974
|
+
const { storage } = config;
|
|
975
|
+
const checkedAt = /* @__PURE__ */ new Date();
|
|
976
|
+
if (record.uploadId && record.completedAt === null) {
|
|
977
|
+
const aborted = await storage.abortMultipart(record.key, record.uploadId);
|
|
978
|
+
if (aborted.error) return aborted;
|
|
979
|
+
}
|
|
980
|
+
const head = await storage.head(record.key);
|
|
981
|
+
if (head.error) return head;
|
|
982
|
+
if (head.data?.metadata["cloudrop-id"] !== record.id) return ok(checkedAt);
|
|
983
|
+
const removed = await storage.delete(record.key);
|
|
984
|
+
return removed.error ? removed : ok(checkedAt);
|
|
985
|
+
}
|
|
986
|
+
function settledAt(record, graceMs) {
|
|
987
|
+
const leftPending = (record.claimedAt ?? record.endedAt ?? record.expiresAt).getTime();
|
|
988
|
+
return Math.min(record.expiresAt.getTime(), leftPending + URL_LIFETIME_MS) + graceMs;
|
|
989
|
+
}
|
|
990
|
+
async function removeDeleted(config, store, record) {
|
|
991
|
+
const checked = await removeLeftovers(config, record);
|
|
992
|
+
if (checked.error) return checked;
|
|
993
|
+
const checkedAt = checked.data;
|
|
994
|
+
if (checkedAt.getTime() < settledAt(record, 36e5)) return ok(void 0);
|
|
995
|
+
return storeCall(config.logger, `mark file ${record.id} cleaned`, () => store.cleaned(record.id, checkedAt));
|
|
996
|
+
}
|
|
952
997
|
// src/server/plugins.ts
|
|
953
998
|
function refusal(plugin, hook, value) {
|
|
954
999
|
if (value === void 0 || value instanceof CloudropError) return value;
|
|
@@ -997,7 +1042,7 @@ async function readOwner(route, context) {
|
|
|
997
1042
|
if (!route.definition.owner) return null;
|
|
998
1043
|
const owner = await route.definition.owner(context);
|
|
999
1044
|
if (owner === null) return null;
|
|
1000
|
-
if (typeof owner !== "string" || owner.length < 1 || owner.length > 255) throw new TypeError(`Route "${route.name}": owner must return null or a string of 1 to 255 characters.`);
|
|
1045
|
+
if (typeof owner !== "string" || owner.length < 1 || owner.length > 255 || owner.includes("\0")) throw new TypeError(`Route "${route.name}": owner must return null or a string of 1 to 255 characters without NUL (U+0000).`);
|
|
1001
1046
|
return owner;
|
|
1002
1047
|
}
|
|
1003
1048
|
const quotaUnavailable = () => new CloudropError("SERVER_ERROR", "The server could not check your storage limit. Try again in a moment.", { retryable: true });
|
|
@@ -1058,47 +1103,233 @@ async function releaseUnstarted(config, shared, unstarted, refusals) {
|
|
|
1058
1103
|
await runOnAbort(plugins, abortContexts(shared, overQuota, "rejected"), logger);
|
|
1059
1104
|
await runOnAbort(plugins, abortContexts(shared, failed, "failed"), logger);
|
|
1060
1105
|
}
|
|
1106
|
+
// src/server/owner-files.ts
|
|
1107
|
+
const DEFAULT_LIMIT$1 = 100;
|
|
1108
|
+
const MAX_LIMIT$1 = 1e3;
|
|
1109
|
+
const LAST_TIME_MS = 0xe677d21fdbff;
|
|
1110
|
+
const isOwnerOrId = (value) => typeof value === "string" && value !== "" && value.length <= 255 && !value.includes("\0");
|
|
1111
|
+
const badOwner = () => fail(invalidInput(`Pass the owner, a string of 1 to 255 characters without NUL (U+0000), as owner() returned it.`));
|
|
1112
|
+
function isStoreCursor(cursor) {
|
|
1113
|
+
const match = /^(\d{1,15}):(.+)$/s.exec(cursor);
|
|
1114
|
+
return match !== null && Number(match[1]) <= LAST_TIME_MS && isOwnerOrId(match[2]);
|
|
1115
|
+
}
|
|
1116
|
+
function readOptions$2(config, given) {
|
|
1117
|
+
if (!isPlainObject(given)) return fail(invalidInput("files takes { owner }, and optionally route, limit and cursor."));
|
|
1118
|
+
const { owner, route, limit = DEFAULT_LIMIT$1, cursor } = given;
|
|
1119
|
+
if (!isOwnerOrId(owner)) return badOwner();
|
|
1120
|
+
if (route !== void 0 && (typeof route !== "string" || !config.routes.has(route))) {
|
|
1121
|
+
const names = [...config.routes.keys()].map((name) => JSON.stringify(name)).join(", ");
|
|
1122
|
+
return fail(invalidInput(`route must be one of this server's routes: ${names}.`));
|
|
1123
|
+
}
|
|
1124
|
+
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT$1) return fail(invalidInput(`limit must be a whole number from 1 to ${MAX_LIMIT$1}.`));
|
|
1125
|
+
if (cursor !== void 0 && (typeof cursor !== "string" || !isStoreCursor(cursor))) return fail(invalidInput("cursor must be one an earlier page of files returned. Leave it out for the first page."));
|
|
1126
|
+
return ok({
|
|
1127
|
+
owner,
|
|
1128
|
+
route: route ?? null,
|
|
1129
|
+
limit,
|
|
1130
|
+
cursor: cursor ?? null
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
async function listFiles(config, store, given) {
|
|
1134
|
+
const options = readOptions$2(config, given);
|
|
1135
|
+
if (options.error) return options;
|
|
1136
|
+
const { owner } = options.data;
|
|
1137
|
+
const page = await storeCall(config.logger, `list the files of ${owner}`, () => store.list({
|
|
1138
|
+
...options.data,
|
|
1139
|
+
statuses: ["completed"]
|
|
1140
|
+
}));
|
|
1141
|
+
return page.error ? page : ok({
|
|
1142
|
+
items: page.data.records,
|
|
1143
|
+
cursor: page.data.cursor
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
// src/server/erase.ts
|
|
1147
|
+
const PAGE = 100;
|
|
1148
|
+
const LIVE = [
|
|
1149
|
+
"pending",
|
|
1150
|
+
"completing",
|
|
1151
|
+
"completed"
|
|
1152
|
+
];
|
|
1153
|
+
const ENDED = [
|
|
1154
|
+
"aborted",
|
|
1155
|
+
"rejected",
|
|
1156
|
+
"failed",
|
|
1157
|
+
"expired",
|
|
1158
|
+
"deleted"
|
|
1159
|
+
];
|
|
1160
|
+
const EVERY = [...LIVE, ...ENDED];
|
|
1161
|
+
function readOptions$1(given) {
|
|
1162
|
+
const options = given ?? {};
|
|
1163
|
+
if (!isPlainObject(options)) return fail(invalidInput("eraseOwner takes options such as { graceMs: 3600000 }."));
|
|
1164
|
+
const { signal } = options;
|
|
1165
|
+
if (signal !== void 0 && !(signal instanceof AbortSignal)) return fail(invalidInput("signal must be an AbortSignal, such as AbortSignal.timeout(60_000)."));
|
|
1166
|
+
const graceMs = readGrace(options.graceMs, "the time a PUT already under way gets to land after its URL ran out");
|
|
1167
|
+
return graceMs.error ? graceMs : ok({
|
|
1168
|
+
graceMs: graceMs.data,
|
|
1169
|
+
signal
|
|
1170
|
+
});
|
|
1171
|
+
}
|
|
1172
|
+
async function deleteCompleted({ config, store, report }, record) {
|
|
1173
|
+
const ended = await storeCall(config.logger, `delete file ${record.id}`, () => store.end(record.id, "completed", "deleted", /* @__PURE__ */ new Date()));
|
|
1174
|
+
if (ended.error) report.failed += 1;
|
|
1175
|
+
else if (ended.data.changed) report.deleted += 1;
|
|
1176
|
+
}
|
|
1177
|
+
async function checkEnded({ config, store, graceMs, report }, record) {
|
|
1178
|
+
const settled = settledAt(record, graceMs);
|
|
1179
|
+
if (record.status !== "failed" && record.cleanedAt !== null && record.cleanedAt.getTime() >= settled) return;
|
|
1180
|
+
const checked = await removeLeftovers(config, record);
|
|
1181
|
+
if (checked.error) {
|
|
1182
|
+
config.logger.error(`cloudrop: eraseOwner could not remove what file ${record.id} left in the bucket. Run eraseOwner again.`, checked.error);
|
|
1183
|
+
report.failed += 1;
|
|
1184
|
+
return;
|
|
1185
|
+
}
|
|
1186
|
+
const checkedAt = checked.data;
|
|
1187
|
+
if (checkedAt.getTime() < settled) {
|
|
1188
|
+
report.waiting += 1;
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
if (record.cleanedAt === null) {
|
|
1192
|
+
if ((await storeCall(config.logger, `mark file ${record.id} cleaned`, () => store.cleaned(record.id, checkedAt))).error) {
|
|
1193
|
+
report.failed += 1;
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
report.cleaned += 1;
|
|
1198
|
+
}
|
|
1199
|
+
async function eachRow(eraser, statuses, handle) {
|
|
1200
|
+
const { config, store, owner, signal } = eraser;
|
|
1201
|
+
let cursor = null;
|
|
1202
|
+
do {
|
|
1203
|
+
if (signal?.aborted) return ok(false);
|
|
1204
|
+
const page = await storeCall(config.logger, `list the ${statuses.join(" or ")} files of ${owner}`, () => store.list({
|
|
1205
|
+
owner,
|
|
1206
|
+
statuses,
|
|
1207
|
+
route: null,
|
|
1208
|
+
limit: PAGE,
|
|
1209
|
+
cursor
|
|
1210
|
+
}));
|
|
1211
|
+
if (page.error) return page;
|
|
1212
|
+
for (const record of page.data.records) {
|
|
1213
|
+
if (signal?.aborted) return ok(false);
|
|
1214
|
+
await handle(eraser, record);
|
|
1215
|
+
}
|
|
1216
|
+
cursor = page.data.cursor;
|
|
1217
|
+
} while (cursor !== null);
|
|
1218
|
+
return ok(true);
|
|
1219
|
+
}
|
|
1220
|
+
const wait = async ({ report }) => {
|
|
1221
|
+
report.waiting += 1;
|
|
1222
|
+
};
|
|
1223
|
+
async function runPasses(eraser, passes) {
|
|
1224
|
+
for (const [statuses, handle] of passes) {
|
|
1225
|
+
const finished = await eachRow(eraser, statuses, handle);
|
|
1226
|
+
if (finished.error || !finished.data) return finished;
|
|
1227
|
+
}
|
|
1228
|
+
return ok(true);
|
|
1229
|
+
}
|
|
1230
|
+
async function lastReads(eraser, startedAt) {
|
|
1231
|
+
const { config, store, owner, report } = eraser;
|
|
1232
|
+
if (report.waiting > 0) return ok(true);
|
|
1233
|
+
const newest = await storeCall(config.logger, `list the newest file of ${owner}`, () => store.list({
|
|
1234
|
+
owner,
|
|
1235
|
+
statuses: EVERY,
|
|
1236
|
+
route: null,
|
|
1237
|
+
limit: 1,
|
|
1238
|
+
cursor: null
|
|
1239
|
+
}));
|
|
1240
|
+
if (newest.error) return newest;
|
|
1241
|
+
if ((newest.data.records[0]?.createdAt.getTime() ?? 0) < startedAt) return eachRow(eraser, LIVE, wait);
|
|
1242
|
+
report.waiting += 1;
|
|
1243
|
+
return ok(true);
|
|
1244
|
+
}
|
|
1245
|
+
async function eraseOwner(config, store, owner, given) {
|
|
1246
|
+
if (!isOwnerOrId(owner)) return badOwner();
|
|
1247
|
+
const options = readOptions$1(given);
|
|
1248
|
+
if (options.error) return options;
|
|
1249
|
+
const report = {
|
|
1250
|
+
revoked: 0,
|
|
1251
|
+
deleted: 0,
|
|
1252
|
+
cleaned: 0,
|
|
1253
|
+
failed: 0,
|
|
1254
|
+
waiting: 0,
|
|
1255
|
+
done: false
|
|
1256
|
+
};
|
|
1257
|
+
if (options.data.signal?.aborted) return ok(report);
|
|
1258
|
+
const startedAt = Date.now();
|
|
1259
|
+
const revoked = await storeCall(config.logger, `revoke the uploads of ${owner}`, () => store.revoke(owner, /* @__PURE__ */ new Date()));
|
|
1260
|
+
if (revoked.error) return revoked;
|
|
1261
|
+
report.revoked = revoked.data;
|
|
1262
|
+
const eraser = {
|
|
1263
|
+
config,
|
|
1264
|
+
store,
|
|
1265
|
+
owner,
|
|
1266
|
+
...options.data,
|
|
1267
|
+
report
|
|
1268
|
+
};
|
|
1269
|
+
const ran = await runPasses(eraser, [
|
|
1270
|
+
[["completed"], deleteCompleted],
|
|
1271
|
+
[LIVE, wait],
|
|
1272
|
+
...ENDED.map((status) => [[status], checkEnded])
|
|
1273
|
+
]);
|
|
1274
|
+
const finished = ran.error || !ran.data ? ran : await lastReads(eraser, startedAt);
|
|
1275
|
+
if (finished.error) return finished;
|
|
1276
|
+
return ok({
|
|
1277
|
+
...report,
|
|
1278
|
+
done: finished.data && report.failed === 0 && report.waiting === 0
|
|
1279
|
+
});
|
|
1280
|
+
}
|
|
1061
1281
|
// src/server/sweep.ts
|
|
1062
1282
|
const DEFAULT_LIMIT = 100;
|
|
1063
1283
|
const MAX_LIMIT = 1e3;
|
|
1064
|
-
const
|
|
1065
|
-
|
|
1066
|
-
const
|
|
1284
|
+
const CURSOR = /^(expire|cleanup):(.*)$/s;
|
|
1285
|
+
function readCursor(cursor) {
|
|
1286
|
+
const match = CURSOR.exec(cursor);
|
|
1287
|
+
if (!match || match[2] && !isStoreCursor(match[2])) return null;
|
|
1288
|
+
return {
|
|
1289
|
+
pass: match[1],
|
|
1290
|
+
cursor: match[2] || null
|
|
1291
|
+
};
|
|
1292
|
+
}
|
|
1293
|
+
function readAll(options) {
|
|
1294
|
+
const { all = false, signal } = options;
|
|
1295
|
+
if (typeof all !== "boolean") return fail(invalidInput("all must be true or false."));
|
|
1296
|
+
if (signal !== void 0 && !(signal instanceof AbortSignal)) return fail(invalidInput("signal must be an AbortSignal, such as AbortSignal.timeout(25_000)."));
|
|
1297
|
+
if (signal !== void 0 && !all) return fail(invalidInput("signal stops a sweep between pages, so it needs all: true. Pass both, or neither."));
|
|
1298
|
+
return ok({
|
|
1299
|
+
all,
|
|
1300
|
+
signal
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1067
1303
|
function readOptions(given) {
|
|
1068
1304
|
const options = given ?? {};
|
|
1069
1305
|
if (!isPlainObject(options)) return fail(invalidInput("sweep takes options such as { limit: 100 }."));
|
|
1070
|
-
const { limit = DEFAULT_LIMIT,
|
|
1306
|
+
const { limit = DEFAULT_LIMIT, cursor } = options;
|
|
1071
1307
|
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) return fail(invalidInput(`limit must be a whole number from 1 to ${MAX_LIMIT}.`));
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1308
|
+
const graceMs = readGrace(options.graceMs, "the life of an upload URL, so no upload can start after a row's deadline once it may be cleaned up");
|
|
1309
|
+
if (graceMs.error) return graceMs;
|
|
1310
|
+
const all = readAll(options);
|
|
1311
|
+
if (all.error) return all;
|
|
1312
|
+
const start = cursor === void 0 ? {
|
|
1313
|
+
pass: "expire",
|
|
1314
|
+
cursor: null
|
|
1315
|
+
} : readCursor(String(cursor));
|
|
1316
|
+
if (!start || cursor !== void 0 && typeof cursor !== "string") return fail(invalidInput("cursor must be one an earlier sweep returned. Leave it out to start over."));
|
|
1079
1317
|
return ok({
|
|
1080
1318
|
limit,
|
|
1081
|
-
graceMs,
|
|
1082
|
-
|
|
1083
|
-
|
|
1319
|
+
graceMs: graceMs.data,
|
|
1320
|
+
...all.data,
|
|
1321
|
+
...start
|
|
1084
1322
|
});
|
|
1085
1323
|
}
|
|
1086
|
-
async function
|
|
1087
|
-
const
|
|
1088
|
-
if (
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
const head = await storage.head(record.key);
|
|
1093
|
-
if (head.error) return head;
|
|
1094
|
-
if (head.data?.metadata["cloudrop-id"] !== record.id) return ok(void 0);
|
|
1095
|
-
return storage.delete(record.key);
|
|
1324
|
+
async function removeAndMark({ config, store, graceMs }, record) {
|
|
1325
|
+
const checked = await removeLeftovers(config, record);
|
|
1326
|
+
if (checked.error) throw checked.error;
|
|
1327
|
+
if (checked.data.getTime() < settledAt(record, graceMs)) return false;
|
|
1328
|
+
await store.cleaned(record.id, checked.data);
|
|
1329
|
+
return true;
|
|
1096
1330
|
}
|
|
1097
|
-
async function
|
|
1098
|
-
const
|
|
1099
|
-
if (removed.error) throw removed.error;
|
|
1100
|
-
}
|
|
1101
|
-
async function expire({ config, store, report }, record) {
|
|
1331
|
+
async function expire(sweeper, record) {
|
|
1332
|
+
const { store, report } = sweeper;
|
|
1102
1333
|
if (record.status === "completing") {
|
|
1103
1334
|
const ended = await store.end(record.id, "completing", "failed", /* @__PURE__ */ new Date(), { cleaned: true });
|
|
1104
1335
|
if (ended.changed) report.reported.push(ended.record);
|
|
@@ -1106,14 +1337,11 @@ async function expire({ config, store, report }, record) {
|
|
|
1106
1337
|
}
|
|
1107
1338
|
const ended = await store.end(record.id, "pending", "expired", /* @__PURE__ */ new Date());
|
|
1108
1339
|
if (!ended.changed) return;
|
|
1109
|
-
await
|
|
1110
|
-
await store.cleaned(record.id, /* @__PURE__ */ new Date());
|
|
1340
|
+
await removeAndMark(sweeper, ended.record);
|
|
1111
1341
|
report.expired += 1;
|
|
1112
1342
|
}
|
|
1113
|
-
async function cleanUp(
|
|
1114
|
-
await
|
|
1115
|
-
await store.cleaned(record.id, /* @__PURE__ */ new Date());
|
|
1116
|
-
report.cleaned += 1;
|
|
1343
|
+
async function cleanUp(sweeper, record) {
|
|
1344
|
+
if (await removeAndMark(sweeper, record)) sweeper.report.cleaned += 1;
|
|
1117
1345
|
}
|
|
1118
1346
|
async function sweepPage(sweeper, pass, page) {
|
|
1119
1347
|
const { config, store, report } = sweeper;
|
|
@@ -1148,10 +1376,8 @@ function listingFailed(report, handled, error) {
|
|
|
1148
1376
|
failed: report.failed + 1
|
|
1149
1377
|
}, null);
|
|
1150
1378
|
}
|
|
1151
|
-
async function
|
|
1152
|
-
const
|
|
1153
|
-
if (options.error) return options;
|
|
1154
|
-
const { limit, graceMs, pass } = options.data;
|
|
1379
|
+
async function sweepOnce(config, store, run) {
|
|
1380
|
+
const { limit, graceMs, pass } = run;
|
|
1155
1381
|
const report = {
|
|
1156
1382
|
expired: 0,
|
|
1157
1383
|
cleaned: 0,
|
|
@@ -1162,11 +1388,12 @@ async function sweep(config, store, given) {
|
|
|
1162
1388
|
const sweeper = {
|
|
1163
1389
|
config,
|
|
1164
1390
|
store,
|
|
1165
|
-
report
|
|
1391
|
+
report,
|
|
1392
|
+
graceMs
|
|
1166
1393
|
};
|
|
1167
1394
|
const now = Date.now();
|
|
1168
1395
|
let budget = limit;
|
|
1169
|
-
let cursor =
|
|
1396
|
+
let cursor = run.cursor;
|
|
1170
1397
|
if (pass === "expire") {
|
|
1171
1398
|
const page = await sweepPage(sweeper, "expire", {
|
|
1172
1399
|
before: now - graceMs,
|
|
@@ -1187,11 +1414,38 @@ async function sweep(config, store, given) {
|
|
|
1187
1414
|
if (page.error) return listingFailed(report, limit - budget, page.error);
|
|
1188
1415
|
return stoppedAt(report, page.data.cursor === null ? null : `cleanup:${page.data.cursor}`);
|
|
1189
1416
|
}
|
|
1417
|
+
async function sweepAll(config, store, run) {
|
|
1418
|
+
const total = {
|
|
1419
|
+
expired: 0,
|
|
1420
|
+
cleaned: 0,
|
|
1421
|
+
failed: 0,
|
|
1422
|
+
reported: [],
|
|
1423
|
+
cursor: null
|
|
1424
|
+
};
|
|
1425
|
+
let position = run;
|
|
1426
|
+
for (let pages = 0; position !== null; pages++) {
|
|
1427
|
+
if (run.signal?.aborted) return stoppedAt(total, `${position.pass}:${position.cursor ?? ""}`);
|
|
1428
|
+
const page = await sweepOnce(config, store, {
|
|
1429
|
+
...run,
|
|
1430
|
+
...position
|
|
1431
|
+
});
|
|
1432
|
+
if (page.error) return pages === 0 ? page : listingFailed(total, 1, page.error);
|
|
1433
|
+
total.expired += page.data.expired;
|
|
1434
|
+
total.cleaned += page.data.cleaned;
|
|
1435
|
+
total.failed += page.data.failed;
|
|
1436
|
+
total.reported.push(...page.data.reported);
|
|
1437
|
+
position = page.data.cursor === null ? null : readCursor(page.data.cursor);
|
|
1438
|
+
}
|
|
1439
|
+
return ok(total);
|
|
1440
|
+
}
|
|
1441
|
+
async function sweep(config, store, given) {
|
|
1442
|
+
const options = readOptions(given);
|
|
1443
|
+
if (options.error) return options;
|
|
1444
|
+
return options.data.all ? sweepAll(config, store, options.data) : sweepOnce(config, store, options.data);
|
|
1445
|
+
}
|
|
1190
1446
|
// src/server/store-api.ts
|
|
1191
1447
|
const OWNERS_PAGE = 100;
|
|
1192
|
-
const isOwnerOrId = (value) => typeof value === "string" && value !== "" && value.length <= 255;
|
|
1193
1448
|
const badId = () => fail(invalidInput("Pass the file id, as returned with the uploaded file."));
|
|
1194
|
-
const badOwner = () => fail(invalidInput(`Pass the owner, a string of 1 to 255 characters, as owner() returned it.`));
|
|
1195
1449
|
const notCompleted = () => new CloudropError("NOT_FOUND", "No completed file has this id. deleteFile deletes only files whose upload completed.");
|
|
1196
1450
|
async function deleteFile(config, store, id) {
|
|
1197
1451
|
const ended = await storeCall(config.logger, `delete file ${id}`, () => store.end(id, "completed", "deleted", /* @__PURE__ */ new Date()));
|
|
@@ -1199,10 +1453,8 @@ async function deleteFile(config, store, id) {
|
|
|
1199
1453
|
const { changed, record } = ended.data;
|
|
1200
1454
|
if (record?.status !== "deleted") return fail(notCompleted());
|
|
1201
1455
|
if (!changed && record.cleanedAt !== null) return ok({ id });
|
|
1202
|
-
const removed = await
|
|
1203
|
-
|
|
1204
|
-
const marked = await storeCall(config.logger, `mark file ${id} cleaned`, () => store.cleaned(id, /* @__PURE__ */ new Date()));
|
|
1205
|
-
return marked.error ? marked : ok({ id });
|
|
1456
|
+
const removed = await removeDeleted(config, store, record);
|
|
1457
|
+
return removed.error ? removed : ok({ id });
|
|
1206
1458
|
}
|
|
1207
1459
|
async function reconcile(config, store, given) {
|
|
1208
1460
|
const options = given ?? {};
|
|
@@ -1239,6 +1491,7 @@ function storeApi(config, whenReady) {
|
|
|
1239
1491
|
const withStore = (name, call) => whenReady(() => config.store ? call(config.store) : Promise.resolve(fail(configError(`api.${name} needs a database store. Pass store to createCloudrop.`))));
|
|
1240
1492
|
return {
|
|
1241
1493
|
file: (id) => withStore("file", async (store) => isOwnerOrId(id) ? storeCall(logger, `get file ${id}`, () => store.get(id)) : badId()),
|
|
1494
|
+
files: (options) => withStore("files", (store) => listFiles(config, store, options)),
|
|
1242
1495
|
usage: (owner) => withStore("usage", async (store) => isOwnerOrId(owner) ? storeCall(logger, `read the usage of ${owner}`, () => store.usage(owner)) : badOwner()),
|
|
1243
1496
|
revokeUploads: (options) => withStore("revokeUploads", async (store) => {
|
|
1244
1497
|
const owner = isPlainObject(options) ? options.owner : void 0;
|
|
@@ -1247,6 +1500,7 @@ function storeApi(config, whenReady) {
|
|
|
1247
1500
|
return revoked.error ? revoked : ok({ revoked: revoked.data });
|
|
1248
1501
|
}),
|
|
1249
1502
|
deleteFile: (id) => withStore("deleteFile", async (store) => isOwnerOrId(id) ? deleteFile(config, store, id) : badId()),
|
|
1503
|
+
eraseOwner: (owner, options) => withStore("eraseOwner", (store) => eraseOwner(config, store, owner, options)),
|
|
1250
1504
|
reconcile: (options) => withStore("reconcile", (store) => reconcile(config, store, options)),
|
|
1251
1505
|
sweep: (options) => withStore("sweep", (store) => sweep(config, store, options))
|
|
1252
1506
|
};
|
|
@@ -1277,7 +1531,8 @@ async function getUrl(storage, key, given) {
|
|
|
1277
1531
|
});
|
|
1278
1532
|
}
|
|
1279
1533
|
function pageProblem(keys, prefix, previous) {
|
|
1280
|
-
|
|
1534
|
+
const folder = prefix.normalize("NFC");
|
|
1535
|
+
if (keys.some((key) => !key.normalize("NFC").startsWith(folder))) return new CloudropError("STORAGE_ERROR", "The storage listed keys outside the prefix, so nothing more was deleted.");
|
|
1281
1536
|
if (keys.some((key) => previous.has(key))) return new CloudropError("STORAGE_ERROR", "Some keys were still listed after they were deleted. Retry deletePrefix.", { retryable: true });
|
|
1282
1537
|
}
|
|
1283
1538
|
async function deletePrefix(storage, prefix) {
|
|
@@ -1453,14 +1708,10 @@ async function readJson(request) {
|
|
|
1453
1708
|
const chunks = [];
|
|
1454
1709
|
let length = 0;
|
|
1455
1710
|
try {
|
|
1456
|
-
const
|
|
1457
|
-
|
|
1458
|
-
length
|
|
1459
|
-
|
|
1460
|
-
await reader?.cancel();
|
|
1461
|
-
return fail(bodyTooLarge());
|
|
1462
|
-
}
|
|
1463
|
-
chunks.push(chunk.value);
|
|
1711
|
+
for await (const chunk of request.body ?? []) {
|
|
1712
|
+
length += chunk.length;
|
|
1713
|
+
if (length > 65536) return fail(bodyTooLarge());
|
|
1714
|
+
chunks.push(chunk);
|
|
1464
1715
|
}
|
|
1465
1716
|
} catch {
|
|
1466
1717
|
return fail(invalidInput("The request body could not be read. Send the request again."));
|
|
@@ -1583,15 +1834,24 @@ function completeContext(request, token, file) {
|
|
|
1583
1834
|
metadata: token.metadata
|
|
1584
1835
|
};
|
|
1585
1836
|
}
|
|
1586
|
-
function
|
|
1587
|
-
|
|
1837
|
+
async function runOnComplete(logger, route, context) {
|
|
1838
|
+
const { onComplete } = route.definition;
|
|
1839
|
+
let data;
|
|
1588
1840
|
try {
|
|
1589
|
-
|
|
1841
|
+
data = await onComplete?.(context);
|
|
1590
1842
|
} catch (error) {
|
|
1591
|
-
logger.error(`cloudrop: onComplete for route "${route}"
|
|
1843
|
+
logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1844
|
+
return fail(unfinished());
|
|
1845
|
+
}
|
|
1846
|
+
try {
|
|
1847
|
+
return ok({
|
|
1848
|
+
data,
|
|
1849
|
+
json: JSON.stringify(data)
|
|
1850
|
+
});
|
|
1851
|
+
} catch (error) {
|
|
1852
|
+
logger.error(`cloudrop: onComplete for route "${route.name}" returned a value JSON cannot encode (a BigInt or a cycle).`, error);
|
|
1592
1853
|
return fail(unfinished());
|
|
1593
1854
|
}
|
|
1594
|
-
return ok(json);
|
|
1595
1855
|
}
|
|
1596
1856
|
// src/server/complete-verify.ts
|
|
1597
1857
|
const DISCARDED = "The uploaded file does not match what was declared, so it was discarded. Upload it again.";
|
|
@@ -1698,31 +1958,23 @@ async function refuse(completion, refusal) {
|
|
|
1698
1958
|
if (ended.data.changed) return cleanUpRefused(completion.config, completion.token, refusal, { rowEnded: true });
|
|
1699
1959
|
return settled(ended.data.record, completion.file);
|
|
1700
1960
|
}
|
|
1701
|
-
async function
|
|
1702
|
-
const
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
} catch (error) {
|
|
1710
|
-
config.logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1711
|
-
return fail(unfinished());
|
|
1712
|
-
}
|
|
1713
|
-
if (result === void 0 || result === null) return ok({
|
|
1961
|
+
async function resultToStore({ request, config, token, route, file }, extra) {
|
|
1962
|
+
const ran = await runOnComplete(config.logger, route, {
|
|
1963
|
+
...completeContext(request, token, file),
|
|
1964
|
+
...extra
|
|
1965
|
+
});
|
|
1966
|
+
if (ran.error) return ran;
|
|
1967
|
+
const { data, json } = ran.data;
|
|
1968
|
+
if (data === void 0 || data === null || json === void 0) return ok({
|
|
1714
1969
|
data: null,
|
|
1715
1970
|
json: null
|
|
1716
1971
|
});
|
|
1717
|
-
|
|
1718
|
-
if (encoded.error) return encoded;
|
|
1719
|
-
const json = encoded.data ?? null;
|
|
1720
|
-
if (json !== null && new TextEncoder().encode(json).length > MAX_RESULT_JSON_BYTES) {
|
|
1972
|
+
if (new TextEncoder().encode(json).length > MAX_RESULT_JSON_BYTES) {
|
|
1721
1973
|
config.logger.error(`cloudrop: onComplete for route "${route.name}" returned more than 16 KiB of JSON. Return ids, not records.`);
|
|
1722
1974
|
return fail(unfinished());
|
|
1723
1975
|
}
|
|
1724
1976
|
return ok({
|
|
1725
|
-
data:
|
|
1977
|
+
data: JSON.parse(json),
|
|
1726
1978
|
json
|
|
1727
1979
|
});
|
|
1728
1980
|
}
|
|
@@ -1738,7 +1990,7 @@ async function withLease(completion) {
|
|
|
1738
1990
|
answer: claimAnswer(claimed.data, file),
|
|
1739
1991
|
recorded: false
|
|
1740
1992
|
};
|
|
1741
|
-
const ran = await
|
|
1993
|
+
const ran = await resultToStore(completion, {});
|
|
1742
1994
|
if (ran.error) {
|
|
1743
1995
|
await calls.release(claim);
|
|
1744
1996
|
return {
|
|
@@ -1746,7 +1998,7 @@ async function withLease(completion) {
|
|
|
1746
1998
|
recorded: false
|
|
1747
1999
|
};
|
|
1748
2000
|
}
|
|
1749
|
-
const done = await calls.complete(claim, ran.data.json);
|
|
2001
|
+
const done = await calls.complete(claim, ran.data.json, completion.route.replace);
|
|
1750
2002
|
if (done.error) return {
|
|
1751
2003
|
answer: done,
|
|
1752
2004
|
recorded: false
|
|
@@ -1760,7 +2012,8 @@ async function withLease(completion) {
|
|
|
1760
2012
|
file,
|
|
1761
2013
|
data: ran.data.data
|
|
1762
2014
|
}),
|
|
1763
|
-
recorded: true
|
|
2015
|
+
recorded: true,
|
|
2016
|
+
replaced: done.data.replaced
|
|
1764
2017
|
};
|
|
1765
2018
|
}
|
|
1766
2019
|
async function inTransaction(completion, transaction) {
|
|
@@ -1789,16 +2042,18 @@ async function inTransaction(completion, transaction) {
|
|
|
1789
2042
|
};
|
|
1790
2043
|
return;
|
|
1791
2044
|
}
|
|
1792
|
-
const ran = await
|
|
2045
|
+
const ran = await resultToStore(completion, storeOfRoute(completion.route.definition) ? { tx } : {});
|
|
1793
2046
|
if (ran.error) return rollBack(ran);
|
|
1794
|
-
const
|
|
2047
|
+
const replace = completion.route.replace;
|
|
2048
|
+
const done = await store.complete(token.id, claim, ran.data.json, /* @__PURE__ */ new Date(), { replace });
|
|
1795
2049
|
if (!done.changed) return rollBack(lostClaim(done.record, file));
|
|
1796
2050
|
attempt = {
|
|
1797
2051
|
answer: ok({
|
|
1798
2052
|
file,
|
|
1799
2053
|
data: ran.data.data
|
|
1800
2054
|
}),
|
|
1801
|
-
recorded: true
|
|
2055
|
+
recorded: true,
|
|
2056
|
+
replaced: done.replaced
|
|
1802
2057
|
};
|
|
1803
2058
|
});
|
|
1804
2059
|
} catch (error) {
|
|
@@ -1811,6 +2066,12 @@ async function inTransaction(completion, transaction) {
|
|
|
1811
2066
|
}
|
|
1812
2067
|
return attempt;
|
|
1813
2068
|
}
|
|
2069
|
+
async function removeReplaced({ config }, store, replaced) {
|
|
2070
|
+
for (const record of replaced) {
|
|
2071
|
+
const removed = await removeDeleted(config, store, record);
|
|
2072
|
+
if (removed.error) config.logger.error(`cloudrop: could not remove the replaced file ${record.id} from the bucket. The sweep removes it later.`, removed.error);
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
1814
2075
|
async function runAfterComplete({ request, config, token, route, file }, data) {
|
|
1815
2076
|
try {
|
|
1816
2077
|
await route.definition.afterComplete?.({
|
|
@@ -1839,9 +2100,12 @@ async function completeStored({ request, config }, store, token, route) {
|
|
|
1839
2100
|
const checked = stored.kind === "verified" ? await checkAfterVerify(request, config, token, completion.file) : stored;
|
|
1840
2101
|
if (checked.kind === "failed") return fail(checked.error);
|
|
1841
2102
|
if (checked.kind === "refused") return refuse(completion, checked);
|
|
1842
|
-
const
|
|
1843
|
-
if (recorded)
|
|
1844
|
-
|
|
2103
|
+
const attempt = store.transaction ? await inTransaction(completion, store.transaction.bind(store)) : await withLease(completion);
|
|
2104
|
+
if (attempt.recorded) {
|
|
2105
|
+
await removeReplaced(completion, store, attempt.replaced);
|
|
2106
|
+
await runAfterComplete(completion, attempt.answer.data?.data);
|
|
2107
|
+
}
|
|
2108
|
+
return attempt.answer;
|
|
1845
2109
|
}
|
|
1846
2110
|
// src/server/complete.ts
|
|
1847
2111
|
async function handleComplete(input) {
|
|
@@ -1850,7 +2114,6 @@ async function handleComplete(input) {
|
|
|
1850
2114
|
if (verified.error) return verified;
|
|
1851
2115
|
const { token, route } = verified.data;
|
|
1852
2116
|
if (config.store && token.stored) return completeStored(input, config.store, token, route);
|
|
1853
|
-
const { logger } = config;
|
|
1854
2117
|
const stored = await verifyUpload(config, token);
|
|
1855
2118
|
if (stored.kind === "failed") return fail(stored.error);
|
|
1856
2119
|
if (stored.kind === "refused") return cleanUpRefused(config, token, stored);
|
|
@@ -1858,23 +2121,11 @@ async function handleComplete(input) {
|
|
|
1858
2121
|
const checked = await checkAfterVerify(request, config, token, file);
|
|
1859
2122
|
if (checked.kind === "failed") return fail(checked.error);
|
|
1860
2123
|
if (checked.kind === "refused") return cleanUpRefused(config, token, checked);
|
|
1861
|
-
const
|
|
1862
|
-
if (
|
|
1863
|
-
file,
|
|
1864
|
-
data: null
|
|
1865
|
-
});
|
|
1866
|
-
let data;
|
|
1867
|
-
try {
|
|
1868
|
-
data = await onComplete(completeContext(request, token, file));
|
|
1869
|
-
} catch (error) {
|
|
1870
|
-
logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1871
|
-
return fail(unfinished());
|
|
1872
|
-
}
|
|
1873
|
-
const encoded = encodeResult(logger, route.name, data);
|
|
1874
|
-
if (encoded.error) return encoded;
|
|
2124
|
+
const ran = await runOnComplete(config.logger, route, completeContext(request, token, file));
|
|
2125
|
+
if (ran.error) return ran;
|
|
1875
2126
|
return ok({
|
|
1876
2127
|
file,
|
|
1877
|
-
data: data === void 0 ? null : data
|
|
2128
|
+
data: ran.data.json === void 0 ? null : ran.data.data
|
|
1878
2129
|
});
|
|
1879
2130
|
}
|
|
1880
2131
|
// src/server/sign.ts
|
|
@@ -2270,6 +2521,12 @@ const INVALID_PART_CODES = /* @__PURE__ */ new Set([
|
|
|
2270
2521
|
"InvalidPartOrder",
|
|
2271
2522
|
"EntityTooSmall"
|
|
2272
2523
|
]);
|
|
2524
|
+
const MAX_CLOCK_SKEW_MS = 899e3;
|
|
2525
|
+
const CLOCK_SKEWED = {
|
|
2526
|
+
matches: (_status, storageCode) => storageCode === "RequestTimeTooSkewed",
|
|
2527
|
+
code: "STORAGE_DENIED",
|
|
2528
|
+
message: "This server's clock is more than 15 minutes off from the storage service's clock. Sync the clock (NTP), then retry."
|
|
2529
|
+
};
|
|
2273
2530
|
const RULES = [
|
|
2274
2531
|
{
|
|
2275
2532
|
matches: (status, storageCode) => status === 404 && storageCode === "NoSuchUpload",
|
|
@@ -2301,15 +2558,16 @@ const RULES = [
|
|
|
2301
2558
|
code: "CHECKSUM_MISMATCH",
|
|
2302
2559
|
message: "The uploaded bytes do not match their checksum. Upload them again."
|
|
2303
2560
|
},
|
|
2561
|
+
CLOCK_SKEWED,
|
|
2304
2562
|
{
|
|
2305
|
-
matches: (_status, storageCode) => storageCode === "
|
|
2563
|
+
matches: (_status, storageCode) => storageCode === "ExpiredRequest",
|
|
2306
2564
|
code: "STORAGE_DENIED",
|
|
2307
|
-
message: "
|
|
2565
|
+
message: "The signed request expired. Sign it again and retry."
|
|
2308
2566
|
},
|
|
2309
2567
|
{
|
|
2310
|
-
matches: (
|
|
2568
|
+
matches: (status) => status === 401,
|
|
2311
2569
|
code: "STORAGE_DENIED",
|
|
2312
|
-
message: "The
|
|
2570
|
+
message: "The storage service did not accept these credentials. Check the access key ID and the secret access key."
|
|
2313
2571
|
},
|
|
2314
2572
|
{
|
|
2315
2573
|
matches: (status) => status === 403,
|
|
@@ -2334,10 +2592,10 @@ const RULES = [
|
|
|
2334
2592
|
retryable: true
|
|
2335
2593
|
}
|
|
2336
2594
|
];
|
|
2337
|
-
function storageError(status, body) {
|
|
2595
|
+
function storageError(status, body, clockOffsetMs = 0) {
|
|
2338
2596
|
const rawCode = xmlValue(body, "Code");
|
|
2339
2597
|
const storageCode = rawCode && /^[A-Za-z0-9]{1,64}$/.test(rawCode) ? rawCode : void 0;
|
|
2340
|
-
const rule = RULES.find((candidate) => candidate.matches(status, storageCode));
|
|
2598
|
+
const rule = status === 403 && storageCode === void 0 && Math.abs(clockOffsetMs) > MAX_CLOCK_SKEW_MS ? CLOCK_SKEWED : RULES.find((candidate) => candidate.matches(status, storageCode));
|
|
2341
2599
|
if (rule) return new CloudropError(rule.code, rule.message, {
|
|
2342
2600
|
status,
|
|
2343
2601
|
storageCode,
|
|
@@ -2375,13 +2633,14 @@ async function send(ctx, method, url, options = {}) {
|
|
|
2375
2633
|
const { http, now, credentials, region } = ctx;
|
|
2376
2634
|
url.search = canonicalQuery(url.searchParams);
|
|
2377
2635
|
const { headers = {}, body, timeoutMs = REQUEST_TIMEOUT_MS } = options;
|
|
2636
|
+
const signedAt = now();
|
|
2378
2637
|
const signed = await signRequest({
|
|
2379
2638
|
method,
|
|
2380
2639
|
url,
|
|
2381
2640
|
headers,
|
|
2382
2641
|
credentials,
|
|
2383
2642
|
region,
|
|
2384
|
-
date:
|
|
2643
|
+
date: signedAt,
|
|
2385
2644
|
payloadHash: body instanceof Blob ? UNSIGNED_PAYLOAD : body ? await sha256Hex(body) : EMPTY_SHA256
|
|
2386
2645
|
});
|
|
2387
2646
|
const sent = method === "HEAD" ? {
|
|
@@ -2395,7 +2654,10 @@ async function send(ctx, method, url, options = {}) {
|
|
|
2395
2654
|
body,
|
|
2396
2655
|
signal: AbortSignal.timeout(timeoutMs)
|
|
2397
2656
|
});
|
|
2398
|
-
if (!response.ok)
|
|
2657
|
+
if (!response.ok) {
|
|
2658
|
+
const clockOffsetMs = Date.parse(response.headers.get("date") ?? "") - signedAt.getTime();
|
|
2659
|
+
return fail(storageError(response.status, await response.text(), clockOffsetMs));
|
|
2660
|
+
}
|
|
2399
2661
|
if (headers.range !== void 0 && response.status !== 206) {
|
|
2400
2662
|
await response.body?.cancel();
|
|
2401
2663
|
return fail(new CloudropError("STORAGE_ERROR", "The storage service ignored the byte range. Check that the endpoint supports ranged GET.", { status: response.status }));
|
|
@@ -2424,6 +2686,28 @@ function configurationReadError(error, missingCode, setting) {
|
|
|
2424
2686
|
storageCode: error.storageCode
|
|
2425
2687
|
}));
|
|
2426
2688
|
}
|
|
2689
|
+
const UNREADABLE_XML = /<!\[CDATA\[|<!--|<\/?[A-Za-z_][\w.-]*:/;
|
|
2690
|
+
function readConfiguration(xml, root, setting) {
|
|
2691
|
+
if (new RegExp(`<${root}(?:\\s[^>]*)?/>`).test(xml)) return ok("");
|
|
2692
|
+
const configuration = xmlBlocks(xml, root)[0];
|
|
2693
|
+
if (configuration === void 0) return fail(incompleteResponse(`${setting} configuration`));
|
|
2694
|
+
if (UNREADABLE_XML.test(configuration)) return fail(new CloudropError("STORAGE_ERROR", `The bucket's ${setting} configuration uses XML that cloudrop does not read (CDATA, comments or namespace prefixes). Check its rules by hand.`));
|
|
2695
|
+
return ok(configuration);
|
|
2696
|
+
}
|
|
2697
|
+
function readCorsRule(rule) {
|
|
2698
|
+
const id = xmlText(rule, "ID");
|
|
2699
|
+
const maxAge = xmlValue(rule, "MaxAgeSeconds");
|
|
2700
|
+
const maxAgeSeconds = wholeNumber(maxAge);
|
|
2701
|
+
if (maxAge !== void 0 && maxAgeSeconds === void 0) return fail(incompleteResponse("valid CORS max age"));
|
|
2702
|
+
return ok({
|
|
2703
|
+
...id === void 0 ? {} : { id },
|
|
2704
|
+
allowedOrigins: xmlValues(rule, "AllowedOrigin"),
|
|
2705
|
+
allowedMethods: xmlValues(rule, "AllowedMethod"),
|
|
2706
|
+
allowedHeaders: xmlValues(rule, "AllowedHeader"),
|
|
2707
|
+
exposeHeaders: xmlValues(rule, "ExposeHeader"),
|
|
2708
|
+
...maxAgeSeconds === void 0 ? {} : { maxAgeSeconds }
|
|
2709
|
+
});
|
|
2710
|
+
}
|
|
2427
2711
|
function readLifecycleRule(rule) {
|
|
2428
2712
|
const days = xmlValue(xmlBlocks(rule, "AbortIncompleteMultipartUpload")[0] ?? "", "DaysAfterInitiation");
|
|
2429
2713
|
if (days !== void 0 && !/^\d{1,9}$/.test(days)) return fail(incompleteResponse("valid multipart abort days"));
|
|
@@ -2431,7 +2715,8 @@ function readLifecycleRule(rule) {
|
|
|
2431
2715
|
id: xmlText(rule, "ID") ?? "",
|
|
2432
2716
|
enabled: xmlValue(rule, "Status") === "Enabled",
|
|
2433
2717
|
prefix: xmlText(rule, "Prefix") ?? "",
|
|
2434
|
-
...days === void 0 ? {} : { abortIncompleteMultipartUploadDays: Number(days) }
|
|
2718
|
+
...days === void 0 ? {} : { abortIncompleteMultipartUploadDays: Number(days) },
|
|
2719
|
+
sourceXml: rule
|
|
2435
2720
|
});
|
|
2436
2721
|
}
|
|
2437
2722
|
function bucketSettingsMethods(ctx) {
|
|
@@ -2439,21 +2724,16 @@ function bucketSettingsMethods(ctx) {
|
|
|
2439
2724
|
async getBucketCors() {
|
|
2440
2725
|
const result = await sendForXml(ctx, "GET", new URL(`${ctx.bucketUrl}?cors=`));
|
|
2441
2726
|
if (result.error) return configurationReadError(result.error, "NoSuchCORSConfiguration", "CORS");
|
|
2442
|
-
const configuration =
|
|
2443
|
-
if (configuration
|
|
2444
|
-
return
|
|
2445
|
-
allowedOrigins: xmlValues(rule, "AllowedOrigin"),
|
|
2446
|
-
allowedMethods: xmlValues(rule, "AllowedMethod"),
|
|
2447
|
-
allowedHeaders: xmlValues(rule, "AllowedHeader"),
|
|
2448
|
-
exposeHeaders: xmlValues(rule, "ExposeHeader")
|
|
2449
|
-
})));
|
|
2727
|
+
const configuration = readConfiguration(result.data, "CORSConfiguration", "CORS");
|
|
2728
|
+
if (configuration.error) return configuration;
|
|
2729
|
+
return allOk(xmlBlocks(configuration.data, "CORSRule").map(readCorsRule));
|
|
2450
2730
|
},
|
|
2451
2731
|
async getBucketLifecycle() {
|
|
2452
2732
|
const result = await sendForXml(ctx, "GET", new URL(`${ctx.bucketUrl}?lifecycle=`));
|
|
2453
2733
|
if (result.error) return configurationReadError(result.error, "NoSuchLifecycleConfiguration", "lifecycle");
|
|
2454
|
-
const configuration =
|
|
2455
|
-
if (configuration
|
|
2456
|
-
return allOk(xmlBlocks(configuration, "Rule").map(readLifecycleRule));
|
|
2734
|
+
const configuration = readConfiguration(result.data, "LifecycleConfiguration", "lifecycle");
|
|
2735
|
+
if (configuration.error) return configuration;
|
|
2736
|
+
return allOk(xmlBlocks(configuration.data, "Rule").map(readLifecycleRule));
|
|
2457
2737
|
}
|
|
2458
2738
|
};
|
|
2459
2739
|
}
|
|
@@ -2713,13 +2993,9 @@ function readListedObject(block) {
|
|
|
2713
2993
|
});
|
|
2714
2994
|
}
|
|
2715
2995
|
async function forEachConcurrently(items, limit, task) {
|
|
2716
|
-
|
|
2996
|
+
const queue = items.values();
|
|
2717
2997
|
const worker = async () => {
|
|
2718
|
-
|
|
2719
|
-
const item = items[next];
|
|
2720
|
-
next += 1;
|
|
2721
|
-
await task(item);
|
|
2722
|
-
}
|
|
2998
|
+
for (const item of queue) await task(item);
|
|
2723
2999
|
};
|
|
2724
3000
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
2725
3001
|
}
|
|
@@ -2936,17 +3212,17 @@ function preset(build) {
|
|
|
2936
3212
|
});
|
|
2937
3213
|
return readsEnvironment ? lazyDriver(checked, () => buildFromEnvironment(build)) : checked;
|
|
2938
3214
|
}
|
|
2939
|
-
function buildFromEnvironment(build) {
|
|
3215
|
+
function buildFromEnvironment(build, env = fromEnv) {
|
|
2940
3216
|
const read = [];
|
|
2941
3217
|
try {
|
|
2942
3218
|
return build((variable) => {
|
|
2943
|
-
const value =
|
|
3219
|
+
const value = env(variable);
|
|
2944
3220
|
if (value) read.push(variable);
|
|
2945
3221
|
return value;
|
|
2946
3222
|
});
|
|
2947
3223
|
} catch (error) {
|
|
2948
3224
|
if (!(error instanceof CloudropError)) throw error;
|
|
2949
|
-
const invalid = read.filter((variable) => readSetting(() => build((name, setting) => name === variable ?
|
|
3225
|
+
const invalid = read.filter((variable) => readSetting(() => build((name, setting) => name === variable ? env(name) : STAND_INS[setting])).error);
|
|
2950
3226
|
if (invalid.length === 0) throw error;
|
|
2951
3227
|
throw configError(`${error.message} Environment variables with an invalid value: ${invalid.join(", ")}.`);
|
|
2952
3228
|
}
|
|
@@ -2966,15 +3242,14 @@ function checkAccountId(accountId) {
|
|
|
2966
3242
|
}
|
|
2967
3243
|
return account;
|
|
2968
3244
|
}
|
|
2969
|
-
function
|
|
2970
|
-
const options = given ?? {};
|
|
3245
|
+
function buildR2(options, create) {
|
|
2971
3246
|
const { jurisdiction } = options;
|
|
2972
3247
|
if (jurisdiction !== void 0 && jurisdiction !== "eu" && jurisdiction !== "fedramp") throw configError("The R2 jurisdiction must be \"eu\" or \"fedramp\". Omit it for the default location.");
|
|
2973
|
-
return
|
|
3248
|
+
return (read) => {
|
|
2974
3249
|
const settings = readSettings("R2", options, R2_VARIABLES, read);
|
|
2975
3250
|
const bucket = checkBucket(settings.bucket, "R2");
|
|
2976
3251
|
const account = checkAccountId(settings.accountId);
|
|
2977
|
-
return
|
|
3252
|
+
return create({
|
|
2978
3253
|
endpoint: `https://${jurisdiction ? `${account}.${jurisdiction}` : account}.r2.cloudflarestorage.com`,
|
|
2979
3254
|
region: "auto",
|
|
2980
3255
|
bucket,
|
|
@@ -2986,13 +3261,12 @@ function r2(given) {
|
|
|
2986
3261
|
publicUrl: options.publicUrl,
|
|
2987
3262
|
fetch: options.fetch
|
|
2988
3263
|
});
|
|
2989
|
-
}
|
|
3264
|
+
};
|
|
2990
3265
|
}
|
|
2991
|
-
function
|
|
2992
|
-
|
|
2993
|
-
return preset((read) => {
|
|
3266
|
+
function buildS3(options, create) {
|
|
3267
|
+
return (read) => {
|
|
2994
3268
|
const { endpoint, region, bucket, accessKeyId, secretAccessKey } = readSettings("S3", options, S3_VARIABLES, read);
|
|
2995
|
-
return
|
|
3269
|
+
return create({
|
|
2996
3270
|
endpoint,
|
|
2997
3271
|
region,
|
|
2998
3272
|
bucket: checkBucket(bucket, "S3"),
|
|
@@ -3004,7 +3278,13 @@ function s3(given) {
|
|
|
3004
3278
|
publicUrl: options.publicUrl,
|
|
3005
3279
|
fetch: options.fetch
|
|
3006
3280
|
});
|
|
3007
|
-
}
|
|
3281
|
+
};
|
|
3282
|
+
}
|
|
3283
|
+
function r2(given) {
|
|
3284
|
+
return preset(buildR2(given ?? {}, createS3Driver));
|
|
3285
|
+
}
|
|
3286
|
+
function s3(given) {
|
|
3287
|
+
return preset(buildS3(given ?? {}, createS3Driver));
|
|
3008
3288
|
}
|
|
3009
3289
|
export { CloudropError, allowAnyone, createCloudrop, forbidden, r2, route, s3, unauthorized };
|
|
3010
3290
|
|