cloudrop 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +60 -0
- package/README.md +5 -3
- package/dist/adapters/drizzle.d.ts +1 -1
- package/dist/adapters/drizzle.js +3 -3
- package/dist/adapters/drizzle.js.map +1 -1
- package/dist/adapters/prisma.d.ts +1 -1
- package/dist/adapters/prisma.js +1 -1
- package/dist/adapters/sql.d.ts +1 -1
- package/dist/adapters/sql.js +1 -1
- package/dist/{client-Dj6ON4MN.d.ts → client-CP_ajLF4.d.ts} +43 -25
- package/dist/{client-Y1zUEbCj.js → client-ePVHwvMv.js} +126 -40
- package/dist/client-ePVHwvMv.js.map +1 -0
- package/dist/client.d.ts +3 -3
- package/dist/client.js +1 -1
- package/dist/errors-CUxMmFAe.js.map +1 -1
- package/dist/index.d.ts +67 -6
- package/dist/index.js +352 -113
- package/dist/index.js.map +1 -1
- package/dist/next.d.ts +1 -1
- package/dist/node.d.ts +1 -1
- package/dist/react.d.ts +18 -4
- package/dist/react.js +23 -3
- package/dist/react.js.map +1 -1
- package/dist/{route-CspI1Ypt.js → route-C8-YjzdY.js} +3 -2
- package/dist/{route-CspI1Ypt.js.map → route-C8-YjzdY.js.map} +1 -1
- package/dist/{store-BfCJejGh.d.ts → store-B539nY3J.d.ts} +41 -3
- package/dist/{transactional-DnvnGsVH.js → transactional-CiNwMiDb.js} +215 -105
- package/dist/transactional-CiNwMiDb.js.map +1 -0
- package/dist/{types-BrAx3dA4.d.ts → types-CFfMw-n_.d.ts} +1 -1
- package/package.json +7 -7
- package/dist/client-Y1zUEbCj.js.map +0 -1
- package/dist/transactional-DnvnGsVH.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { a as quotaExceeded, i as ok, n as configError, r as fail, t as CloudropError } from "./errors-CUxMmFAe.js";
|
|
2
2
|
import { n as isPlainObject, t as invalidInput } from "./input-DTC476_I.js";
|
|
3
|
-
import { a as route, c as unauthorized, i as refusalError, n as forbidden, r as isRefusal, s as storeOfRoute, t as allowAnyone } from "./route-
|
|
3
|
+
import { a as route, c as unauthorized, i as refusalError, n as forbidden, r as isRefusal, s as storeOfRoute, t as allowAnyone } from "./route-C8-YjzdY.js";
|
|
4
4
|
import { n as sizeOfPart, t as countParts } from "./part-size-CvMoobfC.js";
|
|
5
5
|
import "./types-BZUOaz_B.js";
|
|
6
6
|
// src/storage/sigv4.ts
|
|
@@ -62,7 +62,7 @@ function encodeKeyPath(key) {
|
|
|
62
62
|
return key.split("/").map(encodeRfc3986).join("/");
|
|
63
63
|
}
|
|
64
64
|
function toAmzDate(date) {
|
|
65
|
-
return date.toISOString().replace(/[-:]
|
|
65
|
+
return date.toISOString().replace(/[-:]|\.\d{3}/g, "");
|
|
66
66
|
}
|
|
67
67
|
function canonicalPath(url) {
|
|
68
68
|
return url.pathname.split("/").map((segment) => encodeRfc3986(decodeURIComponent(segment))).join("/");
|
|
@@ -558,6 +558,10 @@ function readMaxSize(where, definition, storage) {
|
|
|
558
558
|
return maxSize;
|
|
559
559
|
}
|
|
560
560
|
function readMaxFiles(where, definition) {
|
|
561
|
+
if (definition.replace === true) {
|
|
562
|
+
if (definition.maxFiles !== void 0 && definition.maxFiles !== 1) throw configError(`${where}: replace keeps one file per owner, so maxFiles must be 1. Leave maxFiles out.`);
|
|
563
|
+
return 1;
|
|
564
|
+
}
|
|
561
565
|
const maxFiles = definition.maxFiles ?? DEFAULT_MAX_FILES;
|
|
562
566
|
if (!Number.isInteger(maxFiles) || maxFiles < 1 || maxFiles > MAX_FILES_LIMIT) throw configError(`${where}: maxFiles must be a whole number from 1 to ${MAX_FILES_LIMIT}.`);
|
|
563
567
|
return maxFiles;
|
|
@@ -584,6 +588,7 @@ function readRoute(name, route, storage) {
|
|
|
584
588
|
const definition = route?.definition;
|
|
585
589
|
if (typeof definition !== "object" || definition === null) throw configError(`${where} is not a route. Wrap its options in route({ ... }).`);
|
|
586
590
|
if (!isFunction(definition.authorize)) throw configError(`${where} needs authorize. Use authorize: allowAnyone for a route anyone may upload to.`);
|
|
591
|
+
if (definition.replace !== void 0 && typeof definition.replace !== "boolean") throw configError(`${where}: replace must be true or false.`);
|
|
587
592
|
const accept = readAccept(where, definition);
|
|
588
593
|
const maxSize = readMaxSize(where, definition, storage);
|
|
589
594
|
const maxFiles = readMaxFiles(where, definition);
|
|
@@ -594,7 +599,8 @@ function readRoute(name, route, storage) {
|
|
|
594
599
|
accept,
|
|
595
600
|
maxSize,
|
|
596
601
|
maxFiles,
|
|
597
|
-
visibility: readVisibility(where, definition, storage)
|
|
602
|
+
visibility: readVisibility(where, definition, storage),
|
|
603
|
+
replace: definition.replace === true
|
|
598
604
|
};
|
|
599
605
|
}
|
|
600
606
|
// src/server/store-config.ts
|
|
@@ -609,6 +615,7 @@ const STORE_METHODS = [
|
|
|
609
615
|
"end",
|
|
610
616
|
"revoke",
|
|
611
617
|
"stale",
|
|
618
|
+
"list",
|
|
612
619
|
"cleaned",
|
|
613
620
|
"usage",
|
|
614
621
|
"reconcile",
|
|
@@ -622,8 +629,13 @@ function readStoreObject(given) {
|
|
|
622
629
|
if (store.transaction !== void 0 && !isFunction(store.transaction)) throw configError("store.transaction must be a function, or left out for a store without transactions.");
|
|
623
630
|
return given;
|
|
624
631
|
}
|
|
632
|
+
function checkReplace(name, definition, store) {
|
|
633
|
+
if (!store) throw configError(`Route "${name}": replace needs a database store, which deletes the previous file when the next one completes. Pass store to createCloudrop, or remove replace.`);
|
|
634
|
+
if (!definition.owner) throw configError(`Route "${name}": replace keeps one file per owner, so it needs owner. Add owner to the route, or remove replace.`);
|
|
635
|
+
}
|
|
625
636
|
function checkRoutes(routes, store) {
|
|
626
|
-
for (const { name, definition } of routes) {
|
|
637
|
+
for (const { name, definition, replace } of routes) {
|
|
638
|
+
if (replace) checkReplace(name, definition, store);
|
|
627
639
|
const needsStore = ["owner", "afterComplete"].find((callback) => definition[callback]);
|
|
628
640
|
if (needsStore && !store) throw configError(`Route "${name}": ${needsStore} needs a database store. Pass store to createCloudrop, or remove ${needsStore}.`);
|
|
629
641
|
const routeStore = storeOfRoute(definition);
|
|
@@ -945,10 +957,41 @@ function fileCalls(store, logger, token) {
|
|
|
945
957
|
get: () => call("get", () => store.get(id)),
|
|
946
958
|
claim: (claim, leaseMs) => call("claim", () => store.claim(id, claim, /* @__PURE__ */ new Date(), leaseMs)),
|
|
947
959
|
release: (claim) => call("release", () => store.release(id, claim, /* @__PURE__ */ new Date())),
|
|
948
|
-
complete: (claim, result) => call("complete", () => store.complete(id, claim, result, /* @__PURE__ */ new Date())),
|
|
960
|
+
complete: (claim, result, replace) => call("complete", () => store.complete(id, claim, result, /* @__PURE__ */ new Date(), { replace })),
|
|
949
961
|
end: (from, to) => call("end", () => store.end(id, from, to, /* @__PURE__ */ new Date()))
|
|
950
962
|
};
|
|
951
963
|
}
|
|
964
|
+
const URL_LIFETIME_MS = 9e5;
|
|
965
|
+
const MIN_GRACE_MS = URL_LIFETIME_MS;
|
|
966
|
+
function readGrace(graceMs, what) {
|
|
967
|
+
const value = graceMs ?? 36e5;
|
|
968
|
+
if (typeof value === "number" && Number.isSafeInteger(value) && value >= MIN_GRACE_MS) return ok(value);
|
|
969
|
+
return fail(invalidInput(`graceMs must be at least ${MIN_GRACE_MS} (15 minutes), ${what}.`));
|
|
970
|
+
}
|
|
971
|
+
async function removeLeftovers(config, record) {
|
|
972
|
+
const { storage } = config;
|
|
973
|
+
const checkedAt = /* @__PURE__ */ new Date();
|
|
974
|
+
if (record.uploadId && record.completedAt === null) {
|
|
975
|
+
const aborted = await storage.abortMultipart(record.key, record.uploadId);
|
|
976
|
+
if (aborted.error) return aborted;
|
|
977
|
+
}
|
|
978
|
+
const head = await storage.head(record.key);
|
|
979
|
+
if (head.error) return head;
|
|
980
|
+
if (head.data?.metadata["cloudrop-id"] !== record.id) return ok(checkedAt);
|
|
981
|
+
const removed = await storage.delete(record.key);
|
|
982
|
+
return removed.error ? removed : ok(checkedAt);
|
|
983
|
+
}
|
|
984
|
+
function settledAt(record, graceMs) {
|
|
985
|
+
const leftPending = (record.claimedAt ?? record.endedAt ?? record.expiresAt).getTime();
|
|
986
|
+
return Math.min(record.expiresAt.getTime(), leftPending + URL_LIFETIME_MS) + graceMs;
|
|
987
|
+
}
|
|
988
|
+
async function removeDeleted(config, store, record) {
|
|
989
|
+
const checked = await removeLeftovers(config, record);
|
|
990
|
+
if (checked.error) return checked;
|
|
991
|
+
const checkedAt = checked.data;
|
|
992
|
+
if (checkedAt.getTime() < settledAt(record, 36e5)) return ok(void 0);
|
|
993
|
+
return storeCall(config.logger, `mark file ${record.id} cleaned`, () => store.cleaned(record.id, checkedAt));
|
|
994
|
+
}
|
|
952
995
|
// src/server/plugins.ts
|
|
953
996
|
function refusal(plugin, hook, value) {
|
|
954
997
|
if (value === void 0 || value instanceof CloudropError) return value;
|
|
@@ -1058,47 +1101,228 @@ async function releaseUnstarted(config, shared, unstarted, refusals) {
|
|
|
1058
1101
|
await runOnAbort(plugins, abortContexts(shared, overQuota, "rejected"), logger);
|
|
1059
1102
|
await runOnAbort(plugins, abortContexts(shared, failed, "failed"), logger);
|
|
1060
1103
|
}
|
|
1104
|
+
// src/server/owner-files.ts
|
|
1105
|
+
const DEFAULT_LIMIT$1 = 100;
|
|
1106
|
+
const MAX_LIMIT$1 = 1e3;
|
|
1107
|
+
const CURSOR$1 = /^\d{1,15}:.+$/s;
|
|
1108
|
+
const isOwnerOrId = (value) => typeof value === "string" && value !== "" && value.length <= 255;
|
|
1109
|
+
const badOwner = () => fail(invalidInput(`Pass the owner, a string of 1 to 255 characters, as owner() returned it.`));
|
|
1110
|
+
function readOptions$2(config, given) {
|
|
1111
|
+
if (!isPlainObject(given)) return fail(invalidInput("files takes { owner }, and optionally route, limit and cursor."));
|
|
1112
|
+
const { owner, route, limit = DEFAULT_LIMIT$1, cursor } = given;
|
|
1113
|
+
if (!isOwnerOrId(owner)) return badOwner();
|
|
1114
|
+
if (route !== void 0 && (typeof route !== "string" || !config.routes.has(route))) {
|
|
1115
|
+
const names = [...config.routes.keys()].map((name) => JSON.stringify(name)).join(", ");
|
|
1116
|
+
return fail(invalidInput(`route must be one of this server's routes: ${names}.`));
|
|
1117
|
+
}
|
|
1118
|
+
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT$1) return fail(invalidInput(`limit must be a whole number from 1 to ${MAX_LIMIT$1}.`));
|
|
1119
|
+
if (cursor !== void 0 && (typeof cursor !== "string" || !CURSOR$1.test(cursor))) return fail(invalidInput("cursor must be one an earlier page of files returned. Leave it out for the first page."));
|
|
1120
|
+
return ok({
|
|
1121
|
+
owner,
|
|
1122
|
+
route: route ?? null,
|
|
1123
|
+
limit,
|
|
1124
|
+
cursor: cursor ?? null
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
async function listFiles(config, store, given) {
|
|
1128
|
+
const options = readOptions$2(config, given);
|
|
1129
|
+
if (options.error) return options;
|
|
1130
|
+
const { owner } = options.data;
|
|
1131
|
+
const page = await storeCall(config.logger, `list the files of ${owner}`, () => store.list({
|
|
1132
|
+
...options.data,
|
|
1133
|
+
statuses: ["completed"]
|
|
1134
|
+
}));
|
|
1135
|
+
return page.error ? page : ok({
|
|
1136
|
+
items: page.data.records,
|
|
1137
|
+
cursor: page.data.cursor
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
// src/server/erase.ts
|
|
1141
|
+
const PAGE = 100;
|
|
1142
|
+
const LIVE = [
|
|
1143
|
+
"pending",
|
|
1144
|
+
"completing",
|
|
1145
|
+
"completed"
|
|
1146
|
+
];
|
|
1147
|
+
const ENDED = [
|
|
1148
|
+
"aborted",
|
|
1149
|
+
"rejected",
|
|
1150
|
+
"failed",
|
|
1151
|
+
"expired",
|
|
1152
|
+
"deleted"
|
|
1153
|
+
];
|
|
1154
|
+
const EVERY = [...LIVE, ...ENDED];
|
|
1155
|
+
function readOptions$1(given) {
|
|
1156
|
+
const options = given ?? {};
|
|
1157
|
+
if (!isPlainObject(options)) return fail(invalidInput("eraseOwner takes options such as { graceMs: 3600000 }."));
|
|
1158
|
+
const { signal } = options;
|
|
1159
|
+
if (signal !== void 0 && !(signal instanceof AbortSignal)) return fail(invalidInput("signal must be an AbortSignal, such as AbortSignal.timeout(60_000)."));
|
|
1160
|
+
const graceMs = readGrace(options.graceMs, "the time a PUT already under way gets to land after its URL ran out");
|
|
1161
|
+
return graceMs.error ? graceMs : ok({
|
|
1162
|
+
graceMs: graceMs.data,
|
|
1163
|
+
signal
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
async function deleteCompleted({ config, store, report }, record) {
|
|
1167
|
+
const ended = await storeCall(config.logger, `delete file ${record.id}`, () => store.end(record.id, "completed", "deleted", /* @__PURE__ */ new Date()));
|
|
1168
|
+
if (ended.error) report.failed += 1;
|
|
1169
|
+
else if (ended.data.changed) report.deleted += 1;
|
|
1170
|
+
}
|
|
1171
|
+
async function checkEnded({ config, store, graceMs, report }, record) {
|
|
1172
|
+
const settled = settledAt(record, graceMs);
|
|
1173
|
+
if (record.status !== "failed" && record.cleanedAt !== null && record.cleanedAt.getTime() >= settled) return;
|
|
1174
|
+
const checked = await removeLeftovers(config, record);
|
|
1175
|
+
if (checked.error) {
|
|
1176
|
+
config.logger.error(`cloudrop: eraseOwner could not remove what file ${record.id} left in the bucket. Run eraseOwner again.`, checked.error);
|
|
1177
|
+
report.failed += 1;
|
|
1178
|
+
return;
|
|
1179
|
+
}
|
|
1180
|
+
const checkedAt = checked.data;
|
|
1181
|
+
if (checkedAt.getTime() < settled) {
|
|
1182
|
+
report.waiting += 1;
|
|
1183
|
+
return;
|
|
1184
|
+
}
|
|
1185
|
+
if (record.cleanedAt === null) {
|
|
1186
|
+
if ((await storeCall(config.logger, `mark file ${record.id} cleaned`, () => store.cleaned(record.id, checkedAt))).error) {
|
|
1187
|
+
report.failed += 1;
|
|
1188
|
+
return;
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
report.cleaned += 1;
|
|
1192
|
+
}
|
|
1193
|
+
async function eachRow(eraser, statuses, handle) {
|
|
1194
|
+
const { config, store, owner, signal } = eraser;
|
|
1195
|
+
let cursor = null;
|
|
1196
|
+
do {
|
|
1197
|
+
if (signal?.aborted) return ok(false);
|
|
1198
|
+
const page = await storeCall(config.logger, `list the ${statuses.join(" or ")} files of ${owner}`, () => store.list({
|
|
1199
|
+
owner,
|
|
1200
|
+
statuses,
|
|
1201
|
+
route: null,
|
|
1202
|
+
limit: PAGE,
|
|
1203
|
+
cursor
|
|
1204
|
+
}));
|
|
1205
|
+
if (page.error) return page;
|
|
1206
|
+
for (const record of page.data.records) {
|
|
1207
|
+
if (signal?.aborted) return ok(false);
|
|
1208
|
+
await handle(eraser, record);
|
|
1209
|
+
}
|
|
1210
|
+
cursor = page.data.cursor;
|
|
1211
|
+
} while (cursor !== null);
|
|
1212
|
+
return ok(true);
|
|
1213
|
+
}
|
|
1214
|
+
const wait = async ({ report }) => {
|
|
1215
|
+
report.waiting += 1;
|
|
1216
|
+
};
|
|
1217
|
+
async function runPasses(eraser, passes) {
|
|
1218
|
+
for (const [statuses, handle] of passes) {
|
|
1219
|
+
const finished = await eachRow(eraser, statuses, handle);
|
|
1220
|
+
if (finished.error || !finished.data) return finished;
|
|
1221
|
+
}
|
|
1222
|
+
return ok(true);
|
|
1223
|
+
}
|
|
1224
|
+
async function lastReads(eraser, startedAt) {
|
|
1225
|
+
const { config, store, owner, report } = eraser;
|
|
1226
|
+
if (report.waiting > 0) return ok(true);
|
|
1227
|
+
const newest = await storeCall(config.logger, `list the newest file of ${owner}`, () => store.list({
|
|
1228
|
+
owner,
|
|
1229
|
+
statuses: EVERY,
|
|
1230
|
+
route: null,
|
|
1231
|
+
limit: 1,
|
|
1232
|
+
cursor: null
|
|
1233
|
+
}));
|
|
1234
|
+
if (newest.error) return newest;
|
|
1235
|
+
if ((newest.data.records[0]?.createdAt.getTime() ?? 0) < startedAt) return eachRow(eraser, LIVE, wait);
|
|
1236
|
+
report.waiting += 1;
|
|
1237
|
+
return ok(true);
|
|
1238
|
+
}
|
|
1239
|
+
async function eraseOwner(config, store, owner, given) {
|
|
1240
|
+
if (!isOwnerOrId(owner)) return badOwner();
|
|
1241
|
+
const options = readOptions$1(given);
|
|
1242
|
+
if (options.error) return options;
|
|
1243
|
+
const report = {
|
|
1244
|
+
revoked: 0,
|
|
1245
|
+
deleted: 0,
|
|
1246
|
+
cleaned: 0,
|
|
1247
|
+
failed: 0,
|
|
1248
|
+
waiting: 0,
|
|
1249
|
+
done: false
|
|
1250
|
+
};
|
|
1251
|
+
if (options.data.signal?.aborted) return ok(report);
|
|
1252
|
+
const startedAt = Date.now();
|
|
1253
|
+
const revoked = await storeCall(config.logger, `revoke the uploads of ${owner}`, () => store.revoke(owner, /* @__PURE__ */ new Date()));
|
|
1254
|
+
if (revoked.error) return revoked;
|
|
1255
|
+
report.revoked = revoked.data;
|
|
1256
|
+
const eraser = {
|
|
1257
|
+
config,
|
|
1258
|
+
store,
|
|
1259
|
+
owner,
|
|
1260
|
+
...options.data,
|
|
1261
|
+
report
|
|
1262
|
+
};
|
|
1263
|
+
const ran = await runPasses(eraser, [
|
|
1264
|
+
[["completed"], deleteCompleted],
|
|
1265
|
+
[LIVE, wait],
|
|
1266
|
+
...ENDED.map((status) => [[status], checkEnded])
|
|
1267
|
+
]);
|
|
1268
|
+
const finished = ran.error || !ran.data ? ran : await lastReads(eraser, startedAt);
|
|
1269
|
+
if (finished.error) return finished;
|
|
1270
|
+
return ok({
|
|
1271
|
+
...report,
|
|
1272
|
+
done: finished.data && report.failed === 0 && report.waiting === 0
|
|
1273
|
+
});
|
|
1274
|
+
}
|
|
1061
1275
|
// src/server/sweep.ts
|
|
1062
1276
|
const DEFAULT_LIMIT = 100;
|
|
1063
1277
|
const MAX_LIMIT = 1e3;
|
|
1064
|
-
const DEFAULT_GRACE_MS = 36e5;
|
|
1065
|
-
const MIN_GRACE_MS = 9e5;
|
|
1066
1278
|
const CURSOR = /^(expire|cleanup):((?:\d{1,15}:.+)?)$/s;
|
|
1279
|
+
function readCursor(cursor) {
|
|
1280
|
+
const match = CURSOR.exec(cursor);
|
|
1281
|
+
return match ? {
|
|
1282
|
+
pass: match[1],
|
|
1283
|
+
cursor: match[2] || null
|
|
1284
|
+
} : null;
|
|
1285
|
+
}
|
|
1286
|
+
function readAll(options) {
|
|
1287
|
+
const { all = false, signal } = options;
|
|
1288
|
+
if (typeof all !== "boolean") return fail(invalidInput("all must be true or false."));
|
|
1289
|
+
if (signal !== void 0 && !(signal instanceof AbortSignal)) return fail(invalidInput("signal must be an AbortSignal, such as AbortSignal.timeout(25_000)."));
|
|
1290
|
+
if (signal !== void 0 && !all) return fail(invalidInput("signal stops a sweep between pages, so it needs all: true. Pass both, or neither."));
|
|
1291
|
+
return ok({
|
|
1292
|
+
all,
|
|
1293
|
+
signal
|
|
1294
|
+
});
|
|
1295
|
+
}
|
|
1067
1296
|
function readOptions(given) {
|
|
1068
1297
|
const options = given ?? {};
|
|
1069
1298
|
if (!isPlainObject(options)) return fail(invalidInput("sweep takes options such as { limit: 100 }."));
|
|
1070
|
-
const { limit = DEFAULT_LIMIT,
|
|
1299
|
+
const { limit = DEFAULT_LIMIT, cursor } = options;
|
|
1071
1300
|
if (typeof limit !== "number" || !Number.isInteger(limit) || limit < 1 || limit > MAX_LIMIT) return fail(invalidInput(`limit must be a whole number from 1 to ${MAX_LIMIT}.`));
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1301
|
+
const graceMs = readGrace(options.graceMs, "the life of an upload URL, so no upload can start after a row's deadline once it may be cleaned up");
|
|
1302
|
+
if (graceMs.error) return graceMs;
|
|
1303
|
+
const all = readAll(options);
|
|
1304
|
+
if (all.error) return all;
|
|
1305
|
+
const start = cursor === void 0 ? {
|
|
1306
|
+
pass: "expire",
|
|
1307
|
+
cursor: null
|
|
1308
|
+
} : readCursor(String(cursor));
|
|
1309
|
+
if (!start || cursor !== void 0 && typeof cursor !== "string") return fail(invalidInput("cursor must be one an earlier sweep returned. Leave it out to start over."));
|
|
1079
1310
|
return ok({
|
|
1080
1311
|
limit,
|
|
1081
|
-
graceMs,
|
|
1082
|
-
|
|
1083
|
-
|
|
1312
|
+
graceMs: graceMs.data,
|
|
1313
|
+
...all.data,
|
|
1314
|
+
...start
|
|
1084
1315
|
});
|
|
1085
1316
|
}
|
|
1086
|
-
async function
|
|
1087
|
-
const
|
|
1088
|
-
if (
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
const head = await storage.head(record.key);
|
|
1093
|
-
if (head.error) return head;
|
|
1094
|
-
if (head.data?.metadata["cloudrop-id"] !== record.id) return ok(void 0);
|
|
1095
|
-
return storage.delete(record.key);
|
|
1096
|
-
}
|
|
1097
|
-
async function removeOrThrow(config, record) {
|
|
1098
|
-
const removed = await removeLeftovers(config, record);
|
|
1099
|
-
if (removed.error) throw removed.error;
|
|
1317
|
+
async function removeAndMark({ config, store, graceMs }, record) {
|
|
1318
|
+
const checked = await removeLeftovers(config, record);
|
|
1319
|
+
if (checked.error) throw checked.error;
|
|
1320
|
+
if (checked.data.getTime() < settledAt(record, graceMs)) return false;
|
|
1321
|
+
await store.cleaned(record.id, checked.data);
|
|
1322
|
+
return true;
|
|
1100
1323
|
}
|
|
1101
|
-
async function expire(
|
|
1324
|
+
async function expire(sweeper, record) {
|
|
1325
|
+
const { store, report } = sweeper;
|
|
1102
1326
|
if (record.status === "completing") {
|
|
1103
1327
|
const ended = await store.end(record.id, "completing", "failed", /* @__PURE__ */ new Date(), { cleaned: true });
|
|
1104
1328
|
if (ended.changed) report.reported.push(ended.record);
|
|
@@ -1106,14 +1330,11 @@ async function expire({ config, store, report }, record) {
|
|
|
1106
1330
|
}
|
|
1107
1331
|
const ended = await store.end(record.id, "pending", "expired", /* @__PURE__ */ new Date());
|
|
1108
1332
|
if (!ended.changed) return;
|
|
1109
|
-
await
|
|
1110
|
-
await store.cleaned(record.id, /* @__PURE__ */ new Date());
|
|
1333
|
+
await removeAndMark(sweeper, ended.record);
|
|
1111
1334
|
report.expired += 1;
|
|
1112
1335
|
}
|
|
1113
|
-
async function cleanUp(
|
|
1114
|
-
await
|
|
1115
|
-
await store.cleaned(record.id, /* @__PURE__ */ new Date());
|
|
1116
|
-
report.cleaned += 1;
|
|
1336
|
+
async function cleanUp(sweeper, record) {
|
|
1337
|
+
if (await removeAndMark(sweeper, record)) sweeper.report.cleaned += 1;
|
|
1117
1338
|
}
|
|
1118
1339
|
async function sweepPage(sweeper, pass, page) {
|
|
1119
1340
|
const { config, store, report } = sweeper;
|
|
@@ -1148,10 +1369,8 @@ function listingFailed(report, handled, error) {
|
|
|
1148
1369
|
failed: report.failed + 1
|
|
1149
1370
|
}, null);
|
|
1150
1371
|
}
|
|
1151
|
-
async function
|
|
1152
|
-
const
|
|
1153
|
-
if (options.error) return options;
|
|
1154
|
-
const { limit, graceMs, pass } = options.data;
|
|
1372
|
+
async function sweepOnce(config, store, run) {
|
|
1373
|
+
const { limit, graceMs, pass } = run;
|
|
1155
1374
|
const report = {
|
|
1156
1375
|
expired: 0,
|
|
1157
1376
|
cleaned: 0,
|
|
@@ -1162,11 +1381,12 @@ async function sweep(config, store, given) {
|
|
|
1162
1381
|
const sweeper = {
|
|
1163
1382
|
config,
|
|
1164
1383
|
store,
|
|
1165
|
-
report
|
|
1384
|
+
report,
|
|
1385
|
+
graceMs
|
|
1166
1386
|
};
|
|
1167
1387
|
const now = Date.now();
|
|
1168
1388
|
let budget = limit;
|
|
1169
|
-
let cursor =
|
|
1389
|
+
let cursor = run.cursor;
|
|
1170
1390
|
if (pass === "expire") {
|
|
1171
1391
|
const page = await sweepPage(sweeper, "expire", {
|
|
1172
1392
|
before: now - graceMs,
|
|
@@ -1187,11 +1407,38 @@ async function sweep(config, store, given) {
|
|
|
1187
1407
|
if (page.error) return listingFailed(report, limit - budget, page.error);
|
|
1188
1408
|
return stoppedAt(report, page.data.cursor === null ? null : `cleanup:${page.data.cursor}`);
|
|
1189
1409
|
}
|
|
1410
|
+
async function sweepAll(config, store, run) {
|
|
1411
|
+
const total = {
|
|
1412
|
+
expired: 0,
|
|
1413
|
+
cleaned: 0,
|
|
1414
|
+
failed: 0,
|
|
1415
|
+
reported: [],
|
|
1416
|
+
cursor: null
|
|
1417
|
+
};
|
|
1418
|
+
let position = run;
|
|
1419
|
+
for (let pages = 0; position !== null; pages++) {
|
|
1420
|
+
if (run.signal?.aborted) return stoppedAt(total, `${position.pass}:${position.cursor ?? ""}`);
|
|
1421
|
+
const page = await sweepOnce(config, store, {
|
|
1422
|
+
...run,
|
|
1423
|
+
...position
|
|
1424
|
+
});
|
|
1425
|
+
if (page.error) return pages === 0 ? page : listingFailed(total, 1, page.error);
|
|
1426
|
+
total.expired += page.data.expired;
|
|
1427
|
+
total.cleaned += page.data.cleaned;
|
|
1428
|
+
total.failed += page.data.failed;
|
|
1429
|
+
total.reported.push(...page.data.reported);
|
|
1430
|
+
position = page.data.cursor === null ? null : readCursor(page.data.cursor);
|
|
1431
|
+
}
|
|
1432
|
+
return ok(total);
|
|
1433
|
+
}
|
|
1434
|
+
async function sweep(config, store, given) {
|
|
1435
|
+
const options = readOptions(given);
|
|
1436
|
+
if (options.error) return options;
|
|
1437
|
+
return options.data.all ? sweepAll(config, store, options.data) : sweepOnce(config, store, options.data);
|
|
1438
|
+
}
|
|
1190
1439
|
// src/server/store-api.ts
|
|
1191
1440
|
const OWNERS_PAGE = 100;
|
|
1192
|
-
const isOwnerOrId = (value) => typeof value === "string" && value !== "" && value.length <= 255;
|
|
1193
1441
|
const badId = () => fail(invalidInput("Pass the file id, as returned with the uploaded file."));
|
|
1194
|
-
const badOwner = () => fail(invalidInput(`Pass the owner, a string of 1 to 255 characters, as owner() returned it.`));
|
|
1195
1442
|
const notCompleted = () => new CloudropError("NOT_FOUND", "No completed file has this id. deleteFile deletes only files whose upload completed.");
|
|
1196
1443
|
async function deleteFile(config, store, id) {
|
|
1197
1444
|
const ended = await storeCall(config.logger, `delete file ${id}`, () => store.end(id, "completed", "deleted", /* @__PURE__ */ new Date()));
|
|
@@ -1199,10 +1446,8 @@ async function deleteFile(config, store, id) {
|
|
|
1199
1446
|
const { changed, record } = ended.data;
|
|
1200
1447
|
if (record?.status !== "deleted") return fail(notCompleted());
|
|
1201
1448
|
if (!changed && record.cleanedAt !== null) return ok({ id });
|
|
1202
|
-
const removed = await
|
|
1203
|
-
|
|
1204
|
-
const marked = await storeCall(config.logger, `mark file ${id} cleaned`, () => store.cleaned(id, /* @__PURE__ */ new Date()));
|
|
1205
|
-
return marked.error ? marked : ok({ id });
|
|
1449
|
+
const removed = await removeDeleted(config, store, record);
|
|
1450
|
+
return removed.error ? removed : ok({ id });
|
|
1206
1451
|
}
|
|
1207
1452
|
async function reconcile(config, store, given) {
|
|
1208
1453
|
const options = given ?? {};
|
|
@@ -1239,6 +1484,7 @@ function storeApi(config, whenReady) {
|
|
|
1239
1484
|
const withStore = (name, call) => whenReady(() => config.store ? call(config.store) : Promise.resolve(fail(configError(`api.${name} needs a database store. Pass store to createCloudrop.`))));
|
|
1240
1485
|
return {
|
|
1241
1486
|
file: (id) => withStore("file", async (store) => isOwnerOrId(id) ? storeCall(logger, `get file ${id}`, () => store.get(id)) : badId()),
|
|
1487
|
+
files: (options) => withStore("files", (store) => listFiles(config, store, options)),
|
|
1242
1488
|
usage: (owner) => withStore("usage", async (store) => isOwnerOrId(owner) ? storeCall(logger, `read the usage of ${owner}`, () => store.usage(owner)) : badOwner()),
|
|
1243
1489
|
revokeUploads: (options) => withStore("revokeUploads", async (store) => {
|
|
1244
1490
|
const owner = isPlainObject(options) ? options.owner : void 0;
|
|
@@ -1247,6 +1493,7 @@ function storeApi(config, whenReady) {
|
|
|
1247
1493
|
return revoked.error ? revoked : ok({ revoked: revoked.data });
|
|
1248
1494
|
}),
|
|
1249
1495
|
deleteFile: (id) => withStore("deleteFile", async (store) => isOwnerOrId(id) ? deleteFile(config, store, id) : badId()),
|
|
1496
|
+
eraseOwner: (owner, options) => withStore("eraseOwner", (store) => eraseOwner(config, store, owner, options)),
|
|
1250
1497
|
reconcile: (options) => withStore("reconcile", (store) => reconcile(config, store, options)),
|
|
1251
1498
|
sweep: (options) => withStore("sweep", (store) => sweep(config, store, options))
|
|
1252
1499
|
};
|
|
@@ -1453,14 +1700,10 @@ async function readJson(request) {
|
|
|
1453
1700
|
const chunks = [];
|
|
1454
1701
|
let length = 0;
|
|
1455
1702
|
try {
|
|
1456
|
-
const
|
|
1457
|
-
|
|
1458
|
-
length
|
|
1459
|
-
|
|
1460
|
-
await reader?.cancel();
|
|
1461
|
-
return fail(bodyTooLarge());
|
|
1462
|
-
}
|
|
1463
|
-
chunks.push(chunk.value);
|
|
1703
|
+
for await (const chunk of request.body ?? []) {
|
|
1704
|
+
length += chunk.length;
|
|
1705
|
+
if (length > 65536) return fail(bodyTooLarge());
|
|
1706
|
+
chunks.push(chunk);
|
|
1464
1707
|
}
|
|
1465
1708
|
} catch {
|
|
1466
1709
|
return fail(invalidInput("The request body could not be read. Send the request again."));
|
|
@@ -1583,15 +1826,24 @@ function completeContext(request, token, file) {
|
|
|
1583
1826
|
metadata: token.metadata
|
|
1584
1827
|
};
|
|
1585
1828
|
}
|
|
1586
|
-
function
|
|
1587
|
-
|
|
1829
|
+
async function runOnComplete(logger, route, context) {
|
|
1830
|
+
const { onComplete } = route.definition;
|
|
1831
|
+
let data;
|
|
1832
|
+
try {
|
|
1833
|
+
data = await onComplete?.(context);
|
|
1834
|
+
} catch (error) {
|
|
1835
|
+
logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1836
|
+
return fail(unfinished());
|
|
1837
|
+
}
|
|
1588
1838
|
try {
|
|
1589
|
-
|
|
1839
|
+
return ok({
|
|
1840
|
+
data,
|
|
1841
|
+
json: JSON.stringify(data)
|
|
1842
|
+
});
|
|
1590
1843
|
} catch (error) {
|
|
1591
|
-
logger.error(`cloudrop: onComplete for route "${route}" returned a value JSON cannot encode (a BigInt or a cycle).`, error);
|
|
1844
|
+
logger.error(`cloudrop: onComplete for route "${route.name}" returned a value JSON cannot encode (a BigInt or a cycle).`, error);
|
|
1592
1845
|
return fail(unfinished());
|
|
1593
1846
|
}
|
|
1594
|
-
return ok(json);
|
|
1595
1847
|
}
|
|
1596
1848
|
// src/server/complete-verify.ts
|
|
1597
1849
|
const DISCARDED = "The uploaded file does not match what was declared, so it was discarded. Upload it again.";
|
|
@@ -1698,31 +1950,23 @@ async function refuse(completion, refusal) {
|
|
|
1698
1950
|
if (ended.data.changed) return cleanUpRefused(completion.config, completion.token, refusal, { rowEnded: true });
|
|
1699
1951
|
return settled(ended.data.record, completion.file);
|
|
1700
1952
|
}
|
|
1701
|
-
async function
|
|
1702
|
-
const
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
} catch (error) {
|
|
1710
|
-
config.logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1711
|
-
return fail(unfinished());
|
|
1712
|
-
}
|
|
1713
|
-
if (result === void 0 || result === null) return ok({
|
|
1953
|
+
async function resultToStore({ request, config, token, route, file }, extra) {
|
|
1954
|
+
const ran = await runOnComplete(config.logger, route, {
|
|
1955
|
+
...completeContext(request, token, file),
|
|
1956
|
+
...extra
|
|
1957
|
+
});
|
|
1958
|
+
if (ran.error) return ran;
|
|
1959
|
+
const { data, json } = ran.data;
|
|
1960
|
+
if (data === void 0 || data === null || json === void 0) return ok({
|
|
1714
1961
|
data: null,
|
|
1715
1962
|
json: null
|
|
1716
1963
|
});
|
|
1717
|
-
|
|
1718
|
-
if (encoded.error) return encoded;
|
|
1719
|
-
const json = encoded.data ?? null;
|
|
1720
|
-
if (json !== null && new TextEncoder().encode(json).length > MAX_RESULT_JSON_BYTES) {
|
|
1964
|
+
if (new TextEncoder().encode(json).length > MAX_RESULT_JSON_BYTES) {
|
|
1721
1965
|
config.logger.error(`cloudrop: onComplete for route "${route.name}" returned more than 16 KiB of JSON. Return ids, not records.`);
|
|
1722
1966
|
return fail(unfinished());
|
|
1723
1967
|
}
|
|
1724
1968
|
return ok({
|
|
1725
|
-
data:
|
|
1969
|
+
data: JSON.parse(json),
|
|
1726
1970
|
json
|
|
1727
1971
|
});
|
|
1728
1972
|
}
|
|
@@ -1738,7 +1982,7 @@ async function withLease(completion) {
|
|
|
1738
1982
|
answer: claimAnswer(claimed.data, file),
|
|
1739
1983
|
recorded: false
|
|
1740
1984
|
};
|
|
1741
|
-
const ran = await
|
|
1985
|
+
const ran = await resultToStore(completion, {});
|
|
1742
1986
|
if (ran.error) {
|
|
1743
1987
|
await calls.release(claim);
|
|
1744
1988
|
return {
|
|
@@ -1746,7 +1990,7 @@ async function withLease(completion) {
|
|
|
1746
1990
|
recorded: false
|
|
1747
1991
|
};
|
|
1748
1992
|
}
|
|
1749
|
-
const done = await calls.complete(claim, ran.data.json);
|
|
1993
|
+
const done = await calls.complete(claim, ran.data.json, completion.route.replace);
|
|
1750
1994
|
if (done.error) return {
|
|
1751
1995
|
answer: done,
|
|
1752
1996
|
recorded: false
|
|
@@ -1760,7 +2004,8 @@ async function withLease(completion) {
|
|
|
1760
2004
|
file,
|
|
1761
2005
|
data: ran.data.data
|
|
1762
2006
|
}),
|
|
1763
|
-
recorded: true
|
|
2007
|
+
recorded: true,
|
|
2008
|
+
replaced: done.data.replaced
|
|
1764
2009
|
};
|
|
1765
2010
|
}
|
|
1766
2011
|
async function inTransaction(completion, transaction) {
|
|
@@ -1789,16 +2034,18 @@ async function inTransaction(completion, transaction) {
|
|
|
1789
2034
|
};
|
|
1790
2035
|
return;
|
|
1791
2036
|
}
|
|
1792
|
-
const ran = await
|
|
2037
|
+
const ran = await resultToStore(completion, storeOfRoute(completion.route.definition) ? { tx } : {});
|
|
1793
2038
|
if (ran.error) return rollBack(ran);
|
|
1794
|
-
const
|
|
2039
|
+
const replace = completion.route.replace;
|
|
2040
|
+
const done = await store.complete(token.id, claim, ran.data.json, /* @__PURE__ */ new Date(), { replace });
|
|
1795
2041
|
if (!done.changed) return rollBack(lostClaim(done.record, file));
|
|
1796
2042
|
attempt = {
|
|
1797
2043
|
answer: ok({
|
|
1798
2044
|
file,
|
|
1799
2045
|
data: ran.data.data
|
|
1800
2046
|
}),
|
|
1801
|
-
recorded: true
|
|
2047
|
+
recorded: true,
|
|
2048
|
+
replaced: done.replaced
|
|
1802
2049
|
};
|
|
1803
2050
|
});
|
|
1804
2051
|
} catch (error) {
|
|
@@ -1811,6 +2058,12 @@ async function inTransaction(completion, transaction) {
|
|
|
1811
2058
|
}
|
|
1812
2059
|
return attempt;
|
|
1813
2060
|
}
|
|
2061
|
+
async function removeReplaced({ config }, store, replaced) {
|
|
2062
|
+
for (const record of replaced) {
|
|
2063
|
+
const removed = await removeDeleted(config, store, record);
|
|
2064
|
+
if (removed.error) config.logger.error(`cloudrop: could not remove the replaced file ${record.id} from the bucket. The sweep removes it later.`, removed.error);
|
|
2065
|
+
}
|
|
2066
|
+
}
|
|
1814
2067
|
async function runAfterComplete({ request, config, token, route, file }, data) {
|
|
1815
2068
|
try {
|
|
1816
2069
|
await route.definition.afterComplete?.({
|
|
@@ -1839,9 +2092,12 @@ async function completeStored({ request, config }, store, token, route) {
|
|
|
1839
2092
|
const checked = stored.kind === "verified" ? await checkAfterVerify(request, config, token, completion.file) : stored;
|
|
1840
2093
|
if (checked.kind === "failed") return fail(checked.error);
|
|
1841
2094
|
if (checked.kind === "refused") return refuse(completion, checked);
|
|
1842
|
-
const
|
|
1843
|
-
if (recorded)
|
|
1844
|
-
|
|
2095
|
+
const attempt = store.transaction ? await inTransaction(completion, store.transaction.bind(store)) : await withLease(completion);
|
|
2096
|
+
if (attempt.recorded) {
|
|
2097
|
+
await removeReplaced(completion, store, attempt.replaced);
|
|
2098
|
+
await runAfterComplete(completion, attempt.answer.data?.data);
|
|
2099
|
+
}
|
|
2100
|
+
return attempt.answer;
|
|
1845
2101
|
}
|
|
1846
2102
|
// src/server/complete.ts
|
|
1847
2103
|
async function handleComplete(input) {
|
|
@@ -1850,7 +2106,6 @@ async function handleComplete(input) {
|
|
|
1850
2106
|
if (verified.error) return verified;
|
|
1851
2107
|
const { token, route } = verified.data;
|
|
1852
2108
|
if (config.store && token.stored) return completeStored(input, config.store, token, route);
|
|
1853
|
-
const { logger } = config;
|
|
1854
2109
|
const stored = await verifyUpload(config, token);
|
|
1855
2110
|
if (stored.kind === "failed") return fail(stored.error);
|
|
1856
2111
|
if (stored.kind === "refused") return cleanUpRefused(config, token, stored);
|
|
@@ -1858,23 +2113,11 @@ async function handleComplete(input) {
|
|
|
1858
2113
|
const checked = await checkAfterVerify(request, config, token, file);
|
|
1859
2114
|
if (checked.kind === "failed") return fail(checked.error);
|
|
1860
2115
|
if (checked.kind === "refused") return cleanUpRefused(config, token, checked);
|
|
1861
|
-
const
|
|
1862
|
-
if (
|
|
1863
|
-
file,
|
|
1864
|
-
data: null
|
|
1865
|
-
});
|
|
1866
|
-
let data;
|
|
1867
|
-
try {
|
|
1868
|
-
data = await onComplete(completeContext(request, token, file));
|
|
1869
|
-
} catch (error) {
|
|
1870
|
-
logger.error(`cloudrop: onComplete failed for route "${route.name}".`, error);
|
|
1871
|
-
return fail(unfinished());
|
|
1872
|
-
}
|
|
1873
|
-
const encoded = encodeResult(logger, route.name, data);
|
|
1874
|
-
if (encoded.error) return encoded;
|
|
2116
|
+
const ran = await runOnComplete(config.logger, route, completeContext(request, token, file));
|
|
2117
|
+
if (ran.error) return ran;
|
|
1875
2118
|
return ok({
|
|
1876
2119
|
file,
|
|
1877
|
-
data: data === void 0 ? null : data
|
|
2120
|
+
data: ran.data.json === void 0 ? null : ran.data.data
|
|
1878
2121
|
});
|
|
1879
2122
|
}
|
|
1880
2123
|
// src/server/sign.ts
|
|
@@ -2713,13 +2956,9 @@ function readListedObject(block) {
|
|
|
2713
2956
|
});
|
|
2714
2957
|
}
|
|
2715
2958
|
async function forEachConcurrently(items, limit, task) {
|
|
2716
|
-
|
|
2959
|
+
const queue = items.values();
|
|
2717
2960
|
const worker = async () => {
|
|
2718
|
-
|
|
2719
|
-
const item = items[next];
|
|
2720
|
-
next += 1;
|
|
2721
|
-
await task(item);
|
|
2722
|
-
}
|
|
2961
|
+
for (const item of queue) await task(item);
|
|
2723
2962
|
};
|
|
2724
2963
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
2725
2964
|
}
|