@voltro/plugin-storage 0.84.0 → 0.86.0

package/dist/ingest.js

@@ -0,0 +1,81 @@
+import { StorageError as e, StorageRejected as t } from "./types.js";
+import { Effect as n } from "effect";
+//#region src/ingest.ts
+var r = 67108864, i = (e) => new t({
+	reason: "too-large",
+	detail: `remote body exceeds the ${e}-byte ingest limit`
+}), a = (e, n) => new t({
+	reason: "blocked-source",
+	detail: `will not fetch ${e}: ${n}`
+}), o = (o, s, c = {}) => n.suspend(() => {
+	let l = s ?? r;
+	return !Number.isSafeInteger(l) || l < 0 ? n.fail(new e({
+		provider: "ingest",
+		message: "limits.maxBytes must be a non-negative safe integer",
+		transient: !1
+	})) : n.tryPromise({
+		try: async (t) => {
+			let { ssrfReason: n } = await import("./ingestPolicy.js");
+			t.throwIfAborted();
+			let r = o, s;
+			for (let e = 0;; e++) {
+				let i = n(r, { ...c.allowHosts === void 0 ? {} : { allowHosts: c.allowHosts } });
+				if (i !== null) throw a(r, i);
+				s = await fetch(r, {
+					signal: t,
+					redirect: "manual"
+				});
+				let o = s.headers.get("location");
+				if (s.status < 300 || s.status >= 400 || o === null) break;
+				if (await s.body?.cancel().catch(() => {}), e >= (c.maxRedirects ?? 5)) throw a(r, "too many redirects");
+				r = new URL(o, r).toString();
+			}
+			let u;
+			try {
+				if (!s.ok) throw new e({
+					provider: "ingest",
+					message: `remote server returned HTTP ${s.status}`,
+					transient: s.status === 429 || s.status >= 500,
+					status: s.status
+				});
+				let t = s.headers.get("content-length"), n = s.headers.get("content-encoding")?.trim().toLowerCase();
+				if ((!n || n === "identity") && t !== null && /^\d+$/.test(t) && BigInt(t) > BigInt(l)) throw i(l);
+				let r = /* @__PURE__ */ new Uint8Array(), a = 0;
+				if (s.body !== null) for (u = s.body.getReader();;) {
+					let e = await u.read();
+					if (e.done) break;
+					if (e.value.byteLength > l - a) throw i(l);
+					if (e.value.byteLength !== 0) {
+						let t = a + e.value.byteLength;
+						if (t > r.byteLength) {
+							let e = Math.min(l, Math.max(65536, r.byteLength * 2, t)), n = new Uint8Array(e);
+							n.set(r), r = n;
+						}
+						r.set(e.value, a), a = t;
+					}
+				}
+				return {
+					bytes: a === r.byteLength ? r : r.slice(0, a),
+					contentType: s.headers.get("content-type")?.split(";")[0]?.trim() || "application/octet-stream"
+				};
+			} finally {
+				u === void 0 ? await s.body?.cancel().catch(() => {}) : (await u.cancel().catch(() => {}), u.releaseLock());
+			}
+		},
+		catch: (n) => n instanceof t || n instanceof e ? n : new e({
+			provider: "ingest",
+			message: "remote fetch or body read failed",
+			transient: !0
+		})
+	});
+}), s = (e, t) => n.suspend(() => {
+	if (t === void 0) return e;
+	if (t.aborted) return n.interrupt;
+	let r = n.callback((e) => {
+		let r = () => e(n.interrupt);
+		return t.addEventListener("abort", r, { once: !0 }), t.aborted && r(), n.sync(() => t.removeEventListener("abort", r));
+	});
+	return n.raceFirst(e, r);
+});
+//#endregion
+export { o as fetchIngestBody, s as withIngestAbort };

package/dist/ingestPolicy.js

@@ -0,0 +1 @@
+export { ssrfReason } from "@voltro/runtime";

package/dist/metadataError.js

@@ -0,0 +1,19 @@
+import { StorageError as e, StorageRejected as t } from "./types.js";
+import { causeBranches as n, extractDbCause as r, isTransientContention as i, wireSafeCause as a } from "@voltro/database";
+import { Schema as o } from "effect";
+//#region src/metadataError.ts
+var s = (t, s, c = !1) => {
+	let l = n(t).find(o.is(e));
+	if (l !== void 0) return l;
+	let u = r(t), d = u.dbCode === "22003" || u.dbSqlState === "22003" || u.dbCode === "ER_WARN_DATA_OUT_OF_RANGE" || u.dbErrno === 1264 || u.dbNumber === 8115 || u.dbNumber === 248, f = new e({
+		provider: s,
+		message: `${d ? "Storage metadata numeric value out of range: " : ""}${a(t)}`,
+		transient: d ? !1 : i(t) || c
+	});
+	return Object.defineProperty(f, "cause", {
+		value: t,
+		configurable: !0
+	}), f;
+}, c = (e) => n(e).find(o.is(t));
+//#endregion
+export { s as storageMetadataError, c as storageRejectedCause };

package/dist/objectLifecycle.js

@@ -0,0 +1,209 @@
+import { requireStorageReferenceScope as e } from "./storageNamespaces.js";
+import { StorageRejected as t } from "./types.js";
+import { storageInteger as n, storageIntegerSum as r } from "./storageNumbers.js";
+import { createHash as i, randomUUID as a } from "node:crypto";
+import { and as o, bigint as s, eq as c, neq as l, table as u, text as d } from "@voltro/database";
+import { Schema as f } from "effect";
+//#region src/objectLifecycle.ts
+var p = "_voltro_storage_operation_locks", m = "_voltro_storage_object_heads", h = "_voltro_storage_operations", g = "_voltro_storage_object_claims", _ = [
+	u(p, {
+		id: d().maxLength(64),
+		nonce: d()
+	}).primaryKey(["id"]).nonReactive(),
+	u(m, {
+		id: d().maxLength(64),
+		objectNamespace: d().maxLength(64).nullable(),
+		generation: d(),
+		key: d(),
+		reservations: s()
+	}).primaryKey(["id"]).nonReactive(),
+	u(h, {
+		id: d().maxLength(64),
+		storageInstance: d().nullable(),
+		objectNamespace: d().maxLength(64).nullable(),
+		objectId: d().maxLength(64),
+		fingerprint: d().maxLength(64),
+		generation: d(),
+		key: d(),
+		refId: d(),
+		state: d(),
+		deleteKey: d().nullable()
+	}).primaryKey(["id"]).nonReactive(),
+	u(g, {
+		id: d().maxLength(64),
+		operationId: d().maxLength(64),
+		claim: d()
+	}).primaryKey(["id"]).nonReactive()
+], v = f.Struct({
+	id: f.String,
+	storageInstance: f.String,
+	objectNamespace: f.String,
+	objectId: f.String,
+	fingerprint: f.String,
+	generation: f.String,
+	key: f.String,
+	refId: f.String,
+	state: f.Literals([
+		"reserved",
+		"ready",
+		"deleting",
+		"deleted"
+	]),
+	deleteKey: f.NullOr(f.String)
+}), y = f.Struct({
+	id: f.String,
+	objectNamespace: f.String,
+	generation: f.String,
+	key: f.String,
+	reservations: f.Int.pipe(f.check(f.isGreaterThanOrEqualTo(0)))
+}), b = (e) => f.decodeUnknownSync(y)({
+	...e,
+	reservations: n(e.reservations, "shared-object reservation count")
+}), x = (e) => i("sha256").update(JSON.stringify(e)).digest("hex"), S = (e, t) => x([e, t]), C = (t) => {
+	if (e(t), !t.bucket || !t.operationId || !t.logicalKey || !/^[a-f0-9]{64}$/.test(t.fingerprint) || !/^[a-f0-9]{64}$/.test(t.checksum)) throw Error("Storage operation requires a bucket, operation identity, key and SHA-256 fingerprint");
+	return {
+		id: x([
+			t.storageInstance,
+			t.objectNamespace,
+			t.bucket,
+			t.tenantId,
+			t.operationId
+		]),
+		objectId: x([
+			t.objectNamespace,
+			t.bucket,
+			t.tenantId,
+			t.logicalKey,
+			t.checksum
+		]),
+		lockId: x([t.tenantId])
+	};
+}, w = async (e, t, n) => {
+	let { lockId: r } = C(t);
+	return await e.insertIgnoreWithOutcome(p, {
+		id: r,
+		nonce: a()
+	}, { conflictColumns: ["id"] }), e.transactional(async (e) => {
+		if ((await e.query({
+			table: p,
+			predicate: c("id", r),
+			lock: "update",
+			order: [],
+			take: 1,
+			skip: void 0,
+			projection: void 0
+		})).length !== 1 || await e.update(p, r, { nonce: a() }) === null) throw Error("Storage operation lock is unavailable");
+		return n(e);
+	});
+}, T = async (e, n) => {
+	let r = C(n), i = await e.query({
+		table: h,
+		predicate: c("id", r.id),
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: void 0
+	});
+	if (!i[0]) return;
+	let a = f.decodeUnknownSync(v)(i[0]);
+	if (a.storageInstance !== n.storageInstance || a.objectNamespace !== n.objectNamespace || a.fingerprint !== n.fingerprint || a.objectId !== r.objectId) throw new t({
+		reason: "operation-identity-reused",
+		detail: `storage operation '${n.operationId}' was already used with different inputs`
+	});
+	return a;
+}, E = async (e, t) => {
+	let n = await T(e, t);
+	if (n) return n;
+	let { id: i, objectId: o } = C(t), s = await e.query({
+		table: m,
+		predicate: c("id", o),
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: void 0
+	}), l = s[0] ? b(s[0]) : void 0, u = l?.generation ?? a(), d = l?.key ?? `${t.logicalKey}/g-${u}`;
+	l ? await e.update(m, o, { reservations: String(r(l.reservations, 1, "shared-object reservation count")) }) : await e.insert(m, {
+		id: o,
+		objectNamespace: t.objectNamespace,
+		generation: u,
+		key: d,
+		reservations: "1"
+	});
+	let f = {
+		id: i,
+		storageInstance: t.storageInstance,
+		objectNamespace: t.objectNamespace,
+		objectId: o,
+		fingerprint: t.fingerprint,
+		generation: u,
+		key: d,
+		refId: `file_${a()}`,
+		state: "reserved",
+		deleteKey: null
+	};
+	return await e.insert(h, f), f;
+}, D = async (e, t) => {
+	let n = await T(e, t);
+	if (!n) throw Error("Storage operation is missing");
+	if (n.state === "deleting" || n.state === "deleted") throw Error("Deleted storage operation cannot be confirmed by a late put");
+	return await e.update(h, n.id, { state: "ready" }), {
+		...n,
+		state: "ready"
+	};
+}, O = async (e, t) => {
+	let n = await T(e, t);
+	if (!n) throw Error("Storage operation is missing");
+	if (n.state === "deleting" || n.state === "deleted") return n;
+	if (n.state !== "ready") throw Error("An unconfirmed put cannot be deleted");
+	let r = await e.query({
+		table: m,
+		predicate: c("id", n.objectId),
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: void 0
+	}), i = r[0] ? b(r[0]) : void 0;
+	if (!i || i.generation !== n.generation || i.reservations < 1) throw Error("Storage object generation does not own its reservation");
+	let a = i.reservations === 1 ? i.key : null;
+	return a === null ? await e.update(m, i.id, { reservations: String(i.reservations - 1) }) : await e.delete(m, i.id), a !== null && (await e.query({
+		table: m,
+		predicate: o(c("objectNamespace", t.objectNamespace), c("key", a)),
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: ["id"]
+	})).length && (a = null), await e.update(h, n.id, {
+		state: "deleting",
+		deleteKey: a
+	}), {
+		...n,
+		state: "deleting",
+		deleteKey: a
+	};
+}, k = async (e, t) => {
+	let n = await T(e, t);
+	if (!n || n.state !== "deleting" && n.state !== "deleted") throw Error("Storage deletion has not been accepted");
+	return n.state !== "deleted" && (await e.update(h, n.id, { state: "deleted" }), !0);
+}, A = async (e, t, n) => {
+	if (!n) throw Error("Storage object claim must be a non-empty string");
+	await e.insertIgnoreWithOutcome(g, {
+		id: S(t, n),
+		operationId: t,
+		claim: n
+	}, { conflictColumns: ["id"] });
+}, j = async (e, t, n) => {
+	let r = S(t, n);
+	return {
+		held: await e.delete(g, r),
+		others: (await e.query({
+			table: g,
+			predicate: o(c("operationId", t), l("claim", n)),
+			order: [],
+			take: 1,
+			skip: void 0,
+			projection: void 0
+		})).length > 0
+	};
+}, M = (e, t) => e.deleteMany(g, { where: c("operationId", t) });
+//#endregion
+export { O as beginStorageObjectDeleteInTransaction, A as claimStorageObjectInTransaction, M as clearStorageObjectClaimsInTransaction, k as confirmStorageObjectDeleteInTransaction, D as confirmStorageObjectInTransaction, T as readStorageObjectOperationInTransaction, j as releaseStorageObjectClaimInTransaction, E as reserveStorageObjectInTransaction, S as storageObjectClaimId, C as storageObjectIdentifiers, _ as storageObjectLifecycleTables, w as withStorageOperationTransaction };

package/dist/objectMetadata.js

@@ -0,0 +1,150 @@
+import { requireStorageReferenceScope as e } from "./storageNamespaces.js";
+import { StorageRejected as t } from "./types.js";
+import { beginStorageObjectDeleteInTransaction as n, claimStorageObjectInTransaction as r, clearStorageObjectClaimsInTransaction as i, confirmStorageObjectDeleteInTransaction as a, confirmStorageObjectInTransaction as o, readStorageObjectOperationInTransaction as s, releaseStorageObjectClaimInTransaction as c, reserveStorageObjectInTransaction as l, withStorageOperationTransaction as u } from "./objectLifecycle.js";
+import { dataStoreGrantStore as d, dataStoreRefStore as f, toRef as p } from "./service.js";
+import { and as m, eq as h, json as g, table as _, text as v } from "@voltro/database";
+import { Effect as y, Schema as b } from "effect";
+//#region src/objectMetadata.ts
+var x = "_voltro_storage_put_intents", S = _(x, {
+	id: v().maxLength(64),
+	refId: v().maxLength(64),
+	storageInstance: v().nullable(),
+	objectNamespace: v().maxLength(64).nullable(),
+	identity: g(),
+	ref: g()
+}).primaryKey(["id"]).unique("storagePutRef", ["refId"]).nonReactive(), C = b.Struct({
+	storageInstance: b.String,
+	objectNamespace: b.String,
+	tenantId: b.NullOr(b.String),
+	bucket: b.String,
+	operationId: b.String,
+	fingerprint: b.String,
+	checksum: b.String,
+	logicalKey: b.String
+}), w = async (e, t, n) => {
+	let r = await e.query({
+		table: x,
+		predicate: m(h("storageInstance", n.storageInstance), h("objectNamespace", n.objectNamespace), h("refId", t)),
+		order: [],
+		take: 2,
+		skip: void 0,
+		projection: void 0
+	});
+	if (r.length === 0) return;
+	if (r.length !== 1) throw Error("Storage reference has ambiguous operation ownership");
+	let i = r[0], a = b.decodeUnknownSync(C)(i.identity);
+	if (a.storageInstance !== n.storageInstance || a.objectNamespace !== n.objectNamespace) throw Error("Storage put intent does not match its bound storage scope");
+	if (typeof i.ref != "object" || i.ref === null || Array.isArray(i.ref)) throw Error("Storage operation reference is unreadable");
+	let o = p(i.ref);
+	if (o.id !== t || o.bucket !== a.bucket || o.tenantId !== a.tenantId || o.checksum !== a.checksum || !Number.isSafeInteger(o.size) || o.size < 0 || !Number.isFinite(o.createdAt.getTime())) throw Error("Storage operation reference does not match its identity");
+	return {
+		identity: a,
+		ref: o
+	};
+}, T = (m, h) => {
+	let g = Object.freeze({ ...h });
+	e(g);
+	let _ = (e) => ({
+		...e,
+		...g
+	});
+	return {
+		scope: g,
+		intent: (e) => w(m, e, g),
+		reserve: (e, n, i, a) => u(m, _(e), async (o) => {
+			if (n.tenantId !== e.tenantId || n.bucket !== e.bucket || n.checksum !== e.checksum || !Number.isSafeInteger(n.size) || n.size < 0) throw Error("Storage put metadata does not match its identity");
+			let c = await s(o, _(e));
+			if (c) {
+				if (c.state === "deleting" || c.state === "deleted") throw Error("Storage put identity has already been deleted");
+				let e = await w(o, c.refId, g);
+				if (!e) throw Error("Storage operation lost its put intent");
+				let t = c.state === "ready";
+				return t && a !== void 0 && await r(o, c.id, a), {
+					ref: e.ref,
+					ready: t,
+					outcome: t ? "joined" : "resumed"
+				};
+			}
+			let u = await l(o, _(e)), d = f(o, g);
+			if (!(await y.runPromise(d.consumeUsage(e.tenantId, {
+				bytes: n.size,
+				count: 1
+			}, i))).allowed) throw new t({
+				reason: "quota-exceeded",
+				detail: "Storage operation exceeds tenant quota"
+			});
+			let m = p({
+				...n,
+				id: u.refId,
+				key: u.key
+			});
+			return await o.insert(x, {
+				id: u.id,
+				refId: m.id,
+				...g,
+				identity: _(e),
+				ref: m
+			}), {
+				ref: m,
+				ready: !1,
+				outcome: "created"
+			};
+		}),
+		confirm: (e, t) => u(m, _(e), async (n) => {
+			let i = await s(n, _(e));
+			if (!i) throw Error("Storage put operation is missing");
+			let a = await w(n, i.refId, g);
+			if (!a) throw Error("Storage put intent is missing");
+			if (t !== void 0 && await r(n, i.id, t), i.state === "ready") {
+				let t = await y.runPromise(f(n, g).getById(i.refId));
+				if (!t || t.key !== i.key || t.checksum !== e.checksum) throw Error("Confirmed storage operation lost its reference");
+				return t;
+			}
+			return await o(n, _(e)), await y.runPromise(f(n, g).insert(a.ref)), a.ref;
+		}),
+		beginDelete: async (e, r, a) => {
+			let o = await w(m, e, g);
+			if (!o) throw Error("Storage delete operation is missing");
+			if (r !== void 0 && o.identity.tenantId !== r) throw Error("Storage delete tenant mismatch");
+			return u(m, o.identity, async (r) => {
+				let o = await w(r, e, g);
+				if (!o) throw Error("Storage delete intent disappeared");
+				let l = await s(r, o.identity);
+				if (!l) throw Error("Storage delete lost its operation");
+				let u = l.state === "ready", p = l.state !== "deleting" && l.state !== "deleted";
+				if (a === void 0) p && await i(r, l.id);
+				else if (p) {
+					let n = await c(r, l.id, a);
+					if (n.others) return {
+						released: !0,
+						identity: o.identity
+					};
+					if (!n.held) throw new t({
+						reason: "claim-not-held",
+						detail: `claim '${a}' is not held on storage reference '${e}'; a reference put without a claim is deleted as a whole by a delete that names none`
+					});
+				}
+				let m = await n(r, o.identity);
+				if (u) {
+					let t = f(r, g);
+					if (await y.runPromise(d(r, g).deleteByRef(e)), !await y.runPromise(t.delete(e))) throw Error("Storage deletion lost its confirmed reference");
+					await y.runPromise(t.releaseUsage(o.identity.tenantId, {
+						bytes: o.ref.size,
+						count: 1
+					}));
+				}
+				return m.deleteKey !== null && await y.runPromise(f(r, g).countByKey(o.ref.bucket, m.deleteKey)) > 0 && (await r.update("_voltro_storage_operations", m.id, { deleteKey: null }), m = {
+					...m,
+					deleteKey: null
+				}), {
+					released: !0,
+					operation: m,
+					identity: o.identity
+				};
+			});
+		},
+		confirmDelete: (e) => u(m, _(e), (t) => a(t, _(e)))
+	};
+};
+//#endregion
+export { T as makeStorageObjectMetadata, S as storagePutIntentsTable };

package/dist/objectWriter.js

@@ -0,0 +1,58 @@
+import { StorageError as e, StorageRejected as t } from "./types.js";
+import { storageMetadataError as n, storageRejectedCause as r } from "./metadataError.js";
+import { createHash as i } from "node:crypto";
+import { Effect as a } from "effect";
+//#region src/objectWriter.ts
+var o = (o) => {
+	let s = a.gen(function* () {
+		(yield* o.provider.objectNamespace) !== o.metadata.scope.objectNamespace && (yield* a.fail(new e({
+			provider: "metadata",
+			message: "Storage metadata is bound to another physical object namespace",
+			transient: !1,
+			status: 409
+		})));
+	}), c = (t, i) => a.tryPromise({
+		try: i,
+		catch: (i) => {
+			let a = r(i);
+			if (a !== void 0) return a;
+			let o = n(i, "metadata"), s = new e({
+				provider: "metadata",
+				message: `Storage ${t} was not confirmed: ${o.message}; retain the operation identity for recovery`,
+				transient: o.transient,
+				...o.status === void 0 ? {} : { status: o.status }
+			});
+			return Object.defineProperty(s, "cause", {
+				value: i,
+				configurable: !0
+			}), s;
+		}
+	});
+	return {
+		put: (e) => a.gen(function* () {
+			if (e.bytes.byteLength !== e.ref.size || i("sha256").update(e.bytes).digest("hex") !== e.identity.checksum) return yield* a.fail(new t({
+				reason: "operation-content-mismatch",
+				detail: "Storage operation bytes do not match their reserved identity"
+			}));
+			yield* s;
+			let n = yield* c("put reservation", () => o.metadata.reserve(e.identity, e.ref, e.quota, e.claim)), r = n.outcome;
+			return n.ready ? {
+				ref: n.ref,
+				outcome: r
+			} : (yield* o.provider.put(n.ref.key, e.bytes, {
+				contentType: n.ref.contentType,
+				public: n.ref.visibility === "public"
+			}), {
+				ref: yield* c("put confirmation", () => o.metadata.confirm(e.identity, e.claim)),
+				outcome: r
+			});
+		}),
+		delete: (e, t, n) => a.gen(function* () {
+			yield* s;
+			let r = yield* c("delete acceptance", () => o.metadata.beginDelete(e, t, n));
+			"operation" in r && (r.operation.deleteKey !== null && (yield* o.provider.delete(r.operation.deleteKey)), yield* c("delete confirmation", () => o.metadata.confirmDelete(r.identity)));
+		})
+	};
+};
+//#endregion
+export { o as makeStorageObjectWriter };

package/dist/pathDurability.js

@@ -0,0 +1,16 @@
+import { statSync as e } from "node:fs";
+//#region src/pathDurability.ts
+var t = (t) => {
+	try {
+		let n = e(t), r = e("/");
+		return typeof n.dev != "number" || typeof r.dev != "number" ? "unknown" : n.dev === r.dev ? "same-device-as-root" : "separate-mount";
+	} catch {
+		return "unknown";
+	}
+}, n = (e) => {
+	if (t(e) === "same-device-as-root") return `the capture is going to ${e}, which is on the SAME filesystem as \`/\`.\n  A mounted volume would be a separate one. If this process is a container, that means the capture
+  dies with it — which is the exact failure it exists for. On a development machine this is normal
+  and can be ignored.`;
+};
+//#endregion
+export { n as durabilityNotice, t as pathDurability };