@voltro/plugin-storage 0.84.0 → 0.86.0

package/dist/providers.js

@@ -0,0 +1,493 @@
+import { canonicalStorageRoot as e, filesystemObjectNamespace as t, storageEndpointIdentity as n, storageObjectNamespace as r } from "./storageNamespaces.js";
+import { STORAGE_BLOBS_TABLE as i } from "./table.js";
+import { StorageError as a } from "./types.js";
+import { storageInteger as o } from "./storageNumbers.js";
+import { concatBytes as s, fromNodeReadable as c, toNodeReadable as l } from "./streamBridge.js";
+import { storageEnv as u } from "./env.js";
+import { storageMetadataError as d } from "./metadataError.js";
+import { randomUUID as f } from "node:crypto";
+import { authoredError as p, generateId as m } from "@voltro/database";
+import { Effect as h } from "effect";
+//#region src/providers.ts
+var g = (e, t, n, r) => new a({
+	provider: e,
+	message: t,
+	transient: n,
+	...r === void 0 ? {} : { status: r }
+}), _ = (e) => e === void 0 || e === 408 || e === 429 || e >= 500, v = f(), y = /* @__PURE__ */ new Map(), b = (e, t) => y.get(e)?.get(t)?.bytes, x = (e) => {
+	y.get(e)?.clear();
+}, S = (e = "default") => {
+	let t = y.get(e) ?? /* @__PURE__ */ new Map();
+	return y.set(e, t), {
+		name: "memory",
+		objectNamespace: h.succeed(r([
+			"memory",
+			v,
+			e
+		])),
+		presigns: !1,
+		put: (e, n, r) => h.sync(() => (t.set(e, {
+			bytes: n,
+			contentType: r.contentType
+		}), { size: n.byteLength })),
+		get: (e) => h.suspend(() => {
+			let n = t.get(e);
+			return n ? h.succeed({
+				bytes: n.bytes,
+				contentType: n.contentType
+			}) : h.fail(g("memory", `no object at key '${e}'`, !1, 404));
+		}),
+		delete: (e) => h.sync(() => {
+			t.delete(e);
+		}),
+		publicUrl: () => null,
+		getUrl: () => h.succeed(""),
+		head: (e) => h.sync(() => {
+			let n = t.get(e);
+			return n ? { size: n.bytes.byteLength } : null;
+		})
+	};
+}, C = (n) => {
+	let r = e(n.root), i = async (e) => {
+		if (e.split("/")[0]?.startsWith(".voltro-object-namespace")) throw Error("Storage namespace metadata is not an object key");
+		let { join: t, resolve: n, sep: i } = await import("node:path"), a = n(r), o = n(a, e);
+		if (o.startsWith(n(a, ".voltro-object-namespace"))) throw Error("Storage namespace metadata is not an object key");
+		if (o !== a && !o.startsWith(a + i)) throw Error(`key '${e}' escapes the storage root`);
+		return t(o);
+	}, a = (e, t, n = !1) => h.tryPromise({
+		try: e,
+		catch: (e) => e?.code === "ENOENT" ? g("filesystem", `no object at key '${t}'`, !1, 404) : g("filesystem", String(e?.message ?? e), n)
+	});
+	return {
+		root: r,
+		name: "filesystem",
+		objectNamespace: h.try({
+			try: () => t(r),
+			catch: (e) => g("filesystem", String(e instanceof Error ? e.message : e), !1)
+		}),
+		presigns: !1,
+		put: (e, t, n) => a(async () => {
+			let n = await import("node:fs/promises"), { dirname: r } = await import("node:path"), { randomUUID: a } = await import("node:crypto"), o = await i(e), s = r(o);
+			await n.mkdir(s, { recursive: !0 });
+			let c = `${o}.tmp-${a()}`;
+			try {
+				await n.writeFile(c, t), await n.rename(c, o);
+			} catch (e) {
+				throw await n.rm(c, { force: !0 }).catch(() => {}), e;
+			}
+			return { size: t.byteLength };
+		}, e, !0),
+		get: (e) => a(async () => {
+			let t = await (await import("node:fs/promises")).readFile(await i(e));
+			return {
+				bytes: new Uint8Array(t),
+				contentType: "application/octet-stream"
+			};
+		}, e),
+		delete: (e) => a(async () => {
+			await (await import("node:fs/promises")).rm(await i(e), { force: !0 });
+		}, e),
+		publicUrl: () => null,
+		getUrl: () => h.succeed(""),
+		head: (e) => h.tryPromise({
+			try: async () => {
+				let t = await import("node:fs/promises");
+				try {
+					return { size: (await t.stat(await i(e))).size };
+				} catch (e) {
+					if (e?.code === "ENOENT") return null;
+					throw e;
+				}
+			},
+			catch: (e) => g("filesystem", String(e?.message ?? e), !1)
+		}),
+		getStream: (e) => a(async () => {
+			let t = await import("node:fs/promises"), { createReadStream: n } = await import("node:fs"), r = await i(e), a = await t.stat(r);
+			return {
+				stream: c("filesystem", () => n(r)),
+				contentType: "application/octet-stream",
+				size: a.size
+			};
+		}, e),
+		putStream: (e, t, n) => a(async () => {
+			let n = await import("node:fs/promises"), { dirname: r } = await import("node:path"), { randomUUID: a } = await import("node:crypto"), { createWriteStream: o } = await import("node:fs"), { pipeline: s } = await import("node:stream/promises"), c = await i(e);
+			await n.mkdir(r(c), { recursive: !0 });
+			let u = `${c}.tmp-${a()}`;
+			try {
+				await s(await l(t), o(u));
+				let e = await n.stat(u);
+				return await n.rename(u, c), { size: e.size };
+			} catch (e) {
+				throw await n.rm(u, { force: !0 }).catch(() => {}), e;
+			}
+		}, e, !0),
+		getRange: (e, t, n) => a(async () => {
+			let { createReadStream: r } = await import("node:fs"), a = await i(e), o = [];
+			return await new Promise((e, i) => {
+				let s = r(a, {
+					start: t,
+					end: n
+				});
+				s.on("data", (e) => o.push(e)), s.on("end", () => e()), s.on("error", i);
+			}), s(o);
+		}, e)
+	};
+}, w = (e) => {
+	let t = e.name ?? "s3", i = e.publicAcl ?? !0, a = null, o = () => a || (a = (async () => {
+		let t, n;
+		try {
+			t = await import("@aws-sdk/client-s3"), n = await import("@aws-sdk/s3-request-presigner");
+		} catch (e) {
+			throw Error("the 's3'/'minio' storage provider requires the '@aws-sdk/client-s3' + '@aws-sdk/s3-request-presigner' optional dependencies. Install them to enable it.", { cause: e });
+		}
+		return {
+			client: new t.S3Client({
+				...e.region ? { region: e.region } : { region: "us-east-1" },
+				...e.endpoint ? { endpoint: e.endpoint } : {},
+				...e.forcePathStyle ? { forcePathStyle: !0 } : {},
+				...e.accessKeyId && e.secretAccessKey ? { credentials: {
+					accessKeyId: e.accessKeyId,
+					secretAccessKey: e.secretAccessKey
+				} } : {}
+			}),
+			cmds: t,
+			presign: n.getSignedUrl
+		};
+	})(), a), s = (e) => h.tryPromise({
+		try: e,
+		catch: (e) => {
+			let n = e?.$metadata?.httpStatusCode;
+			return g(t, String(e?.message ?? e), _(n), n);
+		}
+	}), u = async (e) => {
+		let t = e;
+		if (typeof t.transformToByteArray == "function") return t.transformToByteArray();
+		let n = [];
+		for await (let e of t) n.push(e);
+		let r = n.reduce((e, t) => e + t.byteLength, 0), i = new Uint8Array(r), a = 0;
+		for (let e of n) i.set(e, a), a += e.byteLength;
+		return i;
+	};
+	return {
+		name: t,
+		objectNamespace: h.succeed(r([
+			"s3",
+			e.endpoint ? n(e.endpoint) : "aws-s3",
+			e.bucket
+		])),
+		presigns: !0,
+		put: (t, n, r) => s(async () => {
+			let { client: a, cmds: s } = await o();
+			return await a.send(new s.PutObjectCommand({
+				Bucket: e.bucket,
+				Key: t,
+				Body: n,
+				ContentType: r.contentType,
+				...r.public && i ? { ACL: "public-read" } : {}
+			})), { size: n.byteLength };
+		}),
+		get: (t) => s(async () => {
+			let { client: n, cmds: r } = await o(), i = await n.send(new r.GetObjectCommand({
+				Bucket: e.bucket,
+				Key: t
+			}));
+			return {
+				bytes: await u(i.Body),
+				contentType: i.ContentType ?? "application/octet-stream"
+			};
+		}),
+		delete: (t) => s(async () => {
+			let { client: n, cmds: r } = await o();
+			await n.send(new r.DeleteObjectCommand({
+				Bucket: e.bucket,
+				Key: t
+			}));
+		}),
+		publicUrl: (t) => e.cdnBaseUrl ? `${e.cdnBaseUrl.replace(/\/$/, "")}/${t}` : null,
+		...i ? { setObjectPublic: (t, n) => s(async () => {
+			let { client: r, cmds: i } = await o();
+			await r.send(new i.PutObjectAclCommand({
+				Bucket: e.bucket,
+				Key: t,
+				ACL: n ? "public-read" : "private"
+			}));
+		}) } : {},
+		getUrl: (t, n) => s(async () => {
+			let { client: r, cmds: i, presign: a } = await o();
+			return a(r, new i.GetObjectCommand({
+				Bucket: e.bucket,
+				Key: t
+			}), { expiresIn: n });
+		}),
+		getUploadUrl: (t, n, r) => s(async () => {
+			let { client: i, cmds: a, presign: s } = await o();
+			return s(i, new a.PutObjectCommand({
+				Bucket: e.bucket,
+				Key: t,
+				ContentType: r
+			}), { expiresIn: n });
+		}),
+		createMultipartUpload: (t, n, r) => s(async () => {
+			let { client: a, cmds: s } = await o(), c = await a.send(new s.CreateMultipartUploadCommand({
+				Bucket: e.bucket,
+				Key: t,
+				ContentType: n,
+				...r?.public && i ? { ACL: "public-read" } : {}
+			}));
+			if (!c.UploadId) throw Error("s3 createMultipartUpload returned no UploadId");
+			return { uploadId: c.UploadId };
+		}),
+		presignUploadPart: (t, n, r, i) => s(async () => {
+			let { client: a, cmds: s, presign: c } = await o();
+			return c(a, new s.UploadPartCommand({
+				Bucket: e.bucket,
+				Key: t,
+				UploadId: n,
+				PartNumber: r
+			}), { expiresIn: i });
+		}),
+		completeMultipartUpload: (t, n, r) => s(async () => {
+			let { client: i, cmds: a } = await o(), s = await i.send(new a.CompleteMultipartUploadCommand({
+				Bucket: e.bucket,
+				Key: t,
+				UploadId: n,
+				MultipartUpload: { Parts: r.map((e) => ({
+					PartNumber: e.partNumber,
+					ETag: e.etag
+				})) }
+			}));
+			return s.ETag === void 0 ? {} : { etag: s.ETag };
+		}),
+		abortMultipartUpload: (t, n) => s(async () => {
+			let { client: r, cmds: i } = await o();
+			await r.send(new i.AbortMultipartUploadCommand({
+				Bucket: e.bucket,
+				Key: t,
+				UploadId: n
+			}));
+		}),
+		head: (n) => h.tryPromise({
+			try: async () => {
+				let { client: t, cmds: r } = await o();
+				try {
+					let i = await t.send(new r.HeadObjectCommand({
+						Bucket: e.bucket,
+						Key: n
+					}));
+					return {
+						size: i.ContentLength ?? 0,
+						...i.ETag === void 0 ? {} : { etag: i.ETag }
+					};
+				} catch (e) {
+					if (e?.$metadata?.httpStatusCode === 404) return null;
+					throw e;
+				}
+			},
+			catch: (e) => {
+				let n = e?.$metadata?.httpStatusCode;
+				return g(t, String(e?.message ?? e), _(n), n);
+			}
+		}),
+		getStream: (n) => s(async () => {
+			let { client: r, cmds: i } = await o(), a = await r.send(new i.GetObjectCommand({
+				Bucket: e.bucket,
+				Key: n
+			})), s = a.Body;
+			return {
+				stream: c(t, () => s),
+				contentType: a.ContentType ?? "application/octet-stream",
+				...a.ContentLength === void 0 ? {} : { size: a.ContentLength }
+			};
+		}),
+		putStream: (t, n, r) => s(async () => {
+			let { client: a, cmds: s } = await o(), c;
+			try {
+				c = await import("@aws-sdk/lib-storage");
+			} catch (e) {
+				throw Error("s3/minio `putStream` requires the '@aws-sdk/lib-storage' optional dependency (multipart upload). Install it to enable streamed uploads.", { cause: e });
+			}
+			let u = await l(n);
+			return await new c.Upload({
+				client: a,
+				params: {
+					Bucket: e.bucket,
+					Key: t,
+					Body: u,
+					ContentType: r.contentType,
+					...r.public && i ? { ACL: "public-read" } : {}
+				}
+			}).done(), { size: r.totalSize ?? (await a.send(new s.HeadObjectCommand({
+				Bucket: e.bucket,
+				Key: t
+			}))).ContentLength ?? 0 };
+		}),
+		getRange: (t, n, r) => s(async () => {
+			let { client: i, cmds: a } = await o(), s = await i.send(new a.GetObjectCommand({
+				Bucket: e.bucket,
+				Key: t,
+				Range: `bytes=${n}-${r}`
+			}));
+			return u(s.Body);
+		})
+	};
+}, T = (e) => e instanceof Uint8Array ? e : typeof e == "string" ? new Uint8Array(Buffer.from(e, "base64")) : new Uint8Array(e), E = () => {
+	let e = null, t = () => {
+		if (!e) throw p("the 'database' storage provider isn't bound to a DataStore yet");
+		return e;
+	}, n = (e) => h.tryPromise({
+		try: e,
+		catch: (e) => d(e, "database", !0)
+	}), a = async (e, n) => (await t().query({
+		table: i,
+		predicate: {
+			column: "key",
+			op: "eq",
+			value: e
+		},
+		order: [],
+		take: 1,
+		skip: void 0,
+		projection: n
+	}))[0];
+	return {
+		name: "database",
+		objectNamespace: h.succeed(r(["database", i])),
+		presigns: !1,
+		bindDataStore: (t) => {
+			e = t;
+		},
+		put: (e, r, o) => n(async () => {
+			let n = await a(e);
+			return n ? await t().update(i, String(n.id), {
+				data: r,
+				contentType: o.contentType,
+				size: String(r.byteLength)
+			}) : await t().insert(i, {
+				id: m({
+					kind: "typeid",
+					prefix: "blob"
+				}, i),
+				key: e,
+				data: r,
+				contentType: o.contentType,
+				size: String(r.byteLength)
+			}), { size: r.byteLength };
+		}),
+		get: (e) => h.suspend(() => n(() => a(e)).pipe(h.flatMap((t) => t ? h.succeed({
+			bytes: T(t.data),
+			contentType: String(t.contentType)
+		}) : h.fail(g("database", `no object at key '${e}'`, !1, 404))))),
+		delete: (e) => n(async () => {
+			let n = await a(e, ["id"]);
+			n && await t().delete(i, String(n.id));
+		}),
+		publicUrl: () => null,
+		getUrl: () => h.succeed(""),
+		head: (e) => n(async () => {
+			let t = await a(e, ["size"]);
+			return t ? { size: o(t.size, "blob size") } : null;
+		})
+	};
+}, D = (e) => {
+	let t = null, i = () => t || (t = (async () => {
+		let t;
+		try {
+			t = await import("@azure/storage-blob");
+		} catch (e) {
+			throw Error("the 'azure' storage provider requires the '@azure/storage-blob' optional dependency. Install it to enable it.", { cause: e });
+		}
+		let n = u.read("AZURE_STORAGE_ACCOUNT", e.accountName), r = u.read("AZURE_STORAGE_KEY", e.accountKey), i = n && r ? new t.StorageSharedKeyCredential(n, r) : null, a = u.read("AZURE_STORAGE_CONNECTION_STRING", e.connectionString);
+		return {
+			container: (a ? t.BlobServiceClient.fromConnectionString(a) : new t.BlobServiceClient(e.endpoint ?? `https://${n}.blob.core.windows.net`, i ?? void 0)).getContainerClient(e.container),
+			sdk: t,
+			cred: i
+		};
+	})(), t), a = (e) => h.tryPromise({
+		try: e,
+		catch: (e) => {
+			let t = e?.statusCode;
+			return g("azure", String(e?.message ?? e), _(t), t);
+		}
+	}), o = async (e) => {
+		if (!e) return /* @__PURE__ */ new Uint8Array();
+		let t = [];
+		for await (let n of e) t.push(typeof n == "string" ? new TextEncoder().encode(n) : new Uint8Array(n));
+		let n = t.reduce((e, t) => e + t.byteLength, 0), r = new Uint8Array(n), i = 0;
+		for (let e of t) r.set(e, i), i += e.byteLength;
+		return r;
+	}, s = async (t, n, r) => {
+		let { container: a, sdk: o, cred: s } = await i();
+		if (!s) throw Error("azure SAS requires accountName + accountKey (set them or AZURE_STORAGE_ACCOUNT/AZURE_STORAGE_KEY)");
+		let c = a.getBlockBlobClient(t), l = o.generateBlobSASQueryParameters({
+			containerName: e.container,
+			blobName: t,
+			permissions: o.BlobSASPermissions.parse(n),
+			startsOn: /* @__PURE__ */ new Date(Date.now() - 6e4),
+			expiresOn: new Date(Date.now() + r * 1e3)
+		}, s).toString();
+		return `${c.url}?${l}`;
+	};
+	return {
+		name: "azure",
+		objectNamespace: a(async () => {
+			let { container: e } = await i();
+			return r(["azure", n(e.url)]);
+		}),
+		presigns: !0,
+		put: (e, t, n) => a(async () => {
+			let { container: r } = await i();
+			return await r.getBlockBlobClient(e).uploadData(Buffer.from(t), { blobHTTPHeaders: { blobContentType: n.contentType } }), { size: t.byteLength };
+		}),
+		get: (e) => a(async () => {
+			let { container: t } = await i(), n = await t.getBlockBlobClient(e).download();
+			return {
+				bytes: await o(n.readableStreamBody),
+				contentType: n.contentType ?? "application/octet-stream"
+			};
+		}),
+		delete: (e) => a(async () => {
+			let { container: t } = await i();
+			await t.getBlockBlobClient(e).deleteIfExists();
+		}),
+		publicUrl: (t) => e.cdnBaseUrl ? `${e.cdnBaseUrl.replace(/\/$/, "")}/${t}` : null,
+		getUrl: (e, t) => a(() => s(e, "r", t)),
+		getUploadUrl: (e, t, n) => a(() => s(e, "cw", t)),
+		head: (e) => h.tryPromise({
+			try: async () => {
+				let { container: t } = await i();
+				try {
+					let n = await t.getBlockBlobClient(e).getProperties();
+					return {
+						size: n.contentLength ?? 0,
+						...n.etag === void 0 ? {} : { etag: n.etag }
+					};
+				} catch (e) {
+					if (e?.statusCode === 404) return null;
+					throw e;
+				}
+			},
+			catch: (e) => {
+				let t = e?.statusCode;
+				return g("azure", String(e?.message ?? e), _(t), t);
+			}
+		}),
+		getStream: (e) => a(async () => {
+			let { container: t } = await i(), n = await t.getBlockBlobClient(e).download();
+			return {
+				stream: c("azure", () => n.readableStreamBody),
+				contentType: n.contentType ?? "application/octet-stream",
+				...n.contentLength === void 0 ? {} : { size: n.contentLength }
+			};
+		}),
+		putStream: (e, t, n) => a(async () => {
+			let { container: r } = await i(), a = await l(t);
+			return await r.getBlockBlobClient(e).uploadStream(a, 4194304, 5, { blobHTTPHeaders: { blobContentType: n.contentType } }), { size: n.totalSize ?? (await r.getBlockBlobClient(e).getProperties()).contentLength ?? 0 };
+		}),
+		getRange: (e, t, n) => a(async () => {
+			let { container: r } = await i(), a = await r.getBlockBlobClient(e).download(t, n - t + 1);
+			return o(a.readableStreamBody);
+		})
+	};
+};
+//#endregion
+export { D as azureProvider, x as clearStorageBuffer, E as databaseProvider, C as filesystemProvider, S as memoryProvider, b as readStorageBuffer, w as s3Provider };

package/dist/referenceCatalogue.js

@@ -0,0 +1,31 @@
+import { storageObjectNamespace as e } from "./storageNamespaces.js";
+import { randomUUID as t } from "node:crypto";
+import { authoredError as n, eq as r, table as i, text as a } from "@voltro/database";
+//#region src/referenceCatalogue.ts
+var o = "_voltro_storage_reference_catalogue", s = i(o, {
+	id: a().maxLength(64),
+	namespace: a().maxLength(64)
+}).primaryKey(["id"]).nonReactive(), c = async (n) => {
+	await n.insertIgnoreWithOutcome(o, {
+		id: "catalogue",
+		namespace: e(["reference-catalogue", t()])
+	}, { conflictColumns: ["id"] }), await l(n);
+}, l = async (t, i) => {
+	let a = await t.query({
+		table: o,
+		predicate: r("id", "catalogue"),
+		order: [],
+		take: 2,
+		skip: void 0,
+		projection: void 0
+	}), s = a[0]?.namespace;
+	if (a.length !== 1 || typeof s != "string" || !/^[a-f0-9]{64}$/.test(s)) throw n("Storage reference catalogue identity is missing or invalid; initialize it after normal schema reconciliation before serving inspection");
+	return i ? e([
+		"reference-view",
+		s,
+		i.storageInstance,
+		i.objectNamespace
+	]) : s;
+};
+//#endregion
+export { o as STORAGE_REFERENCE_CATALOGUE_TABLE, c as ensureStorageReferenceCatalogue, l as storageReferenceCatalogueId, s as storageReferenceCatalogueTable };