@voltro/data-transfer 0.86.0 → 0.88.0

package/dist/assetAdmission.js

@@ -0,0 +1,91 @@
+import { BundleError as e, IntegrityError as t } from "./errors.js";
+import { admitFilesystemPath as n } from "./filesystemAdmission.js";
+import { AssetEntry as r } from "./manifest.js";
+import { physicalRequest as i } from "./physicalRequest.js";
+import { Effect as a, Schema as o } from "effect";
+import { createHash as s } from "node:crypto";
+import { constants as c, createWriteStream as l } from "node:fs";
+import { lstat as u, mkdtemp as d, open as f, rm as p } from "node:fs/promises";
+import { join as m } from "node:path";
+import { tmpdir as h } from "node:os";
+import { Transform as g } from "node:stream";
+import { pipeline as _ } from "node:stream/promises";
+import { storageAddressIdentity as v } from "@voltro/plugin-storage/errors";
+// src/assetAdmission.ts
+var y = Symbol("admitted-assets"), b = /* @__PURE__ */ new WeakMap(), x = (t) => o.decodeUnknownEffect(r)(t).pipe(a.mapError(() => new e({ reason: "asset index contains invalid key, digest, size or content type" }))), S = o.Struct({
+	...r.fields,
+	sha256: o.optionalKey(r.fields.sha256)
+}), C = (t) => o.decodeUnknownEffect(S)(t).pipe(a.mapError(() => new e({ reason: "asset source metadata contains invalid destination, tenant, key, digest, size or content type" }))), w = (t, n) => b.get(t) === n ? a.void : a.fail(new e({ reason: "asset admission is absent, belongs to another sink or its scope has closed" })), T = (e) => w(e.prepared, e.previousSink).pipe(a.flatMap(() => E({
+	entries: e.prepared.entries,
+	assetsDir: e.prepared.assetsDir,
+	sink: e.sink
+}))), E = (r) => a.gen(function* () {
+	yield* a.tryPromise({
+		try: () => n(r.assetsDir, "directory"),
+		catch: () => new e({ reason: "asset source directory could not be admitted" })
+	});
+	let o = [], S = /* @__PURE__ */ new Map();
+	for (let t of r.entries) {
+		let n = yield* x(t), r = v(n), i = S.get(r);
+		if (i !== void 0) {
+			if (i.sha256 !== n.sha256 || i.size !== n.size || i.contentType !== n.contentType) return yield* a.fail(new e({ reason: "asset index contains conflicting entries for one key" }));
+		} else S.set(r, n), o.push(n);
+	}
+	let C = yield* a.acquireRelease(a.tryPromise({
+		try: () => d(m(h(), "voltro-admitted-assets-")),
+		catch: () => new e({ reason: "cannot create asset admission directory" })
+	}), (e) => a.promise(() => p(e, {
+		recursive: !0,
+		force: !0
+	}))), w = /* @__PURE__ */ new Set();
+	for (let d of o) if (yield* r.sink.has(d), w.has(d.sha256)) {
+		let n = yield* a.tryPromise({
+			try: () => u(m(C, d.sha256)).then((e) => e.size),
+			catch: () => new e({ reason: "admitted asset body is unavailable" })
+		});
+		if (n !== d.size) return yield* a.fail(new t({
+			what: `asset ${d.key} size`,
+			expected: String(d.size),
+			actual: String(n)
+		}));
+	} else {
+		let o = yield* i(async (e) => {
+			let t = m(r.assetsDir, d.sha256);
+			await n(t, "file");
+			let i = await f(t, c.O_RDONLY | c.O_NOFOLLOW | c.O_NONBLOCK);
+			try {
+				if (!(await i.stat()).isFile()) throw Error("asset body must be a regular file");
+				let t = s("sha256"), n = 0, r = new g({ transform(e, r, i) {
+					t.update(e), n += e.length, i(null, e);
+				} });
+				return await _(i.createReadStream({ autoClose: !1 }), r, l(m(C, d.sha256), {
+					flags: "wx",
+					mode: 384
+				}), { signal: e }), {
+					sha256: t.digest("hex"),
+					size: n
+				};
+			} finally {
+				await i.close();
+			}
+		}, () => new e({
+			reason: "asset body could not be admitted",
+			path: d.sha256
+		}));
+		if (o.sha256 !== d.sha256 || o.size !== d.size) return yield* a.fail(new t({
+			what: `asset ${d.key}`,
+			expected: `${d.sha256}/${d.size}`,
+			actual: `${o.sha256}/${o.size}`
+		}));
+		w.add(d.sha256);
+	}
+	let T = Object.freeze({
+		[y]: !0,
+		assetsDir: C,
+		entries: Object.freeze(o.map((e) => Object.freeze(e)))
+	});
+	return b.set(T, r.sink), yield* a.addFinalizer(() => a.sync(() => {
+		b.delete(T);
+	})), T;
+});
+export { S as AssetSourceInfo, w as assertPreparedAssets, x as decodeAssetEntry, C as decodeAssetSourceInfo, E as prepareAssetEntries, T as readmitPreparedAssets };

package/dist/assets.js

@@ -1,47 +1,67 @@
-import { BundleError as e, IntegrityError as t } from "./errors.js";
-import { Effect as n, Stream as r } from "effect";
-import { fromNodeReadable as i, getObjectStream as a, putObjectStream as o, toNodeReadable as s } from "@voltro/plugin-storage";
-//#region src/assets.ts
-var c = (e, t) => ({
+import { BundleError as e } from "./errors.js";
+import { admitFilesystemPath as t } from "./filesystemAdmission.js";
+import { physicalRequest as n } from "./physicalRequest.js";
+import { assertPreparedAssets as r, prepareAssetEntries as i } from "./assetAdmission.js";
+import { Effect as a, Stream as o } from "effect";
+import { createHash as s } from "node:crypto";
+import { fromNodeReadable as c, getObjectStream as l, putObjectStream as u, toNodeReadable as d } from "@voltro/plugin-storage";
+import { storageAddressIdentity as f } from "@voltro/plugin-storage/errors";
+// src/assets.ts
+var p = (e, t) => ({
 	list: t,
-	open: (t) => a(e, t)
-}), l = (e) => ({
-	write: (t, n, r) => o(e, t, n, r),
-	has: (t) => e.head(t).pipe(n.map((e) => e !== null))
-}), u = (t, r) => n.tryPromise({
-	try: async () => {
-		let { createWriteStream: e } = await import("node:fs"), n = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: c } = await import("node:stream"), l = await import("node:path"), u = l.join(r, `.tmp-${a()}`), d = i("sha256"), f = 0, p = new c({ transform(e, t, n) {
-			d.update(e), f += e.length, n(null, e);
+	open: (t) => e.resolve(t).pipe(a.flatMap((e) => l(e, t.key)))
+}), m = (e) => ({
+	write: (t, n, r) => e.resolve(t).pipe(a.flatMap((e) => u(e, t.key, n, r))),
+	has: (t) => e.resolve(t).pipe(a.flatMap((e) => e.head(t.key)), a.map((e) => e !== null)),
+	matches: (t) => a.gen(function* () {
+		let n = yield* e.resolve(t), r = yield* n.head(t.key);
+		if (r === null || r.size !== t.size || r.contentType !== void 0 && r.contentType !== t.contentType) return !1;
+		let i = yield* l(n, t.key);
+		if (i.contentType !== t.contentType) return !1;
+		let a = s("sha256");
+		return (yield* o.runFold(i.stream, () => 0, (e, t) => (a.update(t), e + t.byteLength))) === t.size && a.digest("hex") === t.sha256;
+	})
+}), h = (r, i) => n(async (e) => {
+	let { createWriteStream: n } = await import("node:fs"), a = await import("node:fs/promises"), { pipeline: o, finished: c } = await import("node:stream/promises"), { Transform: l } = await import("node:stream"), u = await import("node:path");
+	await t(i, "directory");
+	let f = await a.mkdtemp(u.join(i, ".voltro-capture-")), p;
+	try {
+		let c = u.join(f, "body"), m = s("sha256"), h = 0, g = new l({ transform(e, t, n) {
+			m.update(e), h += e.length, n(null, e);
 		} });
-		await o(await s(t.stream), p, e(u));
-		let m = d.digest("hex"), h = l.join(r, m);
-		try {
-			await n.access(h), await n.rm(u, { force: !0 });
-		} catch {
-			await n.rename(u, h);
-		}
-		return {
-			sha256: m,
-			size: f,
-			contentType: t.contentType
+		p = await d(r.stream), await o(p, g, n(c, {
+			flags: "wx",
+			mode: 384
+		}), { signal: e });
+		let _ = m.digest("hex"), v = u.join(i, _);
+		return await t(v, "file"), await a.rename(c, v), {
+			sha256: _,
+			size: h,
+			contentType: r.contentType
 		};
-	},
-	catch: (t) => new e({
-		reason: `asset write failed: ${String(t?.message ?? t)}`,
-		path: r
-	})
-}), d = (e, a, o) => n.gen(function* () {
-	let s = yield* n.promise(() => import("node:path")), { createReadStream: c } = yield* n.promise(() => import("node:fs")), { createHash: l } = yield* n.promise(() => import("node:crypto")), u = s.join(a, o.sha256), d = l("sha256"), f = i("bundle", () => c(u)).pipe(r.tap((e) => n.sync(() => d.update(e))));
-	yield* e.write(o.key, f, {
-		contentType: o.contentType,
-		totalSize: o.size
+	} finally {
+		p !== void 0 && (p.destroy(), await c(p, { cleanup: !0 }).catch(() => void 0)), await a.rm(f, {
+			recursive: !0,
+			force: !0
+		});
+	}
+}, () => new e({
+	reason: "asset capture failed",
+	path: i
+})), g = (e, t, n) => a.scoped(a.gen(function* () {
+	yield* _(e, yield* i({
+		sink: e,
+		assetsDir: t,
+		entries: [n]
+	}), n);
+})), _ = (t, n, i) => a.gen(function* () {
+	yield* r(n, t);
+	let o = yield* a.promise(() => import("node:path")), { createReadStream: s } = yield* a.promise(() => import("node:fs"));
+	if (!n.entries.some((e) => f(e) === f(i) && e.sha256 === i.sha256 && e.size === i.size && e.contentType === i.contentType)) return yield* a.fail(new e({ reason: "asset is outside the admitted index" }));
+	let l = o.join(n.assetsDir, i.sha256), u = c("bundle", () => s(l));
+	yield* t.write(i, u, {
+		contentType: i.contentType,
+		totalSize: i.size
 	});
-	let p = d.digest("hex");
-	if (p !== o.sha256) return yield* n.fail(new t({
-		what: `asset ${o.key}`,
-		expected: o.sha256,
-		actual: p
-	}));
 });
-//#endregion
-export { d as restoreAssetFromCas, l as storageAssetSink, c as storageAssetSource, u as writeAssetToCas };
+export { _ as applyAdmittedAsset, g as restoreAssetFromCas, m as storageAssetSink, p as storageAssetSource, h as writeAssetToCas };

package/dist/blobStorage.js

@@ -1,40 +1,63 @@
-import { Effect as e, Stream as t } from "effect";
-import { streamTable as n } from "@voltro/database";
-import { STORAGE_REFS_TABLE as r, fromNodeReadable as i, getObjectStream as a, putObjectStream as o, toNodeReadable as s } from "@voltro/plugin-storage";
-import { Readable as c } from "node:stream";
-//#region src/blobStorage.ts
-var l = (n, r) => ({
-	list: () => {
-		let n = r();
+import { physicalRequest as e } from "./physicalRequest.js";
+import { storageAssetSink as t } from "./assets.js";
+import { assetBundleContext as n, bundleAssetInfoStream as r, storageAssetInfo as i } from "./storageAssetMetadata.js";
+import { Effect as a, Stream as o } from "effect";
+import { streamTable as s } from "@voltro/database";
+import { STORAGE_REFS_TABLE as c, StorageError as l, fromNodeReadable as u, getObjectStream as d, putObjectStream as f, toNodeReadable as p } from "@voltro/plugin-storage";
+import { Readable as m } from "node:stream";
+import { finished as h } from "node:stream/promises";
+// src/blobStorage.ts
+var g = (e, t) => ({
+	list: (e) => {
+		let n = t(e);
 		return { async *[Symbol.asyncIterator]() {
-			let r = await e.runPromise(t.runCollect(n));
-			for (let e of r) yield {
-				key: e.key,
-				contentType: e.contentType,
-				size: e.size
-			};
+			let e = await a.runPromise(o.runCollect(n));
+			for (let t of e) yield t;
 		} };
 	},
 	open: (t) => ({ async *[Symbol.asyncIterator]() {
-		let r = await e.runPromise(a(n, t)), i = await s(r.stream);
-		for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
+		let n = await a.runPromise(e.resolve(t)), r = await a.runPromise(n.head(t.key));
+		if (r !== null && (r.contentType !== void 0 && r.contentType !== t.contentType || r.size !== t.size)) throw Error("archive blob contradicts source metadata");
+		let i = await a.runPromise(d(n, t.key));
+		if (r?.contentType !== void 0 && i.contentType !== t.contentType || i.size !== void 0 && i.size !== t.size) throw i.close !== void 0 && await a.runPromise(i.close), Error("archive blob contradicts source metadata");
+		let o = await p(i.stream);
+		for await (let e of o) yield e instanceof Uint8Array ? e : new Uint8Array(e);
 	} })
-}), u = (e, i) => l(e, () => n(i, {
-	table: r,
+}), _ = (e, t) => g(e, (e) => e === void 0 ? s(t, {
+	table: c,
 	chunkSize: 500
-}).pipe(t.map((e) => ({
-	key: String(e.key),
-	contentType: String(e.contentType ?? "application/octet-stream"),
-	size: Number(e.size ?? 0)
-})))), d = (t) => ({
-	has: (n) => e.runPromise(t.head(n).pipe(e.map((e) => e !== null))),
-	write: async (n, r, a) => {
-		let s = i("storage", () => c.from(r));
-		await e.runPromise(o(t, n, s, {
-			contentType: a.contentType,
-			totalSize: a.size
-		}));
+}).pipe(o.map(i)) : o.unwrap(n(e).pipe(a.map(r)))), v = (e) => ({
+	has: (n, r) => a.runPromise(t(e).has(n), r),
+	matches: (n, r) => a.runPromise(t(e).matches(n), r),
+	write: async (t, n, r, i) => {
+		let o = u("storage", () => m.from(n));
+		await a.runPromise(e.resolve(t).pipe(a.flatMap((e) => f(e, t.key, o, {
+			contentType: r.contentType,
+			totalSize: r.size
+		}))), i);
 	}
+}), y = (t, n) => e(t, () => new l({
+	provider: "archive",
+	message: n,
+	transient: !1
+})), b = (e) => ({
+	has: (t) => y((n) => e.has(t, { signal: n }), "asset destination admission failed"),
+	matches: (t) => y((n) => e.matches(t, { signal: n }), "asset destination content verification failed"),
+	write: (t, n, r) => y(async (i) => {
+		if (r.totalSize === void 0) throw Error("admitted asset size is required");
+		let a = await p(n), o = () => {
+			a.destroy();
+		};
+		i.addEventListener("abort", o, { once: !0 });
+		try {
+			if (i.aborted) throw o(), Error("asset write interrupted");
+			await e.write(t, a, {
+				contentType: r.contentType,
+				size: r.totalSize
+			}, { signal: i });
+		} finally {
+			i.removeEventListener("abort", o), a.destroy(), await h(a, { cleanup: !0 }).catch(() => void 0);
+		}
+	}, "admitted asset write failed")
 });
-//#endregion
-export { d as storageBlobSink, l as storageBlobSource, u as storageRefsBlobSource };
+export { b as blobAssetSink, v as storageBlobSink, g as storageBlobSource, _ as storageRefsBlobSource };