@voltro/plugin-storage 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -0
- package/README.md +50 -0
- package/THIRD-PARTY-NOTICES.md +5 -33
- package/dist/access.js +47 -0
- package/dist/assetRef.js +13 -0
- package/dist/avatar.js +39 -0
- package/dist/constraints.js +93 -0
- package/dist/derivatives.js +30 -0
- package/dist/env.js +156 -0
- package/dist/grant.js +112 -0
- package/dist/index.d.ts +93 -9
- package/dist/index.js +581 -3185
- package/dist/ingest.js +81 -0
- package/dist/ingestPolicy.js +1 -0
- package/dist/metadataError.js +19 -0
- package/dist/objectLifecycle.js +209 -0
- package/dist/objectMetadata.js +150 -0
- package/dist/objectWriter.js +58 -0
- package/dist/pathDurability.js +16 -0
- package/dist/providers.js +493 -0
- package/dist/referenceCatalogue.js +31 -0
- package/dist/rpc.d.ts +2 -0
- package/dist/rpc.js +3 -1
- package/dist/scanners.js +46 -0
- package/dist/service.js +1084 -0
- package/dist/storageNamespaces.js +56 -0
- package/dist/storageNumbers.js +13 -0
- package/dist/storageObjectNamespaces.js +194 -0
- package/dist/storageReferenceInstances.js +52 -0
- package/dist/streamBridge.js +30 -0
- package/dist/table.js +53 -0
- package/dist/transcodeVideo.js +53 -0
- package/dist/transform.js +90 -0
- package/dist/transformCache.js +31 -0
- package/dist/types.d.ts +4 -0
- package/package.json +5 -5
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { createHash as e, randomUUID as t } from "node:crypto";
|
|
2
|
+
import { authoredError as n } from "@voltro/database";
|
|
3
|
+
import { linkSync as r, mkdirSync as i, readFileSync as a, realpathSync as o, rmSync as s, writeFileSync as c } from "node:fs";
|
|
4
|
+
import { basename as l, dirname as u, join as d, resolve as f } from "node:path";
|
|
5
|
+
//#region src/storageNamespaces.ts
|
|
6
|
+
var p = (t) => e("sha256").update(JSON.stringify(t)).digest("hex"), m = (e) => {
|
|
7
|
+
if (!e?.storageInstance?.trim() || !/^[a-f0-9]{64}$/.test(e.objectNamespace)) throw n("Storage requires a configured instance and a stable physical objectNamespace SHA-256 identity");
|
|
8
|
+
}, h = (e) => {
|
|
9
|
+
let t = f(e), n = [];
|
|
10
|
+
for (;;) try {
|
|
11
|
+
return d(o(t), ...n.reverse());
|
|
12
|
+
} catch (e) {
|
|
13
|
+
if (!(e instanceof Error && "code" in e && e.code === "ENOENT")) throw e;
|
|
14
|
+
let r = u(t);
|
|
15
|
+
if (r === t) throw e;
|
|
16
|
+
n.push(l(t)), t = r;
|
|
17
|
+
}
|
|
18
|
+
}, g = (e) => {
|
|
19
|
+
let t = new URL(e);
|
|
20
|
+
return t.username = "", t.password = "", t.search = "", t.hash = "", t.href.replace(/\/+$/, "");
|
|
21
|
+
}, _ = ".voltro-object-namespace", v = (e) => {
|
|
22
|
+
i(e, { recursive: !0 });
|
|
23
|
+
let o = d(e, _), l;
|
|
24
|
+
try {
|
|
25
|
+
l = a(o, "utf8");
|
|
26
|
+
} catch (e) {
|
|
27
|
+
if (!(e instanceof Error && "code" in e && e.code === "ENOENT")) throw e;
|
|
28
|
+
let n = p(["filesystem-volume", t()]), i = `${o}-${t()}`;
|
|
29
|
+
try {
|
|
30
|
+
c(i, n, {
|
|
31
|
+
flag: "wx",
|
|
32
|
+
mode: 384
|
|
33
|
+
});
|
|
34
|
+
try {
|
|
35
|
+
r(i, o);
|
|
36
|
+
} catch (e) {
|
|
37
|
+
if (!(e instanceof Error && "code" in e && e.code === "EEXIST")) throw e;
|
|
38
|
+
}
|
|
39
|
+
l = a(o, "utf8");
|
|
40
|
+
} finally {
|
|
41
|
+
s(i, { force: !0 });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (!/^[a-f0-9]{64}$/.test(l)) throw n("Filesystem storage object namespace marker is invalid; restore the volume marker before serving storage");
|
|
45
|
+
return l;
|
|
46
|
+
}, y = (e) => {
|
|
47
|
+
if (m(e), typeof e.catalogueId != "string" || !e.catalogueId.trim()) throw n("Storage upload tickets require an actual reference catalogue identity");
|
|
48
|
+
return p([
|
|
49
|
+
"upload-ticket-destination",
|
|
50
|
+
e.storageInstance,
|
|
51
|
+
e.catalogueId,
|
|
52
|
+
e.objectNamespace
|
|
53
|
+
]);
|
|
54
|
+
};
|
|
55
|
+
//#endregion
|
|
56
|
+
export { _ as FILESYSTEM_NAMESPACE_FILE, h as canonicalStorageRoot, v as filesystemObjectNamespace, m as requireStorageReferenceScope, g as storageEndpointIdentity, p as storageObjectNamespace, y as storageUploadTicketScope };
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { StorageError as e } from "./types.js";
|
|
2
|
+
//#region src/storageNumbers.ts
|
|
3
|
+
var t = (t, n) => {
|
|
4
|
+
let r = typeof t == "number" ? t : typeof t == "bigint" || typeof t == "string" && /^\d+$/.test(t) ? Number(t) : NaN;
|
|
5
|
+
if (!Number.isSafeInteger(r) || r < 0) throw new e({
|
|
6
|
+
provider: "metadata",
|
|
7
|
+
message: `Storage ${n} must be a non-negative safe integer at most Number.MAX_SAFE_INTEGER`,
|
|
8
|
+
transient: !1
|
|
9
|
+
});
|
|
10
|
+
return r;
|
|
11
|
+
}, n = (e, n, r) => t(e + n, r);
|
|
12
|
+
//#endregion
|
|
13
|
+
export { t as storageInteger, n as storageIntegerSum };
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { requireStorageReferenceScope as e } from "./storageNamespaces.js";
|
|
2
|
+
import { STORAGE_GRANTS_TABLE as t, STORAGE_REFS_TABLE as n } from "./table.js";
|
|
3
|
+
import { storageInteger as r, storageIntegerSum as i } from "./storageNumbers.js";
|
|
4
|
+
import { storageObjectClaimId as a, storageObjectIdentifiers as o } from "./objectLifecycle.js";
|
|
5
|
+
import { createHash as s } from "node:crypto";
|
|
6
|
+
import { authoredError as c, isNull as l, or as u } from "@voltro/database";
|
|
7
|
+
import { Schema as d } from "effect";
|
|
8
|
+
//#region src/storageObjectNamespaces.ts
|
|
9
|
+
var f = "_voltro_storage_put_intents", p = "_voltro_storage_operations", m = "_voltro_storage_object_heads", h = "_voltro_storage_object_claims", g = [
|
|
10
|
+
n,
|
|
11
|
+
t,
|
|
12
|
+
f,
|
|
13
|
+
p
|
|
14
|
+
], _ = d.Struct({
|
|
15
|
+
tenantId: d.NullOr(d.String),
|
|
16
|
+
bucket: d.String,
|
|
17
|
+
operationId: d.String,
|
|
18
|
+
fingerprint: d.String,
|
|
19
|
+
checksum: d.String,
|
|
20
|
+
logicalKey: d.String
|
|
21
|
+
}), v = d.Struct({
|
|
22
|
+
id: d.String,
|
|
23
|
+
objectId: d.String,
|
|
24
|
+
fingerprint: d.String,
|
|
25
|
+
generation: d.String,
|
|
26
|
+
key: d.String,
|
|
27
|
+
refId: d.String,
|
|
28
|
+
state: d.Literals([
|
|
29
|
+
"reserved",
|
|
30
|
+
"ready",
|
|
31
|
+
"deleting",
|
|
32
|
+
"deleted"
|
|
33
|
+
]),
|
|
34
|
+
deleteKey: d.NullOr(d.String)
|
|
35
|
+
}), y = (e) => s("sha256").update(JSON.stringify(e)).digest("hex"), b = (e) => JSON.stringify([e.storageInstance, e.objectNamespace]), x = (e, t) => JSON.stringify([e.objectNamespace, t]), S = (e, t) => e.query({
|
|
36
|
+
table: t,
|
|
37
|
+
predicate: void 0,
|
|
38
|
+
lock: "update",
|
|
39
|
+
order: [],
|
|
40
|
+
take: void 0,
|
|
41
|
+
skip: void 0,
|
|
42
|
+
projection: void 0
|
|
43
|
+
}), C = async (e) => {
|
|
44
|
+
for (let t of [...g, m]) if ((await e.query({
|
|
45
|
+
table: t,
|
|
46
|
+
predicate: t === m ? l("objectNamespace") : u(l("storageInstance"), l("objectNamespace")),
|
|
47
|
+
order: [],
|
|
48
|
+
take: 1,
|
|
49
|
+
skip: void 0,
|
|
50
|
+
projection: ["id"]
|
|
51
|
+
})).length) throw c("Storage physical namespace migration required: assign references and retained operations to their configured storageInstance and provider objectNamespace before starting the app");
|
|
52
|
+
}, w = async (s, l, u) => {
|
|
53
|
+
for (let t of l) e(t);
|
|
54
|
+
if (!l.length || new Set(l.map(b)).size !== l.length) throw c("Provide distinct configured storage instance and physical namespace pairs");
|
|
55
|
+
let g = new Set(l.map(b)), w = /* @__PURE__ */ new Map();
|
|
56
|
+
for (let { refId: t, ...n } of u) {
|
|
57
|
+
if (e(n), !t.trim() || !g.has(b(n))) throw c("Storage namespace assignment must name a reference and a configured physical scope");
|
|
58
|
+
let r = w.get(t);
|
|
59
|
+
if (r && b(r) !== b(n)) throw c("Storage physical namespace assignment is ambiguous");
|
|
60
|
+
w.set(t, n);
|
|
61
|
+
}
|
|
62
|
+
let T = [];
|
|
63
|
+
for (let e of [
|
|
64
|
+
n,
|
|
65
|
+
t,
|
|
66
|
+
f,
|
|
67
|
+
p,
|
|
68
|
+
m,
|
|
69
|
+
h
|
|
70
|
+
]) T.push(await S(s, e));
|
|
71
|
+
let [E, D, O, k, A, j] = T, M = /* @__PURE__ */ new Set(), N = (e, t) => {
|
|
72
|
+
let n = w.get(e);
|
|
73
|
+
if (!n) throw c("Storage physical namespace assignment is incomplete");
|
|
74
|
+
for (let e of ["storageInstance", "objectNamespace"]) if (t[e] != null && t[e] !== n[e]) throw c("Storage metadata already belongs to another physical scope; assignment does not move objects");
|
|
75
|
+
return M.add(e), n;
|
|
76
|
+
};
|
|
77
|
+
for (let e of E) N(String(e.id), e);
|
|
78
|
+
for (let e of D) if (N(String(e.refId), e), !E.some((t) => t.id === e.refId)) throw c("Storage grant has no readable reference");
|
|
79
|
+
let P = /* @__PURE__ */ new Map();
|
|
80
|
+
for (let e of O) {
|
|
81
|
+
if (N(String(e.refId), e), P.has(String(e.id))) throw c("Storage put intent ownership is ambiguous");
|
|
82
|
+
P.set(String(e.id), e);
|
|
83
|
+
}
|
|
84
|
+
let F = /* @__PURE__ */ new Map(), I = k.map((e) => {
|
|
85
|
+
let t = d.decodeUnknownSync(v)(e), n = N(t.refId, e), a = P.get(t.id);
|
|
86
|
+
if (!a || a.refId !== t.refId) throw c("Storage operation lost its retained put intent");
|
|
87
|
+
let s = {
|
|
88
|
+
...d.decodeUnknownSync(_)(a.identity),
|
|
89
|
+
...n
|
|
90
|
+
};
|
|
91
|
+
typeof a.identity == "object" && a.identity !== null && N(t.refId, a.identity);
|
|
92
|
+
let l = o(s), u = y([
|
|
93
|
+
s.bucket,
|
|
94
|
+
s.tenantId,
|
|
95
|
+
s.operationId
|
|
96
|
+
]), f = y([
|
|
97
|
+
s.bucket,
|
|
98
|
+
s.tenantId,
|
|
99
|
+
s.logicalKey,
|
|
100
|
+
s.checksum
|
|
101
|
+
]);
|
|
102
|
+
if (![u, l.id].includes(t.id) || ![f, l.objectId].includes(t.objectId) || t.fingerprint !== s.fingerprint) throw c("Storage operation identity does not match its retained intent");
|
|
103
|
+
if (typeof a.ref != "object" || a.ref === null || Array.isArray(a.ref)) throw c("Storage put intent reference is unreadable");
|
|
104
|
+
let p = a.ref;
|
|
105
|
+
if (p.id !== t.refId || p.key !== t.key || p.checksum !== s.checksum || p.tenantId !== s.tenantId || p.bucket !== s.bucket) throw c("Storage operation reference does not match its retained intent");
|
|
106
|
+
let m = E.find((e) => e.id === t.refId);
|
|
107
|
+
if (t.state === "ready") {
|
|
108
|
+
if (!m || m.key !== t.key || m.checksum !== s.checksum) throw c("Ready storage operation lost its readable reference");
|
|
109
|
+
} else if (m) throw c("Unreadable storage operation still owns a readable reference");
|
|
110
|
+
if (t.state === "ready" || t.state === "reserved") {
|
|
111
|
+
let e = F.get(l.objectId);
|
|
112
|
+
if (e && (e.generation !== t.generation || e.key !== t.key)) throw c("Storage physical namespace has conflicting active generations");
|
|
113
|
+
F.set(l.objectId, {
|
|
114
|
+
id: l.objectId,
|
|
115
|
+
objectNamespace: n.objectNamespace,
|
|
116
|
+
generation: t.generation,
|
|
117
|
+
key: t.key,
|
|
118
|
+
reservations: String(i(e ? r(e.reservations, "shared-object reservation count") : 0, 1, "shared-object reservation count"))
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
return {
|
|
122
|
+
previous: t,
|
|
123
|
+
row: {
|
|
124
|
+
...t,
|
|
125
|
+
...n,
|
|
126
|
+
id: l.id,
|
|
127
|
+
objectId: l.objectId
|
|
128
|
+
},
|
|
129
|
+
intent: a,
|
|
130
|
+
identity: s
|
|
131
|
+
};
|
|
132
|
+
});
|
|
133
|
+
if (I.length !== O.length || new Set(I.map((e) => e.row.id)).size !== I.length) throw c("Storage operation namespace migration has unmatched or conflicting identities");
|
|
134
|
+
for (let e of A) {
|
|
135
|
+
let t = I.filter(({ previous: t }) => t.objectId === e.id && (t.state === "ready" || t.state === "reserved"));
|
|
136
|
+
if (!t.length || t.length !== r(e.reservations, "shared-object reservation count") || t.some(({ previous: t, identity: n }) => t.generation !== e.generation || t.key !== e.key || e.objectNamespace != null && e.objectNamespace !== n.objectNamespace)) throw c("Storage object head does not match its active reservations");
|
|
137
|
+
}
|
|
138
|
+
for (let { previous: e } of I) if ((e.state === "ready" || e.state === "reserved") && !A.some((t) => t.id === e.objectId)) throw c("Storage operation lost its active object head");
|
|
139
|
+
if ([...w.keys()].some((e) => !M.has(e))) throw c("Storage namespace assignment target does not exist");
|
|
140
|
+
let L = new Map(I.map((e) => [e.previous.id, e])), R = j.map((e) => {
|
|
141
|
+
let t = L.get(String(e.operationId));
|
|
142
|
+
if (!t || typeof e.claim != "string" || !e.claim || e.id !== a(String(e.operationId), e.claim)) throw c("Storage operation claim does not match its retained operation");
|
|
143
|
+
return {
|
|
144
|
+
...e,
|
|
145
|
+
id: a(t.row.id, e.claim),
|
|
146
|
+
operationId: t.row.id
|
|
147
|
+
};
|
|
148
|
+
}), z = /* @__PURE__ */ new Set([...E.map((e) => x(w.get(String(e.id)), String(e.key))), ...[...F.values()].map((e) => JSON.stringify([e.objectNamespace, e.key]))]), B = [];
|
|
149
|
+
for (let e of I) {
|
|
150
|
+
let t = z.has(x(e.identity, e.row.key));
|
|
151
|
+
(e.row.state === "deleting" || e.row.state === "deleted") && (t ? e.row.deleteKey = null : e.row.state === "deleted" && e.row.deleteKey === null && (e.row.state = "deleting", e.row.deleteKey = e.row.key), e.row.state === "deleting" && e.row.deleteKey !== null && B.push({
|
|
152
|
+
refId: e.row.refId,
|
|
153
|
+
storageInstance: e.identity.storageInstance,
|
|
154
|
+
objectNamespace: e.identity.objectNamespace
|
|
155
|
+
}));
|
|
156
|
+
}
|
|
157
|
+
let V = 0;
|
|
158
|
+
for (let [e, r, i] of [[
|
|
159
|
+
n,
|
|
160
|
+
E,
|
|
161
|
+
"id"
|
|
162
|
+
], [
|
|
163
|
+
t,
|
|
164
|
+
D,
|
|
165
|
+
"refId"
|
|
166
|
+
]]) for (let t of r) {
|
|
167
|
+
let n = w.get(String(t[i]));
|
|
168
|
+
if (t.storageInstance !== n.storageInstance || t.objectNamespace !== n.objectNamespace) {
|
|
169
|
+
if (await s.update(e, String(t.id), { ...n }) === null) throw c("Storage physical namespace assignment lost its metadata row");
|
|
170
|
+
V++;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
for (let [e, t] of [
|
|
174
|
+
[h, j],
|
|
175
|
+
[f, O],
|
|
176
|
+
[p, k],
|
|
177
|
+
[m, A]
|
|
178
|
+
]) for (let n of t) await s.delete(e, String(n.id));
|
|
179
|
+
for (let e of F.values()) await s.insert(m, e);
|
|
180
|
+
for (let e of I) await s.insert(p, e.row), await s.insert(f, {
|
|
181
|
+
...e.intent,
|
|
182
|
+
id: e.row.id,
|
|
183
|
+
storageInstance: e.identity.storageInstance,
|
|
184
|
+
objectNamespace: e.identity.objectNamespace,
|
|
185
|
+
identity: e.identity
|
|
186
|
+
}), (e.previous.id !== e.row.id || e.intent.objectNamespace !== e.identity.objectNamespace || e.previous.state !== e.row.state || e.previous.deleteKey !== e.row.deleteKey) && V++;
|
|
187
|
+
for (let e of R) await s.insert(h, e);
|
|
188
|
+
return await C(s), {
|
|
189
|
+
assigned: V,
|
|
190
|
+
pendingDeletes: B
|
|
191
|
+
};
|
|
192
|
+
};
|
|
193
|
+
//#endregion
|
|
194
|
+
export { w as assignStorageObjectNamespacesInTransaction, C as requireStorageObjectNamespaces };
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { STORAGE_GRANTS_TABLE as e, STORAGE_REFS_TABLE as t } from "./table.js";
|
|
2
|
+
import { authoredError as n, eq as r, isNull as i } from "@voltro/database";
|
|
3
|
+
//#region src/storageReferenceInstances.ts
|
|
4
|
+
var a = "@voltro/plugin-storage", o = [
|
|
5
|
+
t,
|
|
6
|
+
e,
|
|
7
|
+
"_voltro_storage_put_intents"
|
|
8
|
+
], s = async (e) => {
|
|
9
|
+
for (let t of o) if ((await e.query({
|
|
10
|
+
table: t,
|
|
11
|
+
predicate: i("storageInstance"),
|
|
12
|
+
order: [],
|
|
13
|
+
take: 1,
|
|
14
|
+
skip: void 0,
|
|
15
|
+
projection: ["id"]
|
|
16
|
+
})).length) throw n("Storage reference ownership migration required: assign existing references, grants and put intents to their configured storageInstance before starting the app");
|
|
17
|
+
}, c = async (e, t, i) => {
|
|
18
|
+
if (t.length === 0 || t.some((e) => !e.trim()) || new Set(t).size !== t.length) throw n("Provide distinct configured storage instance names");
|
|
19
|
+
let a = new Set(t), s = /* @__PURE__ */ new Map();
|
|
20
|
+
for (let e of i) {
|
|
21
|
+
if (!e.refId.trim() || !a.has(e.storageInstance)) throw n("Storage reference assignment must name a reference and a configured instance");
|
|
22
|
+
let t = s.get(e.refId);
|
|
23
|
+
if (t !== void 0 && t !== e.storageInstance) throw n("Storage reference assignment is ambiguous");
|
|
24
|
+
s.set(e.refId, e.storageInstance);
|
|
25
|
+
}
|
|
26
|
+
let c = e, l = 0;
|
|
27
|
+
for (let [e, t] of s) {
|
|
28
|
+
let i = !1;
|
|
29
|
+
for (let a of o) {
|
|
30
|
+
let o = await c.query({
|
|
31
|
+
table: a,
|
|
32
|
+
predicate: r(a === "_voltro_storage_refs" ? "id" : "refId", e),
|
|
33
|
+
lock: "update",
|
|
34
|
+
order: [],
|
|
35
|
+
take: void 0,
|
|
36
|
+
skip: void 0,
|
|
37
|
+
projection: void 0
|
|
38
|
+
});
|
|
39
|
+
for (let e of o) {
|
|
40
|
+
if (i = !0, e.storageInstance !== null && e.storageInstance !== void 0 && e.storageInstance !== t) throw n("Storage reference already belongs to another instance; assignment does not move objects");
|
|
41
|
+
if (e.storageInstance !== t) {
|
|
42
|
+
if (await c.update(a, String(e.id), { storageInstance: t }) === null) throw n("Storage reference assignment lost its metadata row");
|
|
43
|
+
l++;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (!i) throw n("Storage reference assignment target does not exist");
|
|
48
|
+
}
|
|
49
|
+
return { assigned: l };
|
|
50
|
+
};
|
|
51
|
+
//#endregion
|
|
52
|
+
export { a as DEFAULT_STORAGE_INSTANCE, c as assignStorageReferenceInstancesInTransaction, s as requireStorageReferenceOwnership };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { StorageError as e } from "./types.js";
|
|
2
|
+
import { Effect as t, Stream as n } from "effect";
|
|
3
|
+
//#region src/streamBridge.ts
|
|
4
|
+
var r = 1 << 20, i = (t, n, r = !1) => new e({
|
|
5
|
+
provider: t,
|
|
6
|
+
message: n,
|
|
7
|
+
transient: r
|
|
8
|
+
}), a = (e) => {
|
|
9
|
+
let t = 0;
|
|
10
|
+
for (let n of e) t += n.byteLength;
|
|
11
|
+
let n = new Uint8Array(t), r = 0;
|
|
12
|
+
for (let t of e) n.set(t, r), r += t.byteLength;
|
|
13
|
+
return n;
|
|
14
|
+
}, o = (e, t = r) => {
|
|
15
|
+
let i = [];
|
|
16
|
+
for (let n = 0; n < e.byteLength; n += t) i.push(e.subarray(n, Math.min(n + t, e.byteLength)));
|
|
17
|
+
return n.fromIterable(i);
|
|
18
|
+
}, s = (e) => e.pipe(n.runCollect, t.map((e) => a(e))), c = (e, t) => n.suspend(() => n.fromAsyncIterable(t(), (t) => i(e, `read stream failed: ${String(t?.message ?? t)}`, !0))).pipe(n.map((e) => e instanceof Uint8Array ? e : new Uint8Array(e))), l = async (e) => {
|
|
19
|
+
let { Readable: t } = await import("node:stream");
|
|
20
|
+
return t.fromWeb(n.toReadableStream(e));
|
|
21
|
+
}, u = (e, n) => e.getStream ? e.getStream(n) : e.get(n).pipe(t.map((e) => ({
|
|
22
|
+
stream: o(e.bytes),
|
|
23
|
+
contentType: e.contentType,
|
|
24
|
+
size: e.bytes.byteLength
|
|
25
|
+
}))), d = (e, n, r, i) => e.putStream ? e.putStream(n, r, i) : s(r).pipe(t.flatMap((t) => e.put(n, t, {
|
|
26
|
+
contentType: i.contentType,
|
|
27
|
+
...i.public === void 0 ? {} : { public: i.public }
|
|
28
|
+
})));
|
|
29
|
+
//#endregion
|
|
30
|
+
export { o as bytesToStream, s as collectStream, a as concatBytes, c as fromNodeReadable, u as getObjectStream, d as putObjectStream, l as toNodeReadable };
|
package/dist/table.js
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { bigint as e, bytes as t, id as n, integer as r, json as i, table as a, text as o, timestamp as s } from "@voltro/database";
|
|
2
|
+
//#region src/table.ts
|
|
3
|
+
var c = "_voltro_storage_refs", l = "_voltro_storage_grants", u = "_voltro_storage_blobs", d = "_voltro_storage_usage", f = a(c, {
|
|
4
|
+
id: n({ prefix: "file" }),
|
|
5
|
+
storageInstance: o().nullable(),
|
|
6
|
+
objectNamespace: o().maxLength(64).nullable(),
|
|
7
|
+
tenantId: o().nullable(),
|
|
8
|
+
ownerId: o().nullable(),
|
|
9
|
+
bucket: o(),
|
|
10
|
+
key: o(),
|
|
11
|
+
contentType: o(),
|
|
12
|
+
size: e(),
|
|
13
|
+
checksum: o(),
|
|
14
|
+
visibility: o(),
|
|
15
|
+
accessPolicy: i().nullable(),
|
|
16
|
+
passwordHash: o().nullable(),
|
|
17
|
+
width: r().nullable(),
|
|
18
|
+
height: r().nullable(),
|
|
19
|
+
duration: r().nullable(),
|
|
20
|
+
placeholder: o().nullable(),
|
|
21
|
+
folder: o().nullable(),
|
|
22
|
+
tags: i().nullable(),
|
|
23
|
+
alt: o().nullable(),
|
|
24
|
+
caption: o().nullable(),
|
|
25
|
+
derivedFrom: o().nullable(),
|
|
26
|
+
kind: o().nullable(),
|
|
27
|
+
createdAt: s()
|
|
28
|
+
}).index(["storageInstance"]).index(["objectNamespace"]).index(["key"]).index(["tenantId"]).index(["ownerId"]).index(["folder"]).index(["derivedFrom"]), p = a(l, {
|
|
29
|
+
id: n({ prefix: "sgrant" }),
|
|
30
|
+
storageInstance: o().nullable(),
|
|
31
|
+
objectNamespace: o().maxLength(64).nullable(),
|
|
32
|
+
refId: o(),
|
|
33
|
+
principalType: o(),
|
|
34
|
+
principalId: o(),
|
|
35
|
+
permission: o(),
|
|
36
|
+
createdAt: s(),
|
|
37
|
+
expiresAt: s().nullable(),
|
|
38
|
+
createdBy: o().nullable()
|
|
39
|
+
}).index(["storageInstance"]).index(["objectNamespace"]).index(["refId"]).index(["principalId"]), m = a(d, {
|
|
40
|
+
id: n({ prefix: "susage" }),
|
|
41
|
+
tenantId: o().maxLength(191),
|
|
42
|
+
bytes: e().default("0"),
|
|
43
|
+
count: e().default("0"),
|
|
44
|
+
updatedAt: s()
|
|
45
|
+
}).unique("byStorageUsageTenant", ["tenantId"]), h = a(u, {
|
|
46
|
+
id: n({ prefix: "blob" }),
|
|
47
|
+
key: o(),
|
|
48
|
+
data: t(),
|
|
49
|
+
contentType: o(),
|
|
50
|
+
size: e()
|
|
51
|
+
}).index(["key"]);
|
|
52
|
+
//#endregion
|
|
53
|
+
export { u as STORAGE_BLOBS_TABLE, l as STORAGE_GRANTS_TABLE, c as STORAGE_REFS_TABLE, d as STORAGE_USAGE_TABLE, h as _voltroStorageBlobsTable, p as _voltroStorageGrantsTable, f as _voltroStorageRefsTable, m as _voltroStorageUsageTable };
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { join as e } from "node:path";
|
|
2
|
+
import { spawn as t } from "node:child_process";
|
|
3
|
+
import { mkdtemp as n, readFile as r, rm as i, writeFile as a } from "node:fs/promises";
|
|
4
|
+
import { tmpdir as o } from "node:os";
|
|
5
|
+
//#region src/transcodeVideo.ts
|
|
6
|
+
var s = (e, n, r) => new Promise((i, a) => {
|
|
7
|
+
let o = t(e, [...n], { stdio: [
|
|
8
|
+
"ignore",
|
|
9
|
+
"ignore",
|
|
10
|
+
"pipe"
|
|
11
|
+
] }), s = "", c = setTimeout(() => {
|
|
12
|
+
o.kill("SIGKILL"), a(/* @__PURE__ */ Error("ffmpeg timed out"));
|
|
13
|
+
}, r);
|
|
14
|
+
o.stderr?.on("data", (e) => {
|
|
15
|
+
s += String(e);
|
|
16
|
+
}), o.on("error", (e) => {
|
|
17
|
+
clearTimeout(c), a(e);
|
|
18
|
+
}), o.on("close", (e) => {
|
|
19
|
+
clearTimeout(c), e === 0 ? i() : a(/* @__PURE__ */ Error(`ffmpeg exited ${String(e)}: ${s.slice(-400)}`));
|
|
20
|
+
});
|
|
21
|
+
}), c = (t) => async ({ bytes: c }) => {
|
|
22
|
+
let l = t.ffmpegPath ?? "ffmpeg", u = t.timeoutMs ?? 3e5, d = await n(e(o(), "voltro-tc-")), f = e(d, "in"), p = [];
|
|
23
|
+
try {
|
|
24
|
+
await a(f, c);
|
|
25
|
+
for (let n of t.renditions) {
|
|
26
|
+
let t = e(d, `out.${n.ext}`);
|
|
27
|
+
try {
|
|
28
|
+
await s(l, [
|
|
29
|
+
"-y",
|
|
30
|
+
"-i",
|
|
31
|
+
f,
|
|
32
|
+
...n.args,
|
|
33
|
+
t
|
|
34
|
+
], u);
|
|
35
|
+
let e = await r(t);
|
|
36
|
+
p.push({
|
|
37
|
+
bytes: new Uint8Array(e),
|
|
38
|
+
contentType: n.contentType,
|
|
39
|
+
kind: n.kind,
|
|
40
|
+
...n.label === void 0 ? {} : { label: n.label }
|
|
41
|
+
}), await i(t, { force: !0 });
|
|
42
|
+
} catch {}
|
|
43
|
+
}
|
|
44
|
+
} finally {
|
|
45
|
+
await i(d, {
|
|
46
|
+
recursive: !0,
|
|
47
|
+
force: !0
|
|
48
|
+
}).catch(() => void 0);
|
|
49
|
+
}
|
|
50
|
+
return p;
|
|
51
|
+
};
|
|
52
|
+
//#endregion
|
|
53
|
+
export { c as ffmpegTranscoder };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { createTransformCache as e } from "./transformCache.js";
|
|
2
|
+
//#region src/transform.ts
|
|
3
|
+
var t = (e, t, n, r, i, a) => {
|
|
4
|
+
let o = n / r, s = Math.min(e, Math.round(t * o)), c = Math.min(t, Math.round(s / o)), l = (e, t) => Math.max(0, Math.min(t, e));
|
|
5
|
+
return {
|
|
6
|
+
left: l(Math.round(i * e - s / 2), e - s),
|
|
7
|
+
top: l(Math.round(a * t - c / 2), t - c),
|
|
8
|
+
width: s,
|
|
9
|
+
height: c
|
|
10
|
+
};
|
|
11
|
+
}, n = {
|
|
12
|
+
webp: "image/webp",
|
|
13
|
+
avif: "image/avif",
|
|
14
|
+
jpeg: "image/jpeg",
|
|
15
|
+
jpg: "image/jpeg",
|
|
16
|
+
png: "image/png"
|
|
17
|
+
}, r = (e) => {
|
|
18
|
+
if (!e) return null;
|
|
19
|
+
let t = new URLSearchParams(e), n = (e) => {
|
|
20
|
+
let n = t.get(e);
|
|
21
|
+
if (n == null) return;
|
|
22
|
+
let r = Number(n);
|
|
23
|
+
return Number.isFinite(r) && r > 0 ? Math.floor(r) : void 0;
|
|
24
|
+
}, r = n("w"), i = n("h"), a = n("q"), o = (t.get("format") ?? t.get("fm") ?? void 0)?.toLowerCase(), s = t.get("fit") ?? void 0, c = (t.get("fp") ?? "").split(",").map(Number), l = c.length === 2 && c.every((e) => Number.isFinite(e) && e >= 0 && e <= 1) ? {
|
|
25
|
+
x: c[0],
|
|
26
|
+
y: c[1]
|
|
27
|
+
} : void 0;
|
|
28
|
+
return r === void 0 && i === void 0 && !o && a === void 0 ? null : {
|
|
29
|
+
...r === void 0 ? {} : { w: r },
|
|
30
|
+
...i === void 0 ? {} : { h: i },
|
|
31
|
+
...o ? { format: o } : {},
|
|
32
|
+
...a === void 0 ? {} : { q: a },
|
|
33
|
+
...s ? { fit: s } : {},
|
|
34
|
+
...l ? { fp: l } : {}
|
|
35
|
+
};
|
|
36
|
+
}, i = (e) => e.startsWith("image/") && !e.includes("svg"), a, o = async () => (a ??= import("sharp").then((e) => e.default ?? e, () => null), a), s = async (e, t) => t ? null : i(e) ? await o() === null ? "no-sharp" : "failed" : "not-an-image", c = async (e, r, a) => {
|
|
37
|
+
if (!i(r)) return null;
|
|
38
|
+
let s = await o();
|
|
39
|
+
if (!s) return null;
|
|
40
|
+
try {
|
|
41
|
+
let i = s(Buffer.from(e), { failOn: "none" }), o = a.fit ?? "cover";
|
|
42
|
+
if (a.fp !== void 0 && a.w !== void 0 && a.h !== void 0 && o === "cover") {
|
|
43
|
+
let e = await i.metadata();
|
|
44
|
+
e.width !== void 0 && e.height !== void 0 && (i = i.extract(t(e.width, e.height, a.w, a.h, a.fp.x, a.fp.y)));
|
|
45
|
+
}
|
|
46
|
+
(a.w !== void 0 || a.h !== void 0) && (i = i.resize({
|
|
47
|
+
...a.w === void 0 ? {} : { width: a.w },
|
|
48
|
+
...a.h === void 0 ? {} : { height: a.h },
|
|
49
|
+
fit: a.fit ?? "cover",
|
|
50
|
+
withoutEnlargement: !0
|
|
51
|
+
}));
|
|
52
|
+
let c = a.format && n[a.format] ? a.format : void 0, l = r;
|
|
53
|
+
if (c) {
|
|
54
|
+
let e = c === "jpg" ? "jpeg" : c;
|
|
55
|
+
i = i.toFormat(e, a.q === void 0 ? {} : { quality: a.q }), l = n[c];
|
|
56
|
+
} else a.q !== void 0 && (r === "image/jpeg" || r === "image/webp") && (i = i.toFormat(r === "image/webp" ? "webp" : "jpeg", { quality: a.q }));
|
|
57
|
+
let u = await i.toBuffer();
|
|
58
|
+
return {
|
|
59
|
+
bytes: new Uint8Array(u),
|
|
60
|
+
contentType: l
|
|
61
|
+
};
|
|
62
|
+
} catch {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
}, l = async (e, t, n) => {
|
|
66
|
+
if (!i(t) || n.rotate === void 0 && n.flip !== !0 && n.flop !== !0 && n.grayscale !== !0 && n.crop === void 0) return null;
|
|
67
|
+
let r = await o();
|
|
68
|
+
if (!r) return null;
|
|
69
|
+
try {
|
|
70
|
+
let i = r(Buffer.from(e), { failOn: "none" });
|
|
71
|
+
return n.crop !== void 0 && (i = i.extract(n.crop)), n.rotate !== void 0 && (i = i.rotate(n.rotate)), n.flip === !0 && (i = i.flip()), n.flop === !0 && (i = i.flop()), n.grayscale === !0 && (i = i.grayscale()), {
|
|
72
|
+
bytes: new Uint8Array(await i.toBuffer()),
|
|
73
|
+
contentType: t
|
|
74
|
+
};
|
|
75
|
+
} catch {
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
}, u = e(), d = (e, t, n) => JSON.stringify([
|
|
79
|
+
e,
|
|
80
|
+
t,
|
|
81
|
+
n.w,
|
|
82
|
+
n.h,
|
|
83
|
+
n.format,
|
|
84
|
+
n.q,
|
|
85
|
+
n.fit,
|
|
86
|
+
n.fp?.x,
|
|
87
|
+
n.fp?.y
|
|
88
|
+
]), f = async (e, t, n, r) => u.getOrCreate(d(e, n, r), () => c(t, n, r));
|
|
89
|
+
//#endregion
|
|
90
|
+
export { l as editImage, t as focalRegion, i as isTransformable, o as loadSharp, r as parseTransformParams, f as transformCached, s as transformFallbackReason, c as transformImage, d as transformKey };
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
//#region src/transformCache.ts
|
|
2
|
+
var e = (e = {
|
|
3
|
+
entries: 200,
|
|
4
|
+
bytes: 67108864,
|
|
5
|
+
pending: 32
|
|
6
|
+
}) => {
|
|
7
|
+
let t = /* @__PURE__ */ new Map(), n = /* @__PURE__ */ new Map(), r = 0;
|
|
8
|
+
return { getOrCreate: (i, a) => {
|
|
9
|
+
let o = t.get(i);
|
|
10
|
+
if (o) return Promise.resolve(o);
|
|
11
|
+
let s = n.get(i);
|
|
12
|
+
if (s) return s;
|
|
13
|
+
if (n.size >= e.pending) return Promise.resolve(null);
|
|
14
|
+
let c = Promise.resolve().then(a).then((n) => {
|
|
15
|
+
if (n && e.entries > 0 && n.bytes.byteLength <= e.bytes) {
|
|
16
|
+
for (; t.size >= e.entries || r + n.bytes.byteLength > e.bytes;) {
|
|
17
|
+
let e = t.keys().next().value;
|
|
18
|
+
if (e === void 0) break;
|
|
19
|
+
r -= t.get(e).bytes.byteLength, t.delete(e);
|
|
20
|
+
}
|
|
21
|
+
t.set(i, n), r += n.bytes.byteLength;
|
|
22
|
+
}
|
|
23
|
+
return n;
|
|
24
|
+
}).finally(() => {
|
|
25
|
+
n.delete(i);
|
|
26
|
+
});
|
|
27
|
+
return n.set(i, c), c;
|
|
28
|
+
} };
|
|
29
|
+
};
|
|
30
|
+
//#endregion
|
|
31
|
+
export { e as createTransformCache };
|
package/dist/types.d.ts
CHANGED
|
@@ -208,6 +208,8 @@ export declare type StorageGuard = (ctx: {
|
|
|
208
208
|
* access policy.
|
|
209
209
|
*/
|
|
210
210
|
export declare interface StorageProvider {
|
|
211
|
+
/** Stable physical object-key namespace; exclude credentials and plugin aliases. */
|
|
212
|
+
readonly objectNamespace: Effect.Effect<string, StorageError>;
|
|
211
213
|
/**
|
|
212
214
|
* Where a FILESYSTEM provider writes. Absent for every other provider.
|
|
213
215
|
*
|
|
@@ -376,6 +378,8 @@ declare const StorageService_base: Context.ServiceClass<StorageService, "@voltro
|
|
|
376
378
|
|
|
377
379
|
/** The service handlers consume: `const storage = yield* StorageService`. */
|
|
378
380
|
export declare interface StorageServiceShape {
|
|
381
|
+
/** Opaque destination binding for manually signed upload-family credentials. */
|
|
382
|
+
readonly uploadTicketScope: () => Effect.Effect<string, StorageError>;
|
|
379
383
|
/** Store bytes. The result is the ref plus this call's `outcome`. */
|
|
380
384
|
readonly put: (input: PutInput) => Effect.Effect<StoredRef, StorageError | StorageRejected | StorageScanRejected>;
|
|
381
385
|
/** Fetch an object's bytes by id. Enforces the tenant guard ONLY — it does
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-storage",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.86.0",
|
|
4
4
|
"description": "File-storage plugin — one StorageProvider contract, switchable backends (S3 / MinIO / Azure / filesystem / memory / database). Public (CDN-direct) + private (access policy + per-object grants) objects, upload constraints, virus-scan hook, on-the-fly image transforms.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -43,10 +43,10 @@
|
|
|
43
43
|
"node": ">=24.0.0"
|
|
44
44
|
},
|
|
45
45
|
"dependencies": {
|
|
46
|
-
"@voltro/database": "0.
|
|
47
|
-
"@voltro/env": "0.
|
|
48
|
-
"@voltro/protocol": "0.
|
|
49
|
-
"@voltro/runtime": "0.
|
|
46
|
+
"@voltro/database": "0.86.0",
|
|
47
|
+
"@voltro/env": "0.86.0",
|
|
48
|
+
"@voltro/protocol": "0.86.0",
|
|
49
|
+
"@voltro/runtime": "0.86.0"
|
|
50
50
|
},
|
|
51
51
|
"optionalDependencies": {
|
|
52
52
|
"@aws-sdk/client-s3": "^3.1134.0",
|