@voltro/plugin-storage 0.86.0 → 0.88.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5484 -0
- package/README.md +27 -0
- package/THIRD-PARTY-NOTICES.md +2503 -690
- package/dist/access.js +26 -27
- package/dist/assetRef.js +1 -2
- package/dist/avatar.js +1 -2
- package/dist/byteSecurity.d.ts +3 -0
- package/dist/byteSecurity.js +6 -0
- package/dist/connection.js +98 -0
- package/dist/constraints.js +1 -2
- package/dist/custody.js +131 -0
- package/dist/derivatives.js +1 -2
- package/dist/env.js +8 -3
- package/dist/grant.js +90 -71
- package/dist/index.d.ts +453 -215
- package/dist/index.js +709 -537
- package/dist/ingest.js +7 -2
- package/dist/ingestPolicy.js +1 -1
- package/dist/metadataError.js +8 -9
- package/dist/objectAbandon.js +15 -0
- package/dist/objectLifecycle.js +135 -102
- package/dist/objectMetadata.js +253 -96
- package/dist/objectWriter.js +51 -23
- package/dist/pathDurability.js +1 -2
- package/dist/physicalReferences.js +53 -0
- package/dist/providers.js +451 -307
- package/dist/referenceCatalogue.js +1 -2
- package/dist/renditionProof.js +81 -0
- package/dist/retainedCapture.js +390 -0
- package/dist/retainedObjects.js +189 -0
- package/dist/rpc.d.ts +1 -15
- package/dist/rpc.js +1 -2
- package/dist/scanners.js +1 -2
- package/dist/server.d.ts +153 -0
- package/dist/server.js +4 -0
- package/dist/service.js +879 -546
- package/dist/sourceIdentity.js +13 -0
- package/dist/sourcePin.js +34 -0
- package/dist/storageNamespaces.js +52 -29
- package/dist/storageNumbers.js +1 -2
- package/dist/storageObjectNamespaces.js +170 -122
- package/dist/storageReferenceInstances.js +1 -2
- package/dist/storageRequest.js +9 -0
- package/dist/storageRouting.js +45 -0
- package/dist/streamBridge.js +5 -3
- package/dist/table.js +2 -2
- package/dist/transcodeVideo.js +85 -48
- package/dist/transform.js +30 -27
- package/dist/transformCache.js +1 -2
- package/dist/types.d.ts +115 -137
- package/dist/types.js +15 -16
- package/package.json +22 -11
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { storageObjectNamespace as e } from "./storageNamespaces.js";
|
|
2
2
|
import { randomUUID as t } from "node:crypto";
|
|
3
3
|
import { authoredError as n, eq as r, table as i, text as a } from "@voltro/database";
|
|
4
|
-
|
|
4
|
+
// src/referenceCatalogue.ts
|
|
5
5
|
var o = "_voltro_storage_reference_catalogue", s = i(o, {
|
|
6
6
|
id: a().maxLength(64),
|
|
7
7
|
namespace: a().maxLength(64)
|
|
@@ -27,5 +27,4 @@ var o = "_voltro_storage_reference_catalogue", s = i(o, {
|
|
|
27
27
|
i.objectNamespace
|
|
28
28
|
]) : s;
|
|
29
29
|
};
|
|
30
|
-
//#endregion
|
|
31
30
|
export { o as STORAGE_REFERENCE_CATALOGUE_TABLE, c as ensureStorageReferenceCatalogue, l as storageReferenceCatalogueId, s as storageReferenceCatalogueTable };
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { StorageError as e } from "./types.js";
|
|
2
|
+
import { retainedSourceDigest as t } from "./sourceIdentity.js";
|
|
3
|
+
import { createHash as n } from "node:crypto";
|
|
4
|
+
import { canonicalJson as r } from "@voltro/protocol";
|
|
5
|
+
// src/renditionProof.ts
|
|
6
|
+
var i = (e) => n("sha256").update(r(e)).digest("hex"), a = (e) => n("sha256").update(e).digest("hex"), o = (e) => typeof e == "string" && /^[a-f0-9]{64}$/.test(e), s = (e, t) => typeof e == "string" && e.length > 0 && e.length <= t, c = () => {
|
|
7
|
+
throw new e({
|
|
8
|
+
provider: "transcode",
|
|
9
|
+
message: "Rendition provenance does not match the selected input and stored output",
|
|
10
|
+
status: 409,
|
|
11
|
+
transient: !1
|
|
12
|
+
});
|
|
13
|
+
}, l = (e) => !s(e?.recipe?.format, 128) || !o(e.recipe.digest) || !s(e.engine?.name, 128) || !s(e.engine.version, 512) || !o(e.engine.digest) ? c() : {
|
|
14
|
+
recipe: {
|
|
15
|
+
format: e.recipe.format,
|
|
16
|
+
digest: e.recipe.digest
|
|
17
|
+
},
|
|
18
|
+
engine: {
|
|
19
|
+
name: e.engine.name,
|
|
20
|
+
version: e.engine.version,
|
|
21
|
+
digest: e.engine.digest
|
|
22
|
+
}
|
|
23
|
+
}, u = (e, n, r, u) => {
|
|
24
|
+
if (r.derivation === void 0) return;
|
|
25
|
+
let d = a(n);
|
|
26
|
+
if (!s(e.id, 64) || !s(e.checksum, 512) || e.size !== n.length || !s(e.contentType, 512) || !s(r.contentType, 512)) return c();
|
|
27
|
+
if (o(e.checksum)) {
|
|
28
|
+
if (e.checksum !== d) return c();
|
|
29
|
+
} else if (![u?.before, u?.after].every((t) => t != null && t.size === e.size && t.etag?.replace(/^"|"$/g, "") === e.checksum)) return c();
|
|
30
|
+
let f = {
|
|
31
|
+
version: 1,
|
|
32
|
+
...l(r.derivation),
|
|
33
|
+
source: {
|
|
34
|
+
id: e.id,
|
|
35
|
+
metadataDigest: t(e),
|
|
36
|
+
checksum: d,
|
|
37
|
+
size: e.size,
|
|
38
|
+
contentType: e.contentType
|
|
39
|
+
},
|
|
40
|
+
output: {
|
|
41
|
+
checksum: a(r.bytes),
|
|
42
|
+
size: r.bytes.length,
|
|
43
|
+
contentType: r.contentType
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
return {
|
|
47
|
+
...f,
|
|
48
|
+
digest: i(f)
|
|
49
|
+
};
|
|
50
|
+
}, d = (e) => {
|
|
51
|
+
if (e == null) return null;
|
|
52
|
+
let t = e, n = t.source, r = t.output;
|
|
53
|
+
if (t.version !== 1 || !o(t.digest) || !n || !r || !s(n.id, 64) || !o(n.metadataDigest) || !o(n.checksum) || !Number.isSafeInteger(n.size) || n.size < 0 || !s(n.contentType, 512) || !o(r.checksum) || !Number.isSafeInteger(r.size) || r.size < 0 || !s(r.contentType, 512)) return c();
|
|
54
|
+
let a = {
|
|
55
|
+
version: 1,
|
|
56
|
+
...l(t),
|
|
57
|
+
source: {
|
|
58
|
+
id: n.id,
|
|
59
|
+
metadataDigest: n.metadataDigest,
|
|
60
|
+
checksum: n.checksum,
|
|
61
|
+
size: n.size,
|
|
62
|
+
contentType: n.contentType
|
|
63
|
+
},
|
|
64
|
+
output: {
|
|
65
|
+
checksum: r.checksum,
|
|
66
|
+
size: r.size,
|
|
67
|
+
contentType: r.contentType
|
|
68
|
+
}
|
|
69
|
+
};
|
|
70
|
+
return i(a) === t.digest ? {
|
|
71
|
+
...a,
|
|
72
|
+
digest: t.digest
|
|
73
|
+
} : c();
|
|
74
|
+
}, f = (e, t) => {
|
|
75
|
+
let n = d(e);
|
|
76
|
+
return n.source.id !== t.derivedFrom || n.output.checksum !== t.checksum || n.output.size !== t.size || n.output.contentType !== t.contentType ? c() : n;
|
|
77
|
+
}, p = (e) => e.renditionProof == null ? e : {
|
|
78
|
+
...e,
|
|
79
|
+
renditionProof: f(e.renditionProof, e)
|
|
80
|
+
};
|
|
81
|
+
export { f as assertRenditionOutput, l as copyRenditionDerivation, u as makeRenditionProof, p as ownRenditionRef, d as readRenditionProof };
|
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
import { requireStorageReferenceScope as e } from "./storageNamespaces.js";
|
|
2
|
+
import { STORAGE_REFS_TABLE as t } from "./table.js";
|
|
3
|
+
import { storageMetadataError as n } from "./metadataError.js";
|
|
4
|
+
import { makeStorageObjectWriter as r } from "./objectWriter.js";
|
|
5
|
+
import { dataStoreRefStore as i, toRef as a } from "./service.js";
|
|
6
|
+
import { makeStorageObjectMetadata as o } from "./objectMetadata.js";
|
|
7
|
+
import { abandonStorageObject as s } from "./objectAbandon.js";
|
|
8
|
+
import { readRetainedStorageObject as c, retainStorageObject as l, retainedPinMatches as u } from "./retainedObjects.js";
|
|
9
|
+
import { pinStorageSource as d } from "./sourcePin.js";
|
|
10
|
+
import { Effect as f, Schema as p, Stream as m } from "effect";
|
|
11
|
+
import { createHash as h, randomUUID as g } from "node:crypto";
|
|
12
|
+
import { and as _, eq as v, gt as y, integer as b, json as x, table as S, text as C } from "@voltro/database";
|
|
13
|
+
import { canonicalJson as w } from "@voltro/protocol";
|
|
14
|
+
// src/retainedCapture.ts
|
|
15
|
+
var T = "_voltro_storage_retained_captures", E = "_voltro_storage_retained_capture_items", D = [S(T, {
|
|
16
|
+
id: C().maxLength(64),
|
|
17
|
+
captureId: C().maxLength(64),
|
|
18
|
+
instance: C().maxLength(256),
|
|
19
|
+
tenantId: C().maxLength(64).nullable(),
|
|
20
|
+
bucket: C().maxLength(128),
|
|
21
|
+
contextDigest: C().maxLength(64),
|
|
22
|
+
state: C().oneOf([
|
|
23
|
+
"capturing",
|
|
24
|
+
"complete",
|
|
25
|
+
"abandoned"
|
|
26
|
+
]),
|
|
27
|
+
nonce: C().maxLength(64),
|
|
28
|
+
config: x(),
|
|
29
|
+
count: b(),
|
|
30
|
+
digest: C().maxLength(64).nullable(),
|
|
31
|
+
cleanupOrdinal: b().default(0)
|
|
32
|
+
}).primaryKey(["id"]).nonReactive(), S(E, {
|
|
33
|
+
id: C().maxLength(64),
|
|
34
|
+
captureId: C().maxLength(64),
|
|
35
|
+
ordinal: b(),
|
|
36
|
+
sourceId: C().maxLength(64),
|
|
37
|
+
sourceDigest: C().maxLength(64),
|
|
38
|
+
operationId: C().maxLength(64),
|
|
39
|
+
pin: x().nullable()
|
|
40
|
+
}).primaryKey(["id"]).index("storage_retained_capture_page", ["captureId", "ordinal"]).nonReactive()], O = class extends p.TaggedError()("RetainedCaptureUnavailable", { reason: p.Literals([
|
|
41
|
+
"scope",
|
|
42
|
+
"identity",
|
|
43
|
+
"state",
|
|
44
|
+
"source",
|
|
45
|
+
"bounds",
|
|
46
|
+
"incomplete",
|
|
47
|
+
"integrity"
|
|
48
|
+
]) }) {}, k = (e) => {
|
|
49
|
+
throw new O({ reason: e });
|
|
50
|
+
}, A = (e) => ({
|
|
51
|
+
id: e.id,
|
|
52
|
+
instance: e.instance,
|
|
53
|
+
objectNamespace: e.objectNamespace,
|
|
54
|
+
tenantId: e.tenantId,
|
|
55
|
+
bucket: e.bucket,
|
|
56
|
+
contextDigest: e.contextDigest,
|
|
57
|
+
...e.source === void 0 ? {} : { source: {
|
|
58
|
+
instance: e.source.instance,
|
|
59
|
+
objectNamespace: e.source.objectNamespace,
|
|
60
|
+
bucket: e.source.bucket
|
|
61
|
+
} }
|
|
62
|
+
}), j = (e) => h("sha256").update(w(e)).digest("hex"), M = (e) => typeof e == "string" && /^[a-f0-9]{64}$/.test(e), N = (e, t) => typeof e == "string" && e.length > 0 && e.length <= t, P = (e) => N(e.id, 64) && N(e.instance, 256) && M(e.objectNamespace) && (e.tenantId === null || N(e.tenantId, 64)) && N(e.bucket, 128) && M(e.contextDigest) && (e.source === void 0 || N(e.source.instance, 256) && M(e.source.objectNamespace) && N(e.source.bucket, 128)), F = (e) => f.tryPromise({
|
|
63
|
+
try: e,
|
|
64
|
+
catch: (e) => n(e, "metadata")
|
|
65
|
+
}), I = (e) => j([
|
|
66
|
+
e.instance,
|
|
67
|
+
e.objectNamespace,
|
|
68
|
+
e.tenantId,
|
|
69
|
+
e.bucket,
|
|
70
|
+
e.id
|
|
71
|
+
]), L = (e) => `retained-capture:${I(e)}`, R = (e, t) => j([I(e), t]), z = (e, t) => j([
|
|
72
|
+
"retained-capture/v1",
|
|
73
|
+
I(e),
|
|
74
|
+
t
|
|
75
|
+
]), B = async (e, t, n = !1) => {
|
|
76
|
+
if (!P(t)) return k("scope");
|
|
77
|
+
let [r] = await e.query({
|
|
78
|
+
table: T,
|
|
79
|
+
predicate: v("id", I(t)),
|
|
80
|
+
order: [],
|
|
81
|
+
take: 1,
|
|
82
|
+
skip: void 0,
|
|
83
|
+
projection: void 0,
|
|
84
|
+
...n ? { lock: "update" } : {}
|
|
85
|
+
});
|
|
86
|
+
if (r === void 0 || r.id !== I(t) || r.captureId !== t.id || r.instance !== t.instance || r.tenantId !== t.tenantId || r.bucket !== t.bucket || r.contextDigest !== t.contextDigest) return k("identity");
|
|
87
|
+
let i = r.config;
|
|
88
|
+
return typeof i != "object" || !i || !Number.isSafeInteger(i.maxBytes) || Number(i.maxBytes) < 0 || !Number.isSafeInteger(r.count) || Number(r.count) < 0 || !Number.isSafeInteger(r.cleanupOrdinal) || Number(r.cleanupOrdinal) < 0 || Number(r.cleanupOrdinal) > Number(r.count) || ![
|
|
89
|
+
"capturing",
|
|
90
|
+
"complete",
|
|
91
|
+
"abandoned"
|
|
92
|
+
].includes(String(r.state)) ? k("integrity") : i.objectNamespace !== t.objectNamespace || w(i.source ?? null) !== w(t.source ?? null) || n && await e.update("_voltro_storage_retained_captures", I(t), { nonce: g() }) === null ? k("identity") : {
|
|
93
|
+
state: r.state,
|
|
94
|
+
count: Number(r.count),
|
|
95
|
+
maxBytes: Number(i.maxBytes),
|
|
96
|
+
digest: r.digest,
|
|
97
|
+
cleanupOrdinal: Number(r.cleanupOrdinal)
|
|
98
|
+
};
|
|
99
|
+
}, V = (e, t) => {
|
|
100
|
+
if (t.captureId !== I(e) || !N(t.sourceId, 64) || t.id !== R(e, t.sourceId) || t.operationId !== z(e, t.sourceId) || !M(t.sourceDigest) || !Number.isSafeInteger(t.ordinal) || Number(t.ordinal) < 1) return k("integrity");
|
|
101
|
+
let n = t.pin;
|
|
102
|
+
return n !== null && (typeof n != "object" || n.version !== 1 || n.tenantId !== e.tenantId || n.bucket !== e.bucket || n.operationId !== t.operationId || n.sourceId !== t.sourceId || n.sourceDigest !== t.sourceDigest || !M(n.checksum) || !N(n.refId, 64) || !Number.isSafeInteger(n.size) || n.size < 0 || !N(n.contentType, 512)) ? k("integrity") : {
|
|
103
|
+
id: String(t.id),
|
|
104
|
+
ordinal: Number(t.ordinal),
|
|
105
|
+
sourceId: t.sourceId,
|
|
106
|
+
sourceDigest: t.sourceDigest,
|
|
107
|
+
operationId: String(t.operationId),
|
|
108
|
+
pin: n
|
|
109
|
+
};
|
|
110
|
+
}, H = async function* (e, t, n) {
|
|
111
|
+
let r = 0;
|
|
112
|
+
for (; r < n;) {
|
|
113
|
+
let i = await e.query({
|
|
114
|
+
table: E,
|
|
115
|
+
predicate: _(v("captureId", I(t)), y("ordinal", r)),
|
|
116
|
+
order: [{
|
|
117
|
+
column: "ordinal",
|
|
118
|
+
direction: "asc"
|
|
119
|
+
}],
|
|
120
|
+
take: Math.min(100, n - r),
|
|
121
|
+
skip: void 0,
|
|
122
|
+
projection: void 0
|
|
123
|
+
});
|
|
124
|
+
if (i.length !== Math.min(100, n - r)) return k("integrity");
|
|
125
|
+
for (let e of i) {
|
|
126
|
+
let n = V(t, e);
|
|
127
|
+
if (n.ordinal !== ++r) return k("integrity");
|
|
128
|
+
yield n;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}, U = async (n, r, i) => {
|
|
132
|
+
if (e({
|
|
133
|
+
storageInstance: r.instance,
|
|
134
|
+
objectNamespace: r.objectNamespace
|
|
135
|
+
}), !N(i, 64) || !N(r.instance, 256) || !N(r.bucket, 128) || r.tenantId !== null && !N(r.tenantId, 64)) return k("scope");
|
|
136
|
+
let [o] = await n.query({
|
|
137
|
+
table: t,
|
|
138
|
+
predicate: _(v("id", i), v("storageInstance", r.instance), v("objectNamespace", r.objectNamespace)),
|
|
139
|
+
lock: "update",
|
|
140
|
+
order: [],
|
|
141
|
+
take: 1,
|
|
142
|
+
skip: void 0,
|
|
143
|
+
projection: void 0
|
|
144
|
+
});
|
|
145
|
+
if (o === void 0) return k("source");
|
|
146
|
+
let s = a(o);
|
|
147
|
+
return !N(s.checksum, 512) || !N(s.contentType, 512) ? k("source") : s.tenantId !== r.tenantId || s.bucket !== r.bucket ? k("scope") : d(s, r.instance);
|
|
148
|
+
}, W = async (e, t, n) => {
|
|
149
|
+
let r = A(t), i = n.maxBytes, a = n.sources;
|
|
150
|
+
if (!P(r)) return k("scope");
|
|
151
|
+
if (!Number.isSafeInteger(i) || i < 0) return k("bounds");
|
|
152
|
+
let [o] = await e.query({
|
|
153
|
+
table: T,
|
|
154
|
+
predicate: v("id", I(r)),
|
|
155
|
+
order: [],
|
|
156
|
+
take: 1,
|
|
157
|
+
skip: void 0,
|
|
158
|
+
projection: void 0
|
|
159
|
+
});
|
|
160
|
+
if (o !== void 0) {
|
|
161
|
+
let t = await B(e, r, !0);
|
|
162
|
+
return t.maxBytes === i ? t.state === "abandoned" ? k("state") : { count: t.count } : k("identity");
|
|
163
|
+
}
|
|
164
|
+
await e.insert(T, {
|
|
165
|
+
id: I(r),
|
|
166
|
+
captureId: r.id,
|
|
167
|
+
instance: r.instance,
|
|
168
|
+
tenantId: r.tenantId,
|
|
169
|
+
bucket: r.bucket,
|
|
170
|
+
contextDigest: r.contextDigest,
|
|
171
|
+
state: "capturing",
|
|
172
|
+
config: {
|
|
173
|
+
maxBytes: i,
|
|
174
|
+
objectNamespace: r.objectNamespace,
|
|
175
|
+
...r.source === void 0 ? {} : { source: r.source }
|
|
176
|
+
},
|
|
177
|
+
nonce: g(),
|
|
178
|
+
count: 0,
|
|
179
|
+
digest: null,
|
|
180
|
+
cleanupOrdinal: 0
|
|
181
|
+
});
|
|
182
|
+
let s = 0;
|
|
183
|
+
for await (let t of a()) {
|
|
184
|
+
let n = typeof t == "string" ? void 0 : JSON.parse(w(t)), a = typeof t == "string" ? t : n?.id;
|
|
185
|
+
if (!N(a, 64)) return k("source");
|
|
186
|
+
let o = R(r, a), [c] = await e.query({
|
|
187
|
+
table: E,
|
|
188
|
+
predicate: v("id", o),
|
|
189
|
+
order: [],
|
|
190
|
+
take: 1,
|
|
191
|
+
skip: void 0,
|
|
192
|
+
projection: ["id", "sourceDigest"]
|
|
193
|
+
});
|
|
194
|
+
if (c !== void 0 && n === void 0) continue;
|
|
195
|
+
let l = await U(e, {
|
|
196
|
+
instance: r.source?.instance ?? r.instance,
|
|
197
|
+
objectNamespace: r.source?.objectNamespace ?? r.objectNamespace,
|
|
198
|
+
tenantId: r.tenantId,
|
|
199
|
+
bucket: r.source?.bucket ?? r.bucket
|
|
200
|
+
}, a);
|
|
201
|
+
if (n !== void 0 && w(l) !== w(n)) return k("source");
|
|
202
|
+
if (c !== void 0) {
|
|
203
|
+
if (c.sourceDigest !== l.sourceDigest) return k("source");
|
|
204
|
+
} else {
|
|
205
|
+
if (l.size > i || s === 2147483647) return k("bounds");
|
|
206
|
+
await e.insert(E, {
|
|
207
|
+
id: o,
|
|
208
|
+
captureId: I(r),
|
|
209
|
+
ordinal: ++s,
|
|
210
|
+
sourceId: a,
|
|
211
|
+
sourceDigest: l.sourceDigest,
|
|
212
|
+
operationId: z(r, a),
|
|
213
|
+
pin: null
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return await e.update(T, I(r), { count: s }), { count: s };
|
|
218
|
+
}, G = (e, t) => async (n, r) => {
|
|
219
|
+
if ((await B(n, e, !0)).state !== "capturing") return k("state");
|
|
220
|
+
if (r.tenantId !== e.tenantId || r.bucket !== e.bucket || r.operationId !== t.operationId || r.retainedSource?.refId !== t.sourceId || r.retainedSource.digest !== t.sourceDigest) return k("identity");
|
|
221
|
+
}, K = (e, t, n, r, i) => f.gen(function* () {
|
|
222
|
+
let a = A(n), o = yield* t.objectNamespace;
|
|
223
|
+
if (o !== a.objectNamespace) return yield* f.fail(new O({ reason: "scope" }));
|
|
224
|
+
let s = i === void 0 ? o : yield* i.provider.objectNamespace;
|
|
225
|
+
if (w(a.source ?? {
|
|
226
|
+
instance: a.instance,
|
|
227
|
+
objectNamespace: a.objectNamespace,
|
|
228
|
+
bucket: a.bucket
|
|
229
|
+
}) !== w(i === void 0 ? {
|
|
230
|
+
instance: a.instance,
|
|
231
|
+
objectNamespace: a.objectNamespace,
|
|
232
|
+
bucket: a.bucket
|
|
233
|
+
} : {
|
|
234
|
+
instance: i.instance,
|
|
235
|
+
objectNamespace: s,
|
|
236
|
+
bucket: i.bucket
|
|
237
|
+
})) return yield* f.fail(new O({ reason: "scope" }));
|
|
238
|
+
let u = yield* F(() => B(e, a));
|
|
239
|
+
if (u.state === "abandoned") return yield* f.fail(new O({ reason: "state" }));
|
|
240
|
+
if (u.state === "complete") {
|
|
241
|
+
if (!M(u.digest)) return yield* f.fail(new O({ reason: "integrity" }));
|
|
242
|
+
let n = Y(e, {
|
|
243
|
+
...A(a),
|
|
244
|
+
count: u.count,
|
|
245
|
+
digest: u.digest
|
|
246
|
+
});
|
|
247
|
+
for (;;) {
|
|
248
|
+
let r = yield* F(() => n.next());
|
|
249
|
+
if (r.done) break;
|
|
250
|
+
let i = yield* c(e, t, r.value, {
|
|
251
|
+
storageInstance: a.instance,
|
|
252
|
+
tenantId: a.tenantId,
|
|
253
|
+
bucket: a.bucket,
|
|
254
|
+
maxBytes: u.maxBytes
|
|
255
|
+
});
|
|
256
|
+
yield* m.runDrain(i.stream);
|
|
257
|
+
}
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
let d = H(e, a, u.count);
|
|
261
|
+
for (;;) {
|
|
262
|
+
let n = yield* F(() => d.next());
|
|
263
|
+
if (n.done) break;
|
|
264
|
+
let o = n.value, s = yield* l(e, t, {
|
|
265
|
+
storageInstance: a.instance,
|
|
266
|
+
tenantId: a.tenantId,
|
|
267
|
+
bucket: a.bucket,
|
|
268
|
+
maxBytes: u.maxBytes,
|
|
269
|
+
namespace: a.instance,
|
|
270
|
+
sourceId: o.sourceId,
|
|
271
|
+
expectedSourceDigest: o.sourceDigest,
|
|
272
|
+
operationId: o.operationId,
|
|
273
|
+
claim: L(a),
|
|
274
|
+
admit: G(a, o),
|
|
275
|
+
...r === void 0 ? {} : { quota: r },
|
|
276
|
+
...i === void 0 ? {} : { source: {
|
|
277
|
+
...i,
|
|
278
|
+
storageInstance: i.instance
|
|
279
|
+
} }
|
|
280
|
+
});
|
|
281
|
+
yield* F(() => e.transactional(async (e) => {
|
|
282
|
+
if ((await B(e, a, !0)).state !== "capturing") return k("state");
|
|
283
|
+
await e.update(E, o.id, { pin: s });
|
|
284
|
+
}));
|
|
285
|
+
}
|
|
286
|
+
}), q = async (e, t) => {
|
|
287
|
+
let n = A(t), r = await B(e, n, !0);
|
|
288
|
+
if (r.state === "abandoned") return k("state");
|
|
289
|
+
let a = h("sha256").update(w({
|
|
290
|
+
...A(n),
|
|
291
|
+
count: r.count
|
|
292
|
+
}) + "\n"), s = o(e, {
|
|
293
|
+
storageInstance: n.instance,
|
|
294
|
+
objectNamespace: n.objectNamespace
|
|
295
|
+
});
|
|
296
|
+
for await (let t of H(e, n, r.count)) {
|
|
297
|
+
if (t.pin === null) return k("incomplete");
|
|
298
|
+
let r = await s.operation({
|
|
299
|
+
tenantId: n.tenantId,
|
|
300
|
+
bucket: n.bucket,
|
|
301
|
+
operationId: t.operationId
|
|
302
|
+
});
|
|
303
|
+
if (r?.state !== "ready" || !u(t.pin, r.identity, r.ref)) return k("integrity");
|
|
304
|
+
let o = await f.runPromise(i(e, {
|
|
305
|
+
storageInstance: n.instance,
|
|
306
|
+
objectNamespace: n.objectNamespace
|
|
307
|
+
}).getById(t.pin.refId));
|
|
308
|
+
if (o === null || o.key !== r.ref.key || !u(t.pin, r.identity, o)) return k("integrity");
|
|
309
|
+
a.update(j({
|
|
310
|
+
ordinal: t.ordinal,
|
|
311
|
+
pin: t.pin
|
|
312
|
+
}) + "\n");
|
|
313
|
+
}
|
|
314
|
+
let c = a.digest("hex");
|
|
315
|
+
return r.state === "complete" && r.digest !== c ? k("integrity") : (await e.update(T, I(n), {
|
|
316
|
+
state: "complete",
|
|
317
|
+
digest: c
|
|
318
|
+
}), {
|
|
319
|
+
...A(n),
|
|
320
|
+
count: r.count,
|
|
321
|
+
digest: c
|
|
322
|
+
});
|
|
323
|
+
}, J = (e, t, n) => f.gen(function* () {
|
|
324
|
+
let i = A(n);
|
|
325
|
+
if ((yield* t.objectNamespace) !== i.objectNamespace) return yield* f.fail(new O({ reason: "scope" }));
|
|
326
|
+
let a = yield* F(() => e.transactional(async (e) => {
|
|
327
|
+
let t = await B(e, i, !0);
|
|
328
|
+
return await e.update(T, I(i), { state: "abandoned" }), t;
|
|
329
|
+
})), c = r({
|
|
330
|
+
metadata: o(e, {
|
|
331
|
+
storageInstance: i.instance,
|
|
332
|
+
objectNamespace: i.objectNamespace
|
|
333
|
+
}),
|
|
334
|
+
provider: t
|
|
335
|
+
}), l = a.cleanupOrdinal;
|
|
336
|
+
for (let n = 0; n < a.count; n++) {
|
|
337
|
+
let n = l % a.count + 1, r = yield* F(() => e.transactional(async (e) => {
|
|
338
|
+
let t = await B(e, i, !0);
|
|
339
|
+
return t.state !== "abandoned" || t.count !== a.count ? k("integrity") : (t.cleanupOrdinal === l && await e.update(T, I(i), { cleanupOrdinal: n }), e.query({
|
|
340
|
+
table: E,
|
|
341
|
+
predicate: _(v("captureId", I(i)), v("ordinal", n)),
|
|
342
|
+
order: [],
|
|
343
|
+
take: 2,
|
|
344
|
+
skip: void 0,
|
|
345
|
+
projection: void 0
|
|
346
|
+
}));
|
|
347
|
+
})), o = yield* F(async () => r.length === 1 ? V(i, r[0]) : k("integrity")), u = yield* s(e, t, {
|
|
348
|
+
storageInstance: i.instance,
|
|
349
|
+
objectNamespace: i.objectNamespace,
|
|
350
|
+
tenantId: i.tenantId,
|
|
351
|
+
bucket: i.bucket,
|
|
352
|
+
operationId: o.operationId
|
|
353
|
+
});
|
|
354
|
+
u.state === "ready" && (yield* c.delete(u.refId, i.tenantId, L(i))), l = n;
|
|
355
|
+
}
|
|
356
|
+
}), Y = async function* (e, t) {
|
|
357
|
+
let n = {
|
|
358
|
+
...A(t),
|
|
359
|
+
count: t.count,
|
|
360
|
+
digest: t.digest
|
|
361
|
+
}, r = await B(e, n);
|
|
362
|
+
if (r.state !== "complete" || r.digest !== n.digest || r.count !== n.count) return k("state");
|
|
363
|
+
let i = h("sha256").update(w({
|
|
364
|
+
...A(n),
|
|
365
|
+
count: n.count
|
|
366
|
+
}) + "\n");
|
|
367
|
+
for await (let t of H(e, n, r.count)) {
|
|
368
|
+
if (t.pin === null) return k("integrity");
|
|
369
|
+
i.update(j({
|
|
370
|
+
ordinal: t.ordinal,
|
|
371
|
+
pin: t.pin
|
|
372
|
+
}) + "\n"), yield t.pin;
|
|
373
|
+
}
|
|
374
|
+
if (i.digest("hex") !== n.digest) return k("integrity");
|
|
375
|
+
}, X = async (e, t, n) => {
|
|
376
|
+
let r = await B(e, t);
|
|
377
|
+
if (r.state !== "complete" || r.digest !== t.digest || r.count !== t.count) return k("state");
|
|
378
|
+
let [i] = await e.query({
|
|
379
|
+
table: E,
|
|
380
|
+
predicate: v("id", R(t, n.sourceId)),
|
|
381
|
+
order: [],
|
|
382
|
+
take: 1,
|
|
383
|
+
skip: void 0,
|
|
384
|
+
projection: void 0
|
|
385
|
+
});
|
|
386
|
+
if (i === void 0) return k("identity");
|
|
387
|
+
let a = V(t, i);
|
|
388
|
+
if (a.pin === null || w(a.pin) !== w(n)) return k("integrity");
|
|
389
|
+
};
|
|
390
|
+
export { T as RETAINED_CAPTURES, E as RETAINED_CAPTURE_ITEMS, O as RetainedCaptureUnavailable, J as abandonRetainedCapture, X as admitRetainedCaptureObject, U as inspectRetainedSource, W as registerRetainedCapture, Y as retainedCaptureItems, D as retainedCaptureTables, K as runRetainedCapture, q as sealRetainedCapture };
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import { StorageError as e } from "./types.js";
|
|
2
|
+
import { getObjectStream as t, putObjectStream as n } from "./streamBridge.js";
|
|
3
|
+
import { storageMetadataError as r } from "./metadataError.js";
|
|
4
|
+
import { filesystemProvider as i } from "./providers.js";
|
|
5
|
+
import { retainedSourceDigest as a } from "./sourceIdentity.js";
|
|
6
|
+
import { makeStorageObjectWriter as o } from "./objectWriter.js";
|
|
7
|
+
import { dataStoreRefStore as s } from "./service.js";
|
|
8
|
+
import { makeAdmittedStorageObjectMetadata as c, makeStorageObjectMetadata as l } from "./objectMetadata.js";
|
|
9
|
+
import { Effect as u, Schema as d, Stream as f } from "effect";
|
|
10
|
+
import { createHash as p } from "node:crypto";
|
|
11
|
+
import { join as m } from "node:path";
|
|
12
|
+
import { canonicalJson as h } from "@voltro/protocol";
|
|
13
|
+
import { mkdtemp as g, rm as _ } from "node:fs/promises";
|
|
14
|
+
import { tmpdir as v } from "node:os";
|
|
15
|
+
// src/retainedObjects.ts
|
|
16
|
+
var y = class extends d.TaggedError()("RetainedObjectUnavailable", { reason: d.Literals([
|
|
17
|
+
"scope",
|
|
18
|
+
"source",
|
|
19
|
+
"operation",
|
|
20
|
+
"integrity",
|
|
21
|
+
"bounds",
|
|
22
|
+
"private-storage",
|
|
23
|
+
"temporary-storage"
|
|
24
|
+
]) }) {}, b = (e) => u.fail(new y({ reason: e })), x = (e) => p("sha256").update(h(e)).digest("hex"), S = (e) => u.tryPromise({
|
|
25
|
+
try: e,
|
|
26
|
+
catch: (e) => r(e, "metadata")
|
|
27
|
+
}), C = (e) => typeof e == "string" && /^[a-f0-9]{64}$/.test(e), w = (e) => typeof e == "string" && e.length > 0, T = (e) => (e.tenantId === null || w(e.tenantId)) && w(e.bucket) && Number.isSafeInteger(e.maxBytes) && e.maxBytes >= 0, E = (e, t) => ({
|
|
28
|
+
version: 1,
|
|
29
|
+
tenantId: e.tenantId,
|
|
30
|
+
bucket: e.bucket,
|
|
31
|
+
operationId: e.operationId,
|
|
32
|
+
sourceId: e.retainedSource.refId,
|
|
33
|
+
sourceDigest: e.retainedSource.digest,
|
|
34
|
+
refId: t.id,
|
|
35
|
+
checksum: t.checksum,
|
|
36
|
+
size: t.size,
|
|
37
|
+
contentType: t.contentType
|
|
38
|
+
}), D = (e, t, n) => e.version === 1 && w(e.sourceId) && C(e.sourceDigest) && C(e.checksum) && Number.isSafeInteger(e.size) && e.size >= 0 && w(e.contentType) && t.retainedSource !== void 0 && h(E(t, n)) === h(e) && n.visibility === "private" && n.ownerId === null && n.passwordHash === null && n.accessPolicy === null && n.derivedFrom == null, O = (t, n, r) => f.suspend(() => {
|
|
39
|
+
let i = 0, a = p("sha256"), o = () => new e({
|
|
40
|
+
provider: "integrity",
|
|
41
|
+
message: "Retained storage bytes do not match their pinned identity",
|
|
42
|
+
transient: !1,
|
|
43
|
+
status: 409
|
|
44
|
+
});
|
|
45
|
+
return t.pipe(f.mapEffect((e) => u.suspend(() => {
|
|
46
|
+
if (e.length > n - i) return u.fail(o());
|
|
47
|
+
let t = Uint8Array.from(e);
|
|
48
|
+
return i += t.length, a.update(t), u.succeed(t);
|
|
49
|
+
})), f.concat(f.fromEffect(u.suspend(() => {
|
|
50
|
+
let e = a.digest("hex");
|
|
51
|
+
return i !== n || r !== void 0 && e !== r ? u.fail(o()) : u.succeed(/* @__PURE__ */ new Uint8Array());
|
|
52
|
+
}))));
|
|
53
|
+
}), k = (e, n, r, i) => u.gen(function* () {
|
|
54
|
+
if (!T(i) || r.tenantId !== i.tenantId || r.bucket !== i.bucket) return yield* b("scope");
|
|
55
|
+
if (!Number.isSafeInteger(r.size) || r.size < 0 || r.size > i.maxBytes) return yield* b("bounds");
|
|
56
|
+
let a = yield* n.objectNamespace, o = {
|
|
57
|
+
storageInstance: i.storageInstance,
|
|
58
|
+
objectNamespace: a
|
|
59
|
+
}, c = yield* S(() => l(e, o).operation({
|
|
60
|
+
tenantId: i.tenantId,
|
|
61
|
+
bucket: i.bucket,
|
|
62
|
+
operationId: r.operationId
|
|
63
|
+
}));
|
|
64
|
+
if (c === void 0 || c.state !== "ready" || !D(r, c.identity, c.ref)) return yield* b("operation");
|
|
65
|
+
let d = yield* s(e, o).getById(r.refId);
|
|
66
|
+
return d === null || !D(r, c.identity, d) || d.key !== c.ref.key ? yield* b("integrity") : n.publicUrl?.(d.key) == null ? {
|
|
67
|
+
stream: f.unwrap(t(n, d.key).pipe(u.map((e) => O(e.stream, r.size, r.checksum)))),
|
|
68
|
+
contentType: r.contentType,
|
|
69
|
+
size: r.size
|
|
70
|
+
} : yield* b("private-storage");
|
|
71
|
+
}), A = (e, r, d) => u.gen(function* () {
|
|
72
|
+
if (!T(d) || !w(d.sourceId) || !w(d.operationId) || !w(d.claim) || d.namespace !== void 0 && !w(d.namespace)) return yield* b("scope");
|
|
73
|
+
let h = (e) => `_retained/${x([
|
|
74
|
+
d.tenantId,
|
|
75
|
+
d.bucket,
|
|
76
|
+
d.namespace ?? null,
|
|
77
|
+
e
|
|
78
|
+
])}`;
|
|
79
|
+
if (d.expectedSourceDigest !== void 0 && !C(d.expectedSourceDigest)) return yield* b("source");
|
|
80
|
+
let D = yield* r.objectNamespace, A = {
|
|
81
|
+
storageInstance: d.storageInstance,
|
|
82
|
+
objectNamespace: D
|
|
83
|
+
}, j = d.admit === void 0 ? l(e, A) : c(e, A, d.admit), M = o({
|
|
84
|
+
metadata: j,
|
|
85
|
+
provider: r
|
|
86
|
+
}), N = yield* S(() => j.operation(d));
|
|
87
|
+
if (N !== void 0) {
|
|
88
|
+
if (N.identity.retainedSource?.refId !== d.sourceId || !C(N.identity.retainedSource.digest) || d.expectedSourceDigest !== void 0 && N.identity.retainedSource.digest !== d.expectedSourceDigest || N.identity.logicalKey !== h(N.identity.checksum) || N.state === "deleting" || N.state === "deleted") return yield* b("operation");
|
|
89
|
+
let n = E(N.identity, N.ref);
|
|
90
|
+
if (n.size > d.maxBytes) return yield* b("bounds");
|
|
91
|
+
if (r.publicUrl?.(N.ref.key) != null) return yield* b("private-storage");
|
|
92
|
+
if (N.state === "ready") {
|
|
93
|
+
let t = yield* k(e, r, n, d);
|
|
94
|
+
return yield* f.runDrain(t.stream), yield* M.putStream({
|
|
95
|
+
identity: N.identity,
|
|
96
|
+
ref: N.ref,
|
|
97
|
+
stream: f.empty,
|
|
98
|
+
claim: d.claim,
|
|
99
|
+
...d.quota === void 0 ? {} : { quota: d.quota }
|
|
100
|
+
}), n;
|
|
101
|
+
}
|
|
102
|
+
if ((yield* r.head(N.ref.key)) !== null) {
|
|
103
|
+
let e = f.unwrap(t(r, N.ref.key).pipe(u.map((e) => e.stream))), n = yield* M.putStream({
|
|
104
|
+
identity: N.identity,
|
|
105
|
+
ref: N.ref,
|
|
106
|
+
stream: e,
|
|
107
|
+
claim: d.claim,
|
|
108
|
+
...d.quota === void 0 ? {} : { quota: d.quota }
|
|
109
|
+
});
|
|
110
|
+
return E(N.identity, n.ref);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
let P = d.source?.provider ?? r, F = d.source?.bucket ?? d.bucket, I = {
|
|
114
|
+
storageInstance: d.source?.storageInstance ?? d.storageInstance,
|
|
115
|
+
objectNamespace: yield* P.objectNamespace
|
|
116
|
+
}, L = yield* s(e, I).getById(d.sourceId);
|
|
117
|
+
if (L === null) return yield* b("source");
|
|
118
|
+
if (L.tenantId !== d.tenantId || L.bucket !== F) return yield* b("scope");
|
|
119
|
+
if (!Number.isSafeInteger(L.size) || L.size < 0 || L.size > d.maxBytes) return yield* b("bounds");
|
|
120
|
+
let R = a(L);
|
|
121
|
+
if (d.expectedSourceDigest !== void 0 && R !== d.expectedSourceDigest) return yield* b("source");
|
|
122
|
+
if (N !== void 0 && N.identity.retainedSource?.digest !== R) return yield* b("operation");
|
|
123
|
+
let z = yield* u.tryPromise({
|
|
124
|
+
try: () => g(m(v(), "voltro-retained-input-")),
|
|
125
|
+
catch: () => new y({ reason: "temporary-storage" })
|
|
126
|
+
});
|
|
127
|
+
return yield* u.gen(function* () {
|
|
128
|
+
let e = i({ root: z }), a = L.checksum !== "" && !C(L.checksum), o = (e) => e !== void 0 && e.replace(/^"|"$/g, "") === L.checksum;
|
|
129
|
+
if (a) {
|
|
130
|
+
let e = yield* P.head(L.key);
|
|
131
|
+
if (e === null || e.size !== L.size || !o(e.etag)) return yield* b("integrity");
|
|
132
|
+
}
|
|
133
|
+
let s = p("sha256"), c = N?.identity.checksum ?? (C(L.checksum) ? L.checksum : void 0), l = f.unwrap(t(P, L.key).pipe(u.map((e) => O(e.stream, L.size, c)))).pipe(f.tap((e) => u.sync(() => {
|
|
134
|
+
s.update(e);
|
|
135
|
+
})));
|
|
136
|
+
if (yield* n(e, "body", l, {
|
|
137
|
+
contentType: L.contentType,
|
|
138
|
+
totalSize: L.size
|
|
139
|
+
}), a) {
|
|
140
|
+
let e = yield* P.head(L.key);
|
|
141
|
+
if (e === null || e.size !== L.size || !o(e.etag)) return yield* b("integrity");
|
|
142
|
+
}
|
|
143
|
+
let m = s.digest("hex"), g = N?.identity ?? {
|
|
144
|
+
tenantId: d.tenantId,
|
|
145
|
+
bucket: d.bucket,
|
|
146
|
+
operationId: d.operationId,
|
|
147
|
+
checksum: m,
|
|
148
|
+
logicalKey: h(m),
|
|
149
|
+
retainedSource: {
|
|
150
|
+
refId: L.id,
|
|
151
|
+
digest: R
|
|
152
|
+
},
|
|
153
|
+
fingerprint: x({
|
|
154
|
+
kind: "retained-storage-input/v1",
|
|
155
|
+
logicalKey: h(m),
|
|
156
|
+
sourceDigest: R,
|
|
157
|
+
checksum: m,
|
|
158
|
+
size: L.size,
|
|
159
|
+
contentType: L.contentType
|
|
160
|
+
})
|
|
161
|
+
};
|
|
162
|
+
if (r.publicUrl?.(g.logicalKey) != null) return yield* b("private-storage");
|
|
163
|
+
let _ = N?.ref ?? {
|
|
164
|
+
id: "reserved",
|
|
165
|
+
tenantId: d.tenantId,
|
|
166
|
+
bucket: d.bucket,
|
|
167
|
+
key: g.logicalKey,
|
|
168
|
+
checksum: m,
|
|
169
|
+
ownerId: null,
|
|
170
|
+
contentType: L.contentType,
|
|
171
|
+
size: L.size,
|
|
172
|
+
visibility: "private",
|
|
173
|
+
accessPolicy: null,
|
|
174
|
+
passwordHash: null,
|
|
175
|
+
createdAt: /* @__PURE__ */ new Date()
|
|
176
|
+
}, v = f.unwrap(t(e, "body").pipe(u.map((e) => e.stream)));
|
|
177
|
+
return E(g, (yield* M.putStream({
|
|
178
|
+
identity: g,
|
|
179
|
+
ref: _,
|
|
180
|
+
stream: v,
|
|
181
|
+
claim: d.claim,
|
|
182
|
+
...d.quota === void 0 ? {} : { quota: d.quota }
|
|
183
|
+
})).ref);
|
|
184
|
+
}).pipe(u.ensuring(u.promise(() => _(z, {
|
|
185
|
+
recursive: !0,
|
|
186
|
+
force: !0
|
|
187
|
+
}))));
|
|
188
|
+
});
|
|
189
|
+
export { y as RetainedObjectUnavailable, k as readRetainedStorageObject, A as retainStorageObject, D as retainedPinMatches };
|