@voltro/runtime 0.86.0 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5321 -0
- package/README.md +43 -0
- package/THIRD-PARTY-NOTICES.md +860 -221
- package/dist/analyticsSink.js +8 -14
- package/dist/apiKeys.js +82 -78
- package/dist/apiRoleBucketSampler.js +64 -0
- package/dist/apiRoleWorkloadHistory.js +56 -0
- package/dist/apiRoleWorkloadMetrics.js +78 -0
- package/dist/apiRoleWorkloadStorage.js +95 -0
- package/dist/approvals.js +67 -74
- package/dist/authMiddlewareLayer.js +7 -6
- package/dist/bindEvent.js +48 -28
- package/dist/bootListener.js +106 -39
- package/dist/bootStoreCodec.js +38 -37
- package/dist/circuitBreaker.js +1 -72
- package/dist/composeAnalytics.js +31 -10
- package/dist/computedQueryCache.js +231 -0
- package/dist/connectionsFacade.js +32 -30
- package/dist/coordinatedSchedule.js +84 -78
- package/dist/dataStore.js +154 -121
- package/dist/declaredTablesGuard.js +2 -1
- package/dist/dispatcher.js +403 -328
- package/dist/effectStore.js +2 -2
- package/dist/engineClients.js +24 -0
- package/dist/externalizedTasks.js +19 -6
- package/dist/fieldCipher.js +109 -40
- package/dist/fieldKeyRegistry.js +295 -0
- package/dist/fieldKeyRekey.js +286 -0
- package/dist/fieldKeyScan.js +136 -0
- package/dist/fieldKeyring.js +77 -0
- package/dist/hot.d.ts +56 -0
- package/dist/hot.js +3 -0
- package/dist/hotDataStore.js +148 -0
- package/dist/hotDataStoreSnapshot.js +98 -0
- package/dist/hotDataStoreTables.js +18 -0
- package/dist/http-range.d.ts +53 -0
- package/dist/http-range.js +52 -0
- package/dist/httpClientSsrf.js +12 -37
- package/dist/httpClientTracing.js +12 -9
- package/dist/httpCompression.js +17 -17
- package/dist/httpInputStream.js +82 -0
- package/dist/httpRouteMounts.js +1 -0
- package/dist/idempotencyStore.js +92 -52
- package/dist/index.d.ts +1164 -451
- package/dist/index.js +143 -132
- package/dist/kvFacade.js +23 -17
- package/dist/memoryQueryJoins.js +16 -16
- package/dist/memorySetOperations.js +43 -0
- package/dist/metrics.js +38 -24
- package/dist/outbox.js +1 -1
- package/dist/readRecorder.js +35 -34
- package/dist/readScopePredicate.js +12 -0
- package/dist/replicatedDataStore.js +22 -4
- package/dist/rowFilter.js +56 -39
- package/dist/rpcConnections.js +8 -8
- package/dist/rpcServer.js +522 -449
- package/dist/rpcWorkloadAdmission.js +50 -0
- package/dist/rywPositionStore.js +82 -47
- package/dist/scheduler.js +6 -1
- package/dist/servePipeline.js +215 -146
- package/dist/setReadScopes.js +28 -0
- package/dist/snapshotValueCodec.js +46 -0
- package/dist/sourceResolution.js +34 -24
- package/dist/storeMiddleware.js +211 -209
- package/dist/subscriber.js +1 -0
- package/dist/tenantScope.js +8 -20
- package/dist/traceparent.js +8 -19
- package/dist/webSocketGatewayRuntime.js +111 -71
- package/dist/workflowRuntime.js +9 -9
- package/package.json +19 -7
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
import { ENVELOPE_PATTERN as e, cellText as t, declaredEncryptedColumns as n, declaredRenames as r, discoverSealedColumns as i, reasonOf as a } from "./fieldKeyScan.js";
|
|
2
|
+
import { Effect as o } from "effect";
|
|
3
|
+
import { ciphertextKeyId as s, fieldBinding as c, quoteIdent as l } from "@voltro/database";
|
|
4
|
+
import { SqlClient as u } from "effect/sql";
|
|
5
|
+
import { STEP_JOURNAL_BINDING as d, WORKFLOW_JOURNAL_BINDING as f } from "@voltro/workflow";
|
|
6
|
+
//#region src/fieldKeyRekey.ts
|
|
7
|
+
var p = (e) => {
|
|
8
|
+
let t = e.renames?.get(e.table), n = t?.columns.get(e.column);
|
|
9
|
+
return {
|
|
10
|
+
tables: t?.previousTable === void 0 ? [e.table] : [e.table, t.previousTable],
|
|
11
|
+
columns: n === void 0 ? [e.column] : [e.column, n]
|
|
12
|
+
};
|
|
13
|
+
}, m = (e) => {
|
|
14
|
+
let t = p(e);
|
|
15
|
+
return t.tables.length > 1 || t.columns.length > 1;
|
|
16
|
+
}, h = 32, g = (e, t) => {
|
|
17
|
+
typeof e == "string" && e !== "" && e.length <= 191 && !e.startsWith("enc:") ? t.add(e) : (typeof e == "number" || typeof e == "bigint") && t.add(String(e));
|
|
18
|
+
}, _ = (e) => {
|
|
19
|
+
let t = /* @__PURE__ */ new Map(), n;
|
|
20
|
+
try {
|
|
21
|
+
n = JSON.parse(e);
|
|
22
|
+
} catch {
|
|
23
|
+
return t;
|
|
24
|
+
}
|
|
25
|
+
let r = (e, n, i) => {
|
|
26
|
+
if (typeof e == "string") {
|
|
27
|
+
if (!e.startsWith("enc:")) return;
|
|
28
|
+
let r = /* @__PURE__ */ new Set();
|
|
29
|
+
for (let e of i) for (let t of Object.values(e)) g(t, r);
|
|
30
|
+
t.set(e, {
|
|
31
|
+
key: n,
|
|
32
|
+
ids: r
|
|
33
|
+
});
|
|
34
|
+
} else if (Array.isArray(e)) for (let t of e) r(t, n, i);
|
|
35
|
+
else if (typeof e == "object" && e) {
|
|
36
|
+
let t = e;
|
|
37
|
+
for (let [e, n] of Object.entries(t)) r(n, e, [t, ...i]);
|
|
38
|
+
}
|
|
39
|
+
};
|
|
40
|
+
return r(n, void 0, []), t;
|
|
41
|
+
}, v = (e, t, n) => {
|
|
42
|
+
if (!e.startsWith("enc:v2:")) return [void 0];
|
|
43
|
+
let r = n.rowId === void 0 || n.rowId === null ? void 0 : String(n.rowId);
|
|
44
|
+
if (e.startsWith("enc:v2:c:")) {
|
|
45
|
+
let e = p(n), t = [
|
|
46
|
+
...e.tables.flatMap((t) => e.columns.map((e) => c(t, e))),
|
|
47
|
+
f,
|
|
48
|
+
d,
|
|
49
|
+
...[...n.declared].flatMap(([e, t]) => [...t].map((t) => c(e, t)))
|
|
50
|
+
];
|
|
51
|
+
return [...new Map(t.map((e) => [e.column, e])).values()];
|
|
52
|
+
}
|
|
53
|
+
let i = [], a = /* @__PURE__ */ new Set(), o = (e, t, n) => {
|
|
54
|
+
if (n === void 0) return;
|
|
55
|
+
let r = c(e, t, n);
|
|
56
|
+
r.row === void 0 || a.has(r.row) || (a.add(r.row), i.push(r));
|
|
57
|
+
}, s = p(n);
|
|
58
|
+
for (let e of s.tables) for (let t of s.columns) o(e, t, r);
|
|
59
|
+
let l = /* @__PURE__ */ new Set();
|
|
60
|
+
for (let e of t?.ids ?? []) l.add(e);
|
|
61
|
+
for (let e of Object.values(n.row)) g(e, l);
|
|
62
|
+
let u = [...l].slice(0, h), m = t?.key ?? n.column;
|
|
63
|
+
for (let [e, t] of n.declared) if (t.has(m)) for (let t of u) o(e, m, t);
|
|
64
|
+
return i;
|
|
65
|
+
}, y = (t, n, r) => {
|
|
66
|
+
let i = n.activeKeyId, a = [...new Set(t.match(e) ?? [])];
|
|
67
|
+
if (a.length === 0) return {
|
|
68
|
+
text: t,
|
|
69
|
+
rekeyed: 0,
|
|
70
|
+
current: 0,
|
|
71
|
+
unresolved: 0
|
|
72
|
+
};
|
|
73
|
+
let o = _(t), l = t, u = 0, d = 0, f = 0, h = m(r), g = (e) => {
|
|
74
|
+
if (e === void 0 || !h) return e;
|
|
75
|
+
let t = p(r);
|
|
76
|
+
return !t.tables.some((n) => t.columns.some((t) => e.column === `${n}:${t}`)) || e.column === `${r.table}:${r.column}` ? e : e.row === void 0 ? c(r.table, r.column) : c(r.table, r.column, r.rowId);
|
|
77
|
+
};
|
|
78
|
+
for (let e of a) {
|
|
79
|
+
let a = t.split(e).length - 1;
|
|
80
|
+
if (s(e) === i && !h) {
|
|
81
|
+
d += a;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
let c, p = !1;
|
|
85
|
+
for (let t of v(e, o.get(e), r)) {
|
|
86
|
+
let r;
|
|
87
|
+
try {
|
|
88
|
+
r = n.decrypt(e, t);
|
|
89
|
+
} catch {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
let a = g(t);
|
|
93
|
+
if (s(e) === i && a === t) {
|
|
94
|
+
p = !0;
|
|
95
|
+
break;
|
|
96
|
+
}
|
|
97
|
+
c = n.sealWith(i, r, a);
|
|
98
|
+
break;
|
|
99
|
+
}
|
|
100
|
+
p ? d += a : c === void 0 ? f += a : (l = l.split(e).join(c), u += a);
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
text: l,
|
|
104
|
+
rekeyed: u,
|
|
105
|
+
current: d,
|
|
106
|
+
unresolved: f
|
|
107
|
+
};
|
|
108
|
+
}, b = (t, n, r) => {
|
|
109
|
+
let i = _(t), a = 0, o = 0;
|
|
110
|
+
for (let s of t.match(e) ?? []) v(s, i.get(s), r).some((e) => {
|
|
111
|
+
try {
|
|
112
|
+
return n.decrypt(s, e), !0;
|
|
113
|
+
} catch {
|
|
114
|
+
return !1;
|
|
115
|
+
}
|
|
116
|
+
}) ? a += 1 : o += 1;
|
|
117
|
+
return {
|
|
118
|
+
opened: a,
|
|
119
|
+
unopened: o
|
|
120
|
+
};
|
|
121
|
+
}, x = 200, S = (e, t, n, r, i) => e === "postgres" ? r`CAST(${r(t)} AS TEXT) = ${i}` : e === "mysql" || e === "mariadb" ? n === "json" ? r`CAST(${r(t)} AS CHAR) = ${i}` : r`CAST(${r(t)} AS BINARY) = CAST(${i} AS BINARY)` : e === "mssql" ? r`CAST(${r(t)} AS VARBINARY(MAX)) = CAST(${i} AS VARBINARY(MAX))` : r`${r(t)} = ${i}`, C = (e, t, s, c = {}) => o.gen(function* () {
|
|
122
|
+
let l = yield* u.SqlClient, d = n(s), f = r(s), { columns: p, unreadable: m } = yield* i(e, d), h = [...m], g = [];
|
|
123
|
+
for (let n of p) {
|
|
124
|
+
if (n.found === 0) continue;
|
|
125
|
+
let r = yield* w(l, e, t, d, f, n, c.dryRun === !0, c.beforeWrite).pipe(o.catch((e) => o.sync(() => {
|
|
126
|
+
h.push({
|
|
127
|
+
table: n.table,
|
|
128
|
+
column: n.column,
|
|
129
|
+
reason: a(e)
|
|
130
|
+
});
|
|
131
|
+
})));
|
|
132
|
+
r !== void 0 && (g.push(r), c.onProgress?.(r));
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
active: t.activeKeyId,
|
|
136
|
+
locations: g,
|
|
137
|
+
unreadable: h
|
|
138
|
+
};
|
|
139
|
+
}), w = (e, n, r, i, a, s, c, u) => o.gen(function* () {
|
|
140
|
+
let d = {
|
|
141
|
+
cells: 0,
|
|
142
|
+
rekeyed: 0,
|
|
143
|
+
current: 0,
|
|
144
|
+
unresolved: 0,
|
|
145
|
+
changedMeanwhile: 0,
|
|
146
|
+
unaddressable: 0
|
|
147
|
+
}, f = s.key, p = n === "postgres" && s.type === "json" ? (yield* e.unsafe(`SELECT data_type AS t FROM information_schema.columns WHERE table_name = '${s.table.replaceAll("'", "''")}' AND column_name = '${s.column.replaceAll("'", "''")}' AND table_schema = current_schema()`).unprepared)[0]?.t : void 0, m = (e) => typeof e == "number" || typeof e == "bigint" ? String(e) : `'${String(e).replaceAll("'", "''")}'`, h = (r, i, a) => o.gen(function* () {
|
|
148
|
+
u !== void 0 && (yield* u({
|
|
149
|
+
table: s.table,
|
|
150
|
+
column: s.column,
|
|
151
|
+
rowId: r
|
|
152
|
+
}));
|
|
153
|
+
let o = p === "jsonb" ? e`CAST(${a} AS JSONB)` : p === "json" ? e`CAST(${a} AS JSON)` : e`${a}`;
|
|
154
|
+
yield* e`UPDATE ${e(s.table)} SET ${e(s.column)} = ${o} WHERE ${e(f)} = ${r} AND ${S(n, s.column, s.type, e, i)}`;
|
|
155
|
+
let c = (yield* e.unsafe(`SELECT ${s.asText} AS v FROM ${l(s.table, n)} WHERE ${l(f, n)} = ${m(r)}`).unprepared)[0]?.v;
|
|
156
|
+
return t(c) === a;
|
|
157
|
+
}), g = (r) => o.gen(function* () {
|
|
158
|
+
if (u !== void 0) for (let e of r) yield* u({
|
|
159
|
+
table: s.table,
|
|
160
|
+
column: s.column,
|
|
161
|
+
rowId: e.rowId
|
|
162
|
+
});
|
|
163
|
+
let i = (t) => p === "jsonb" ? e`CAST(${t} AS JSONB)` : p === "json" ? e`CAST(${t} AS JSON)` : e`${t}`, a = e.join(" ", !1)(r.map((t) => e`WHEN ${t.rowId} THEN ${i(t.next)}`)), o = e.join(" ", !1)(r.map((t) => e`WHEN ${e(f)} = ${t.rowId} AND ${S(n, s.column, s.type, e, t.text)} THEN 1`));
|
|
164
|
+
yield* e`UPDATE ${e(s.table)} SET ${e(s.column)} = CASE ${e(f)} ${a} END WHERE ${e(f)} IN ${e.in(r.map((e) => e.rowId))} AND (CASE ${o} ELSE 0 END) = 1`;
|
|
165
|
+
let c = yield* e.unsafe(`SELECT ${s.asText} AS v, ${l(f, n)} AS k FROM ${l(s.table, n)} WHERE ${l(f, n)} IN (${r.map((e) => m(e.rowId)).join(", ")})`).unprepared, d = new Map(c.map((e) => [String(e.k), t(e.v)]));
|
|
166
|
+
return r.map((e) => d.get(String(e.rowId)) === e.next);
|
|
167
|
+
}), _;
|
|
168
|
+
for (;;) {
|
|
169
|
+
let n = yield* e.unsafe(s.page(_, x)).unprepared, l = [];
|
|
170
|
+
for (let e of n) {
|
|
171
|
+
let n = t(e.__v), { __v: o, __k: u, ...p } = e, m = y(n, r, {
|
|
172
|
+
table: s.table,
|
|
173
|
+
column: s.column,
|
|
174
|
+
rowId: u,
|
|
175
|
+
row: p,
|
|
176
|
+
declared: i,
|
|
177
|
+
renames: a
|
|
178
|
+
});
|
|
179
|
+
d.cells += 1, d.current += m.current, d.unresolved += m.unresolved, m.rekeyed !== 0 && (f === void 0 ? d.unaddressable += m.rekeyed : c ? d.rekeyed += m.rekeyed : l.push({
|
|
180
|
+
rowId: u,
|
|
181
|
+
text: n,
|
|
182
|
+
next: m.text,
|
|
183
|
+
rekeyed: m.rekeyed
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
if (l.length > 0) {
|
|
187
|
+
let e = yield* g(l).pipe(o.catch(() => o.forEach(l, (e) => h(e.rowId, e.text, e.next))));
|
|
188
|
+
l.forEach((t, n) => {
|
|
189
|
+
e[n] === !0 ? d.rekeyed += t.rekeyed : d.changedMeanwhile += 1;
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
if (f === void 0 || n.length < x) break;
|
|
193
|
+
_ = n[n.length - 1].__k;
|
|
194
|
+
}
|
|
195
|
+
return {
|
|
196
|
+
table: s.table,
|
|
197
|
+
column: s.column,
|
|
198
|
+
...d
|
|
199
|
+
};
|
|
200
|
+
}), T = (e, a, s) => o.gen(function* () {
|
|
201
|
+
let o = yield* u.SqlClient, c = n(s), l = r(s), { columns: d, unreadable: f } = yield* i(e, c), p = [], m = 0;
|
|
202
|
+
for (let e of d) {
|
|
203
|
+
if (e.found === 0) continue;
|
|
204
|
+
let n = 0, r;
|
|
205
|
+
for (;;) {
|
|
206
|
+
let i = yield* o.unsafe(e.page(r, x)).unprepared;
|
|
207
|
+
for (let r of i) {
|
|
208
|
+
let { __v: i, __k: o, ...s } = r, u = b(t(i), a, {
|
|
209
|
+
table: e.table,
|
|
210
|
+
column: e.column,
|
|
211
|
+
rowId: o,
|
|
212
|
+
row: s,
|
|
213
|
+
declared: c,
|
|
214
|
+
renames: l
|
|
215
|
+
});
|
|
216
|
+
m += u.opened, n += u.unopened;
|
|
217
|
+
}
|
|
218
|
+
if (e.key === void 0 || i.length < x) break;
|
|
219
|
+
r = i[i.length - 1].__k;
|
|
220
|
+
}
|
|
221
|
+
n > 0 && p.push({
|
|
222
|
+
table: e.table,
|
|
223
|
+
column: e.column,
|
|
224
|
+
count: n
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
return {
|
|
228
|
+
opened: m,
|
|
229
|
+
unopened: p,
|
|
230
|
+
unreadable: f
|
|
231
|
+
};
|
|
232
|
+
}), E = (t) => {
|
|
233
|
+
let n = t.match(e);
|
|
234
|
+
return n !== null && n.length === 1 && n[0] === t;
|
|
235
|
+
}, D = (a, s, c) => o.gen(function* () {
|
|
236
|
+
let o = yield* u.SqlClient, d = n(c), f = r(c), { columns: p, unreadable: m, tables: h } = yield* i(a, d), g = [], _ = [], v = (e, t) => s !== void 0 && b(e, s, t).unopened === 0;
|
|
237
|
+
for (let n of p) {
|
|
238
|
+
if (n.found === 0) continue;
|
|
239
|
+
let r = 0, i = 0, c;
|
|
240
|
+
for (;;) {
|
|
241
|
+
let l = yield* o.unsafe(n.page(c, x)).unprepared;
|
|
242
|
+
for (let c of l) {
|
|
243
|
+
let l = t(c.__v), { __v: u, __k: p, ...m } = c, h = {
|
|
244
|
+
table: n.table,
|
|
245
|
+
column: n.column,
|
|
246
|
+
rowId: p,
|
|
247
|
+
row: m,
|
|
248
|
+
declared: d,
|
|
249
|
+
renames: f
|
|
250
|
+
};
|
|
251
|
+
if (E(l)) {
|
|
252
|
+
if (v(l, h)) continue;
|
|
253
|
+
if (n.key === void 0) {
|
|
254
|
+
i += 1;
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
yield* o`UPDATE ${o(n.table)} SET ${o(n.column)} = NULL WHERE ${o(n.key)} = ${p} AND ${S(a, n.column, n.type, o, l)}`, r += 1;
|
|
258
|
+
} else i += s === void 0 ? (l.match(e) ?? []).length : b(l, s, h).unopened;
|
|
259
|
+
}
|
|
260
|
+
if (n.key === void 0 || l.length < x) break;
|
|
261
|
+
c = l[l.length - 1].__k;
|
|
262
|
+
}
|
|
263
|
+
r > 0 && g.push({
|
|
264
|
+
table: n.table,
|
|
265
|
+
column: n.column,
|
|
266
|
+
count: r
|
|
267
|
+
}), i > 0 && _.push({
|
|
268
|
+
table: n.table,
|
|
269
|
+
column: n.column,
|
|
270
|
+
count: i
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
let y = new Set(s?.keyIds ?? []), C = (e) => l(e, a), w = [];
|
|
274
|
+
if (h.has("_voltro_field_keys")) {
|
|
275
|
+
w = (yield* o.unsafe(`SELECT ${C("keyId")} AS k FROM ${C("_voltro_field_keys")}`).unprepared).map((e) => String(e.k)).filter((e) => !y.has(e));
|
|
276
|
+
for (let e of w) yield* o`DELETE FROM ${o("_voltro_field_keys")} WHERE ${o("keyId")} = ${e}`;
|
|
277
|
+
}
|
|
278
|
+
return h.has("_voltro_field_key_holders") && (yield* o.unsafe(`DELETE FROM ${C("_voltro_field_key_holders")}`).unprepared), {
|
|
279
|
+
nulled: g,
|
|
280
|
+
unreadable: _,
|
|
281
|
+
forgottenKeys: w,
|
|
282
|
+
unreadableColumns: m
|
|
283
|
+
};
|
|
284
|
+
});
|
|
285
|
+
//#endregion
|
|
286
|
+
export { D as nullUnreadableSealedValues, b as openCell, y as rekeyCell, C as rekeySealedValues, T as verifySealedValues };
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { FIELD_KEYS_TABLE as e, FIELD_KEY_HOLDERS_TABLE as t } from "./fieldKeyRegistry.js";
|
|
2
|
+
import { Effect as n } from "effect";
|
|
3
|
+
import { ciphertextKeyId as r, encryptedColumnsOf as i, quoteIdent as a } from "@voltro/database";
|
|
4
|
+
import { SqlClient as o } from "effect/sql";
|
|
5
|
+
import { introspectSchema as s } from "@voltro/database/sql";
|
|
6
|
+
//#region src/fieldKeyScan.ts
|
|
7
|
+
var c = /enc:v[12]:[A-Za-z0-9+/=:_-]+/g, l = 500, u = /* @__PURE__ */ new Set(["text", "json"]), d = (e, t, n) => e === "postgres" ? `CAST(${t} AS TEXT)` : n === "json" && (e === "mysql" || e === "mariadb") ? `CAST(${t} AS CHAR)` : t, f = (e) => {
|
|
8
|
+
let t = e instanceof Error ? e.message : String(e);
|
|
9
|
+
for (let n = e?.cause; n instanceof Error; n = n.cause) t = n.message;
|
|
10
|
+
return t;
|
|
11
|
+
}, p = (e) => typeof e == "number" || typeof e == "bigint" ? String(e) : `'${String(e).replaceAll("'", "''")}'`, m = (e) => typeof e == "string" ? e : e instanceof Uint8Array ? new TextDecoder().decode(e) : e == null ? "" : JSON.stringify(e), h = (e, t) => n.gen(function* () {
|
|
12
|
+
let r = yield* o.SqlClient, i = (t) => a(t, e), c = yield* s(r), l = [], m = [];
|
|
13
|
+
for (let a of c.tables) {
|
|
14
|
+
if (a.name === "_voltro_field_keys" || a.name === "_voltro_field_key_holders") continue;
|
|
15
|
+
let o = a.indexes.find((e) => e.name === `${a.name}_pkey`)?.columns, s = o?.length === 1 ? o[0] : void 0;
|
|
16
|
+
for (let o of a.columns) {
|
|
17
|
+
let c = t.get(a.name)?.has(o.name) === !0;
|
|
18
|
+
if (!c && !u.has(o.type)) continue;
|
|
19
|
+
let h = d(e, `${i("__t")}.${i(o.name)}`, o.type), g = `${h} LIKE '%enc:v%'`, _ = `${i(a.name)} ${i("__t")}`, v = yield* r.unsafe(`SELECT COUNT(*) AS n FROM ${_} WHERE ${g}`).unprepared.pipe(n.map((e) => Number(e[0]?.n ?? 0)), n.catch((e) => n.sync(() => {
|
|
20
|
+
m.push({
|
|
21
|
+
table: a.name,
|
|
22
|
+
column: o.name,
|
|
23
|
+
reason: f(e)
|
|
24
|
+
});
|
|
25
|
+
})));
|
|
26
|
+
v === void 0 || v === 0 && !c || l.push({
|
|
27
|
+
table: a.name,
|
|
28
|
+
column: o.name,
|
|
29
|
+
type: o.type,
|
|
30
|
+
declared: c,
|
|
31
|
+
key: s,
|
|
32
|
+
found: v,
|
|
33
|
+
asText: d(e, i(o.name), o.type),
|
|
34
|
+
page: (t, n) => {
|
|
35
|
+
if (s === void 0) return `SELECT ${h} AS ${i("__v")}, ${i("__t")}.* FROM ${_} WHERE ${g}`;
|
|
36
|
+
let r = `${i("__t")}.${i(s)} AS ${i("__k")}, ${h} AS ${i("__v")}, ${i("__t")}.*`, a = `WHERE ${g}${t === void 0 ? "" : ` AND ${i("__t")}.${i(s)} > ${p(t)}`} ORDER BY ${i("__t")}.${i(s)}`;
|
|
37
|
+
return e === "mssql" ? `SELECT TOP ${n} ${r} FROM ${_} ${a}` : `SELECT ${r} FROM ${_} ${a} LIMIT ${n}`;
|
|
38
|
+
}
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
columns: l,
|
|
44
|
+
unreadable: m,
|
|
45
|
+
tables: new Set(c.tables.map((e) => e.name))
|
|
46
|
+
};
|
|
47
|
+
}), g = (e) => {
|
|
48
|
+
let t = /* @__PURE__ */ new Map();
|
|
49
|
+
for (let n of e) {
|
|
50
|
+
let e = n;
|
|
51
|
+
if (typeof e != "object" || !e || typeof e.fields != "object" || e.tableName === void 0) continue;
|
|
52
|
+
let r = i(e);
|
|
53
|
+
r.length > 0 && t.set(e.tableName, new Set(r));
|
|
54
|
+
}
|
|
55
|
+
return t;
|
|
56
|
+
}, _ = (e) => {
|
|
57
|
+
let t = /* @__PURE__ */ new Map();
|
|
58
|
+
for (let n of e) {
|
|
59
|
+
let e = n;
|
|
60
|
+
if (typeof e != "object" || !e || typeof e.fields != "object" || e.tableName === void 0) continue;
|
|
61
|
+
let r = /* @__PURE__ */ new Map();
|
|
62
|
+
for (let t of i(e)) {
|
|
63
|
+
let n = e.fields[t]?.renamedFrom;
|
|
64
|
+
n !== void 0 && r.set(t, n);
|
|
65
|
+
}
|
|
66
|
+
(e.previousTableName !== void 0 || r.size > 0) && t.set(e.tableName, {
|
|
67
|
+
...e.previousTableName === void 0 ? {} : { previousTable: e.previousTableName },
|
|
68
|
+
columns: r
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return t;
|
|
72
|
+
}, v = (e, t, n = 1) => {
|
|
73
|
+
e[t] = (e[t] ?? 0) + n;
|
|
74
|
+
}, y = (i, s) => n.gen(function* () {
|
|
75
|
+
let u = yield* o.SqlClient, d = (e) => a(e, i), p = (e) => u.unsafe(e).unprepared, { columns: _, unreadable: y, tables: b } = yield* h(i, g(s)), x = [...y], S = [];
|
|
76
|
+
for (let e of _) {
|
|
77
|
+
let t = yield* n.gen(function* () {
|
|
78
|
+
let t = e.declared ? Number((yield* p(`SELECT COUNT(*) AS n FROM ${d(e.table)} WHERE ${d(e.column)} IS NOT NULL AND ${e.asText} NOT LIKE 'enc:v%'`))[0]?.n ?? 0) : 0, n = {}, i = 0, a = 0, o = (e) => {
|
|
79
|
+
let t = m(e).match(c) ?? [];
|
|
80
|
+
t.length > 0 && (a += 1);
|
|
81
|
+
for (let e of t) {
|
|
82
|
+
let t = r(e);
|
|
83
|
+
t === void 0 ? i += 1 : v(n, t);
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
if (e.found > 0) {
|
|
87
|
+
let t;
|
|
88
|
+
for (;;) {
|
|
89
|
+
let n = yield* p(e.page(t, l));
|
|
90
|
+
for (let e of n) o(e.__v);
|
|
91
|
+
if (e.key === void 0 || n.length < l) break;
|
|
92
|
+
t = n[n.length - 1].__k;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
table: e.table,
|
|
97
|
+
column: e.column,
|
|
98
|
+
declared: e.declared,
|
|
99
|
+
cells: a,
|
|
100
|
+
byKey: n,
|
|
101
|
+
unnamed: i,
|
|
102
|
+
plaintext: t
|
|
103
|
+
};
|
|
104
|
+
}).pipe(n.catch((t) => n.sync(() => {
|
|
105
|
+
x.push({
|
|
106
|
+
table: e.table,
|
|
107
|
+
column: e.column,
|
|
108
|
+
reason: f(t)
|
|
109
|
+
});
|
|
110
|
+
})));
|
|
111
|
+
t !== void 0 && S.push(t);
|
|
112
|
+
}
|
|
113
|
+
let C = (e) => e == null ? null : e instanceof Date ? e.toISOString() : String(e), w = b.has("_voltro_field_keys") ? (yield* p(`SELECT ${d("keyId")} AS k, ${d("state")} AS s, ${d("activatedAt")} AS a, ${d("requestedAt")} AS r FROM ${d(e)}`)).map((e) => ({
|
|
114
|
+
keyId: String(e.k),
|
|
115
|
+
state: String(e.s),
|
|
116
|
+
activatedAt: C(e.a),
|
|
117
|
+
requestedAt: C(e.r)
|
|
118
|
+
})).sort((e, t) => e.keyId.localeCompare(t.keyId)) : [], T = b.has("_voltro_field_key_holders") ? (yield* p(`SELECT ${d("holderId")} AS h, ${d("keyIds")} AS k, ${d("activeKeyId")} AS a, ${d("expiresAt")} AS e FROM ${d(t)}`)).map((e) => ({
|
|
119
|
+
holderId: String(e.h),
|
|
120
|
+
keyIds: String(e.k).split(","),
|
|
121
|
+
activeKeyId: String(e.a),
|
|
122
|
+
expiresAt: C(e.e) ?? ""
|
|
123
|
+
})).sort((e, t) => e.holderId.localeCompare(t.holderId)) : [], E = {};
|
|
124
|
+
for (let e of S) for (let [t, n] of Object.entries(e.byKey)) v(E, t, n);
|
|
125
|
+
return S.sort((e, t) => e.table.localeCompare(t.table) || e.column.localeCompare(t.column)), {
|
|
126
|
+
keys: w,
|
|
127
|
+
holders: T,
|
|
128
|
+
locations: S,
|
|
129
|
+
totals: E,
|
|
130
|
+
unreadable: x,
|
|
131
|
+
unnamed: S.reduce((e, t) => e + t.unnamed, 0),
|
|
132
|
+
plaintext: S.reduce((e, t) => e + t.plaintext, 0)
|
|
133
|
+
};
|
|
134
|
+
});
|
|
135
|
+
//#endregion
|
|
136
|
+
export { c as ENVELOPE_PATTERN, m as cellText, g as declaredEncryptedColumns, _ as declaredRenames, h as discoverSealedColumns, y as fieldKeyStatus, f as reasonOf };
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { FieldKeyringConfigError as e, deriveKey as t, fieldKeyIdOf as n } from "./fieldCipher.js";
|
|
2
|
+
//#region src/fieldKeyring.ts
|
|
3
|
+
var r = "VOLTRO_FIELD_ENCRYPTION_KEYS", i = "VOLTRO_FIELD_ENCRYPTION_ACTIVE", a = "VOLTRO_FIELD_ENCRYPTION_KEY", o = "VOLTRO_FIELD_ENCRYPTION_KEY_PREVIOUS", s = [
|
|
4
|
+
r,
|
|
5
|
+
i,
|
|
6
|
+
a,
|
|
7
|
+
o
|
|
8
|
+
], c = (e) => e !== void 0 && e.trim() !== "", l = (t) => {
|
|
9
|
+
let n = t.trim();
|
|
10
|
+
if (n.startsWith("{")) {
|
|
11
|
+
let t;
|
|
12
|
+
try {
|
|
13
|
+
t = JSON.parse(n);
|
|
14
|
+
} catch {
|
|
15
|
+
throw new e(`${r} starts like JSON but is not valid JSON`);
|
|
16
|
+
}
|
|
17
|
+
if (typeof t != "object" || !t || Array.isArray(t)) throw new e(`${r} as JSON must be an object of id → key`);
|
|
18
|
+
return Object.entries(t).map(([t, n]) => {
|
|
19
|
+
if (typeof n != "string") throw new e(`${r}: key "${t}" is not a string`);
|
|
20
|
+
return {
|
|
21
|
+
id: t,
|
|
22
|
+
secret: n
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
return n.split(",").map((e) => e.trim()).filter((e) => e !== "").map((t) => {
|
|
27
|
+
let n = t.indexOf("=");
|
|
28
|
+
if (n <= 0) throw new e(`${r}: every entry is id=key, and one is not`);
|
|
29
|
+
return {
|
|
30
|
+
id: t.slice(0, n).trim(),
|
|
31
|
+
secret: t.slice(n + 1).trim()
|
|
32
|
+
};
|
|
33
|
+
});
|
|
34
|
+
}, u = (e, n) => t(e).equals(t(n)), d = (t) => {
|
|
35
|
+
let s = t[r], d = t[a], p = t[o], m = t[i]?.trim();
|
|
36
|
+
if (c(s)) {
|
|
37
|
+
let t = l(s);
|
|
38
|
+
for (let [n, i] of [[a, d], [o, p]]) if (c(i) && !t.some((e) => u(e.secret, i))) throw new e(`${n} is set beside ${r} but is not one of its keys — put it into ${r} or remove it`);
|
|
39
|
+
if (t.length === 0) throw new e(`${r} is set but holds no key`);
|
|
40
|
+
let n = c(d) ? t.find((e) => u(e.secret, d))?.id : void 0, h = c(m) ? m : t.length === 1 ? t[0].id : n;
|
|
41
|
+
if (h === void 0) throw new e(`${r} holds ${t.length} keys: name the one new values are sealed under in ${i}`);
|
|
42
|
+
return f({
|
|
43
|
+
active: h,
|
|
44
|
+
keys: t
|
|
45
|
+
}), {
|
|
46
|
+
active: h,
|
|
47
|
+
keys: t
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
if (!c(d)) {
|
|
51
|
+
if (c(p)) throw new e(`${o} is set without ${a}`);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (c(p) && u(d, p)) throw new e(`${a} and ${o} are the same key — a rotation needs a NEW key in ${a}`);
|
|
55
|
+
let h = [{
|
|
56
|
+
id: n(d),
|
|
57
|
+
secret: d
|
|
58
|
+
}, ...c(p) ? [{
|
|
59
|
+
id: n(p),
|
|
60
|
+
secret: p
|
|
61
|
+
}] : []], g = {
|
|
62
|
+
active: h[0].id,
|
|
63
|
+
keys: h
|
|
64
|
+
};
|
|
65
|
+
return f(g), g;
|
|
66
|
+
}, f = (t) => {
|
|
67
|
+
let n = /* @__PURE__ */ new Set();
|
|
68
|
+
for (let r of t.keys) {
|
|
69
|
+
if (!/^[a-z0-9][a-z0-9-]{0,15}$/.test(r.id)) throw new e(`field encryption: "${r.id}" is not a key id — use a short lowercase slug (letters, digits, "-", at most 16)`);
|
|
70
|
+
if (n.has(r.id)) throw new e(`field encryption: key id "${r.id}" is configured twice`);
|
|
71
|
+
n.add(r.id);
|
|
72
|
+
}
|
|
73
|
+
for (let n = 0; n < t.keys.length; n++) for (let r = n + 1; r < t.keys.length; r++) if (u(t.keys[n].secret, t.keys[r].secret)) throw new e(`field encryption: keys "${t.keys[n].id}" and "${t.keys[r].id}" are the same key — a rotation needs a NEW key, and two ids for one key make every count per key wrong`);
|
|
74
|
+
if (!n.has(t.active)) throw new e(`field encryption: the active key "${t.active}" is not one of the configured keys (${[...n].join(", ")})`);
|
|
75
|
+
};
|
|
76
|
+
//#endregion
|
|
77
|
+
export { i as FIELD_ENCRYPTION_ACTIVE_ENV, r as FIELD_ENCRYPTION_KEYS_ENV, a as FIELD_ENCRYPTION_KEY_ENV, o as FIELD_ENCRYPTION_PREVIOUS_KEY_ENV, s as FIELD_KEYRING_ENV_NAMES, d as resolveFieldKeyring };
|
package/dist/hot.d.ts
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { ColumnBuilder } from '@voltro/database';
|
|
2
|
+
import { DataStore } from '@voltro/database';
|
|
3
|
+
import { DialectId } from '@voltro/database';
|
|
4
|
+
import { FieldDefinitions } from '@voltro/database';
|
|
5
|
+
import { HotStoreShape } from '@voltro/kv/hot/contract';
|
|
6
|
+
import { Table } from '@voltro/database';
|
|
7
|
+
|
|
8
|
+
/** Small-deployment SQL backend. One database lock serializes complete batches.
|
|
9
|
+
* The lock row contains no application key, tenant identity or payload. All
|
|
10
|
+
* application rows, including sorted members, are physically removed on delete.
|
|
11
|
+
* SQL stores own transaction retries; callbacks perform no external side effects.
|
|
12
|
+
*/
|
|
13
|
+
export declare const dataStoreHotStore: (store: DataStore, options: HotDataStoreOptions) => HotStoreShape;
|
|
14
|
+
|
|
15
|
+
export declare interface HotDataStoreOptions {
|
|
16
|
+
/** Actual SQL dialect: SQLite-family writer admission differs from row locks. */
|
|
17
|
+
readonly dialect: DialectId;
|
|
18
|
+
readonly namespace: string;
|
|
19
|
+
/** Trusted server/test time, sampled after acquiring the transaction lock. */
|
|
20
|
+
readonly now?: () => number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export declare const hotDataStoreTables: readonly [ Table<"_voltro_hot_kv", FieldDefinitions<{
|
|
24
|
+
readonly id: ColumnBuilder<string, "id", boolean>;
|
|
25
|
+
readonly keyHash: ColumnBuilder<string, "text", boolean>;
|
|
26
|
+
readonly namespace: ColumnBuilder<string | null, "text", boolean>;
|
|
27
|
+
readonly key: ColumnBuilder<string | null, "text", boolean>;
|
|
28
|
+
readonly payload: ColumnBuilder<string | null, "text", boolean>;
|
|
29
|
+
readonly expiresAtMs: ColumnBuilder<number | null, "real", boolean>;
|
|
30
|
+
}>, false, never>, Table<"_voltro_hot_z", FieldDefinitions<{
|
|
31
|
+
readonly id: ColumnBuilder<string, "id", boolean>;
|
|
32
|
+
readonly keyHash: ColumnBuilder<string, "text", boolean>;
|
|
33
|
+
readonly memberHash: ColumnBuilder<string, "text", boolean>;
|
|
34
|
+
readonly member: ColumnBuilder<string, "text", boolean>;
|
|
35
|
+
readonly score: ColumnBuilder<number, "real", boolean>;
|
|
36
|
+
}>, false, never>];
|
|
37
|
+
|
|
38
|
+
/** Activated with the database HotStore; importing definitions performs no I/O. */
|
|
39
|
+
export declare const hotKvTable: Table<"_voltro_hot_kv", FieldDefinitions<{
|
|
40
|
+
readonly id: ColumnBuilder<string, "id", boolean>;
|
|
41
|
+
readonly keyHash: ColumnBuilder<string, "text", boolean>;
|
|
42
|
+
readonly namespace: ColumnBuilder<string | null, "text", boolean>;
|
|
43
|
+
readonly key: ColumnBuilder<string | null, "text", boolean>;
|
|
44
|
+
readonly payload: ColumnBuilder<string | null, "text", boolean>;
|
|
45
|
+
readonly expiresAtMs: ColumnBuilder<number | null, "real", boolean>;
|
|
46
|
+
}>, false, never>;
|
|
47
|
+
|
|
48
|
+
export declare const hotZTable: Table<"_voltro_hot_z", FieldDefinitions<{
|
|
49
|
+
readonly id: ColumnBuilder<string, "id", boolean>;
|
|
50
|
+
readonly keyHash: ColumnBuilder<string, "text", boolean>;
|
|
51
|
+
readonly memberHash: ColumnBuilder<string, "text", boolean>;
|
|
52
|
+
readonly member: ColumnBuilder<string, "text", boolean>;
|
|
53
|
+
readonly score: ColumnBuilder<number, "real", boolean>;
|
|
54
|
+
}>, false, never>;
|
|
55
|
+
|
|
56
|
+
export { }
|
package/dist/hot.js
ADDED