@voltro/data-transfer 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -0
- package/THIRD-PARTY-NOTICES.md +5 -33
- package/dist/assets.js +47 -0
- package/dist/blobStorage.js +40 -0
- package/dist/bundleArchive.js +165 -0
- package/dist/bundleCrypto.js +106 -0
- package/dist/casAssetPhase.js +43 -0
- package/dist/compress.js +6 -0
- package/dist/consistency.js +14 -0
- package/dist/crossDialect.js +58 -0
- package/dist/driftClassification.js +58 -0
- package/dist/environmentLocal.js +122 -0
- package/dist/errors.js +82 -0
- package/dist/exporter.js +257 -0
- package/dist/importer.js +536 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +28 -2673
- package/dist/ledger.js +21 -0
- package/dist/liveGuard.js +11 -0
- package/dist/manifest.js +52 -0
- package/dist/masking.js +246 -0
- package/dist/maskingPreview.js +48 -0
- package/dist/nativeDump.js +369 -0
- package/dist/ndjson.js +13 -0
- package/dist/profile.js +4 -0
- package/dist/progress.js +17 -0
- package/dist/readme.js +43 -0
- package/dist/replacePreflight.js +23 -0
- package/dist/rollbackCapture.js +36 -0
- package/dist/rowFailureReason.js +36 -0
- package/dist/schemaDrift.js +40 -0
- package/dist/scope.js +40 -0
- package/dist/stagedSwap.js +164 -0
- package/dist/subset.js +90 -0
- package/dist/topo.js +24 -0
- package/dist/typedCodec.js +36 -0
- package/package.json +3 -3
package/THIRD-PARTY-NOTICES.md
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# Third-Party Notices — @voltro/data-transfer
|
|
2
2
|
|
|
3
|
-
This package
|
|
3
|
+
This package uses the open-source software listed below. Each is the
|
|
4
4
|
property of its respective copyright holders and is used under the terms of
|
|
5
5
|
its license. This file is provided for attribution; it grants no rights in
|
|
6
6
|
@voltro/data-transfer itself, which is proprietary (see LICENSE).
|
|
7
7
|
|
|
8
|
-
Generated from the resolved runtime dependency closure (
|
|
8
|
+
Generated from the resolved runtime dependency closure (108 packages).
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
@@ -9624,34 +9624,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
|
9624
9624
|
SOFTWARE.
|
|
9625
9625
|
```
|
|
9626
9626
|
|
|
9627
|
-
## layerr@3.0.0
|
|
9628
|
-
|
|
9629
|
-
License: MIT
|
|
9630
|
-
|
|
9631
|
-
```
|
|
9632
|
-
MIT License
|
|
9633
|
-
|
|
9634
|
-
Copyright (c) 2020 Perry Mitchell
|
|
9635
|
-
|
|
9636
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
9637
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
9638
|
-
in the Software without restriction, including without limitation the rights
|
|
9639
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9640
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
9641
|
-
furnished to do so, subject to the following conditions:
|
|
9642
|
-
|
|
9643
|
-
The above copyright notice and this permission notice shall be included in all
|
|
9644
|
-
copies or substantial portions of the Software.
|
|
9645
|
-
|
|
9646
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
9647
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
9648
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
9649
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
9650
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
9651
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
9652
|
-
SOFTWARE.
|
|
9653
|
-
```
|
|
9654
|
-
|
|
9655
9627
|
## lib0@0.2.117
|
|
9656
9628
|
|
|
9657
9629
|
License: MIT
|
|
@@ -10459,14 +10431,14 @@ Apache License
|
|
|
10459
10431
|
limitations under the License.
|
|
10460
10432
|
```
|
|
10461
10433
|
|
|
10462
|
-
##
|
|
10434
|
+
## ulid@3.0.2
|
|
10463
10435
|
|
|
10464
10436
|
License: MIT
|
|
10465
10437
|
|
|
10466
10438
|
```
|
|
10467
|
-
MIT License
|
|
10439
|
+
The MIT License (MIT)
|
|
10468
10440
|
|
|
10469
|
-
Copyright (c)
|
|
10441
|
+
Copyright (c) 2017 Alizain Feerasta
|
|
10470
10442
|
|
|
10471
10443
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10472
10444
|
of this software and associated documentation files (the "Software"), to deal
|
package/dist/assets.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { BundleError as e, IntegrityError as t } from "./errors.js";
|
|
2
|
+
import { Effect as n, Stream as r } from "effect";
|
|
3
|
+
import { fromNodeReadable as i, getObjectStream as a, putObjectStream as o, toNodeReadable as s } from "@voltro/plugin-storage";
|
|
4
|
+
//#region src/assets.ts
|
|
5
|
+
var c = (e, t) => ({
|
|
6
|
+
list: t,
|
|
7
|
+
open: (t) => a(e, t)
|
|
8
|
+
}), l = (e) => ({
|
|
9
|
+
write: (t, n, r) => o(e, t, n, r),
|
|
10
|
+
has: (t) => e.head(t).pipe(n.map((e) => e !== null))
|
|
11
|
+
}), u = (t, r) => n.tryPromise({
|
|
12
|
+
try: async () => {
|
|
13
|
+
let { createWriteStream: e } = await import("node:fs"), n = await import("node:fs/promises"), { createHash: i, randomUUID: a } = await import("node:crypto"), { pipeline: o } = await import("node:stream/promises"), { Transform: c } = await import("node:stream"), l = await import("node:path"), u = l.join(r, `.tmp-${a()}`), d = i("sha256"), f = 0, p = new c({ transform(e, t, n) {
|
|
14
|
+
d.update(e), f += e.length, n(null, e);
|
|
15
|
+
} });
|
|
16
|
+
await o(await s(t.stream), p, e(u));
|
|
17
|
+
let m = d.digest("hex"), h = l.join(r, m);
|
|
18
|
+
try {
|
|
19
|
+
await n.access(h), await n.rm(u, { force: !0 });
|
|
20
|
+
} catch {
|
|
21
|
+
await n.rename(u, h);
|
|
22
|
+
}
|
|
23
|
+
return {
|
|
24
|
+
sha256: m,
|
|
25
|
+
size: f,
|
|
26
|
+
contentType: t.contentType
|
|
27
|
+
};
|
|
28
|
+
},
|
|
29
|
+
catch: (t) => new e({
|
|
30
|
+
reason: `asset write failed: ${String(t?.message ?? t)}`,
|
|
31
|
+
path: r
|
|
32
|
+
})
|
|
33
|
+
}), d = (e, a, o) => n.gen(function* () {
|
|
34
|
+
let s = yield* n.promise(() => import("node:path")), { createReadStream: c } = yield* n.promise(() => import("node:fs")), { createHash: l } = yield* n.promise(() => import("node:crypto")), u = s.join(a, o.sha256), d = l("sha256"), f = i("bundle", () => c(u)).pipe(r.tap((e) => n.sync(() => d.update(e))));
|
|
35
|
+
yield* e.write(o.key, f, {
|
|
36
|
+
contentType: o.contentType,
|
|
37
|
+
totalSize: o.size
|
|
38
|
+
});
|
|
39
|
+
let p = d.digest("hex");
|
|
40
|
+
if (p !== o.sha256) return yield* n.fail(new t({
|
|
41
|
+
what: `asset ${o.key}`,
|
|
42
|
+
expected: o.sha256,
|
|
43
|
+
actual: p
|
|
44
|
+
}));
|
|
45
|
+
});
|
|
46
|
+
//#endregion
|
|
47
|
+
export { d as restoreAssetFromCas, l as storageAssetSink, c as storageAssetSource, u as writeAssetToCas };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { Effect as e, Stream as t } from "effect";
|
|
2
|
+
import { streamTable as n } from "@voltro/database";
|
|
3
|
+
import { STORAGE_REFS_TABLE as r, fromNodeReadable as i, getObjectStream as a, putObjectStream as o, toNodeReadable as s } from "@voltro/plugin-storage";
|
|
4
|
+
import { Readable as c } from "node:stream";
|
|
5
|
+
//#region src/blobStorage.ts
|
|
6
|
+
var l = (n, r) => ({
|
|
7
|
+
list: () => {
|
|
8
|
+
let n = r();
|
|
9
|
+
return { async *[Symbol.asyncIterator]() {
|
|
10
|
+
let r = await e.runPromise(t.runCollect(n));
|
|
11
|
+
for (let e of r) yield {
|
|
12
|
+
key: e.key,
|
|
13
|
+
contentType: e.contentType,
|
|
14
|
+
size: e.size
|
|
15
|
+
};
|
|
16
|
+
} };
|
|
17
|
+
},
|
|
18
|
+
open: (t) => ({ async *[Symbol.asyncIterator]() {
|
|
19
|
+
let r = await e.runPromise(a(n, t)), i = await s(r.stream);
|
|
20
|
+
for await (let e of i) yield e instanceof Uint8Array ? e : new Uint8Array(e);
|
|
21
|
+
} })
|
|
22
|
+
}), u = (e, i) => l(e, () => n(i, {
|
|
23
|
+
table: r,
|
|
24
|
+
chunkSize: 500
|
|
25
|
+
}).pipe(t.map((e) => ({
|
|
26
|
+
key: String(e.key),
|
|
27
|
+
contentType: String(e.contentType ?? "application/octet-stream"),
|
|
28
|
+
size: Number(e.size ?? 0)
|
|
29
|
+
})))), d = (t) => ({
|
|
30
|
+
has: (n) => e.runPromise(t.head(n).pipe(e.map((e) => e !== null))),
|
|
31
|
+
write: async (n, r, a) => {
|
|
32
|
+
let s = i("storage", () => c.from(r));
|
|
33
|
+
await e.runPromise(o(t, n, s, {
|
|
34
|
+
contentType: a.contentType,
|
|
35
|
+
totalSize: a.size
|
|
36
|
+
}));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
//#endregion
|
|
40
|
+
export { d as storageBlobSink, l as storageBlobSource, u as storageRefsBlobSource };
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { createHash as e } from "node:crypto";
|
|
2
|
+
import { createReadStream as t, createWriteStream as n } from "node:fs";
|
|
3
|
+
import { PassThrough as r } from "node:stream";
|
|
4
|
+
import { mkdir as i, readdir as a, stat as o } from "node:fs/promises";
|
|
5
|
+
import { dirname as s, join as c, resolve as l, sep as u } from "node:path";
|
|
6
|
+
//#region src/bundleArchive.ts
|
|
7
|
+
var d = Buffer.from("VBUNDLE2"), f = 0, p = 1, m = 2, h = 32, g = async (e) => {
|
|
8
|
+
let t = [], n = async (r) => {
|
|
9
|
+
let i;
|
|
10
|
+
try {
|
|
11
|
+
i = await a(c(e, r), { withFileTypes: !0 });
|
|
12
|
+
} catch {
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
for (let e of i) {
|
|
16
|
+
let i = r ? `${r}/${e.name}` : e.name;
|
|
17
|
+
e.isDirectory() ? await n(i) : e.isFile() && t.push(i);
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
return await n(""), t.sort();
|
|
21
|
+
}, _ = (e, t) => {
|
|
22
|
+
let n = Buffer.from(e, "utf8"), r = Buffer.alloc(5 + n.length + 8);
|
|
23
|
+
return r.writeUInt8(p, 0), r.writeUInt32BE(n.length, 1), n.copy(r, 5), r.writeBigUInt64BE(BigInt(t), 5 + n.length), r;
|
|
24
|
+
}, v = (e, t, n) => {
|
|
25
|
+
let r = Buffer.from(e, "utf8"), i = Buffer.from(t, "utf8"), a = Buffer.alloc(5 + r.length + 2 + i.length + 8), o = 0;
|
|
26
|
+
return a.writeUInt8(m, o), o += 1, a.writeUInt32BE(r.length, o), o += 4, r.copy(a, o), o += r.length, a.writeUInt16BE(i.length, o), o += 2, i.copy(a, o), o += i.length, a.writeBigUInt64BE(BigInt(n), o), a;
|
|
27
|
+
};
|
|
28
|
+
async function* y(n) {
|
|
29
|
+
yield new Uint8Array(d);
|
|
30
|
+
for (let e of await g(n.dir)) {
|
|
31
|
+
let r = c(n.dir, e), i = await o(r);
|
|
32
|
+
if (yield new Uint8Array(_(e, i.size)), i.size > 0) for await (let e of t(r)) yield new Uint8Array(e);
|
|
33
|
+
}
|
|
34
|
+
if (n.blobs) for await (let t of n.blobs.list()) {
|
|
35
|
+
yield new Uint8Array(v(t.key, t.contentType, t.size));
|
|
36
|
+
let r = e("sha256"), i = 0;
|
|
37
|
+
for await (let e of n.blobs.open(t.key)) {
|
|
38
|
+
let t = Buffer.from(e);
|
|
39
|
+
r.update(t), i += t.length, yield new Uint8Array(t);
|
|
40
|
+
}
|
|
41
|
+
if (i !== t.size) throw Error(`blob '${t.key}' size mismatch: header ${t.size}, streamed ${i}`);
|
|
42
|
+
yield new Uint8Array(r.digest());
|
|
43
|
+
}
|
|
44
|
+
yield new Uint8Array(Buffer.from([f]));
|
|
45
|
+
}
|
|
46
|
+
var b = (e, t) => new Promise((n, r) => e.write(t, (e) => e ? r(e) : n())), x = (e) => new Promise((t) => e.end(t)), S = async (t, a) => {
|
|
47
|
+
let o = l(a.dir);
|
|
48
|
+
await i(o, { recursive: !0 });
|
|
49
|
+
let c = Buffer.alloc(0), g = t[Symbol.asyncIterator](), _ = !1, v = async () => {
|
|
50
|
+
let { value: e, done: t } = await g.next();
|
|
51
|
+
return t ? (_ = !0, !1) : (c = c.length === 0 ? Buffer.from(e) : Buffer.concat([c, Buffer.from(e)]), !0);
|
|
52
|
+
}, y = async (e) => {
|
|
53
|
+
for (; c.length < e;) if (!await v()) throw Error("truncated bundle archive");
|
|
54
|
+
}, S = (e) => {
|
|
55
|
+
let t = c.subarray(0, e);
|
|
56
|
+
return c = c.subarray(e), t;
|
|
57
|
+
};
|
|
58
|
+
try {
|
|
59
|
+
if (await y(d.length), !S(d.length).equals(d)) throw Error("not a bundle archive (bad magic)");
|
|
60
|
+
for (;;) {
|
|
61
|
+
await y(1);
|
|
62
|
+
let t = S(1).readUInt8(0);
|
|
63
|
+
if (t === f) return;
|
|
64
|
+
if (t === p) {
|
|
65
|
+
await y(4);
|
|
66
|
+
let e = S(4).readUInt32BE(0);
|
|
67
|
+
await y(e);
|
|
68
|
+
let t = S(e).toString("utf8");
|
|
69
|
+
await y(8);
|
|
70
|
+
let r = S(8).readBigUInt64BE(0), a = l(o, t);
|
|
71
|
+
if (a !== o && !a.startsWith(o + u)) throw Error(`archive path '${t}' escapes the destination`);
|
|
72
|
+
await i(s(a), { recursive: !0 });
|
|
73
|
+
let d = n(a);
|
|
74
|
+
for (; r > 0n;) {
|
|
75
|
+
c.length === 0 && await y(1);
|
|
76
|
+
let e = r < BigInt(c.length) ? Number(r) : c.length;
|
|
77
|
+
await b(d, S(e)), r -= BigInt(e);
|
|
78
|
+
}
|
|
79
|
+
await x(d);
|
|
80
|
+
} else if (t === m) {
|
|
81
|
+
await y(4);
|
|
82
|
+
let t = S(4).readUInt32BE(0);
|
|
83
|
+
await y(t);
|
|
84
|
+
let n = S(t).toString("utf8");
|
|
85
|
+
await y(2);
|
|
86
|
+
let i = S(2).readUInt16BE(0);
|
|
87
|
+
await y(i);
|
|
88
|
+
let o = S(i).toString("utf8");
|
|
89
|
+
await y(8);
|
|
90
|
+
let s = S(8).readBigUInt64BE(0), l = await a.blobs.has(n), u = e("sha256"), d = new r(), f = l ? Promise.resolve() : a.blobs.write(n, d, {
|
|
91
|
+
contentType: o,
|
|
92
|
+
size: Number(s)
|
|
93
|
+
}), p = s;
|
|
94
|
+
for (; p > 0n;) {
|
|
95
|
+
c.length === 0 && await y(1);
|
|
96
|
+
let e = p < BigInt(c.length) ? Number(p) : c.length, t = S(e);
|
|
97
|
+
u.update(t), l || await b(d, t), p -= BigInt(e);
|
|
98
|
+
}
|
|
99
|
+
l ? d.destroy() : await x(d), await f, await y(h);
|
|
100
|
+
let m = S(h);
|
|
101
|
+
if (!u.digest().equals(m)) throw Error(`blob '${n}' failed integrity (sha mismatch)`);
|
|
102
|
+
} else throw Error(`unknown bundle entry kind ${t}`);
|
|
103
|
+
}
|
|
104
|
+
} finally {
|
|
105
|
+
_ || await Promise.resolve(g.return?.()).catch(() => void 0);
|
|
106
|
+
}
|
|
107
|
+
}, C = async (e, t) => {
|
|
108
|
+
let n = new Set(t), r = /* @__PURE__ */ new Map(), i = Buffer.alloc(0), a = e[Symbol.asyncIterator](), o = async () => {
|
|
109
|
+
let { value: e, done: t } = await a.next();
|
|
110
|
+
return !t && (i = i.length === 0 ? Buffer.from(e) : Buffer.concat([i, Buffer.from(e)]), !0);
|
|
111
|
+
}, s = async (e) => {
|
|
112
|
+
for (; i.length < e;) if (!await o()) return !1;
|
|
113
|
+
return !0;
|
|
114
|
+
}, c = (e) => {
|
|
115
|
+
let t = i.subarray(0, e);
|
|
116
|
+
return i = i.subarray(e), t;
|
|
117
|
+
}, l = async (e) => {
|
|
118
|
+
let t = [], n = e;
|
|
119
|
+
for (; n > 0n;) {
|
|
120
|
+
if (i.length === 0 && !await s(1)) return null;
|
|
121
|
+
let e = n < BigInt(i.length) ? Number(n) : i.length;
|
|
122
|
+
t.push(c(e)), n -= BigInt(e);
|
|
123
|
+
}
|
|
124
|
+
return Buffer.concat(t);
|
|
125
|
+
};
|
|
126
|
+
if (!await s(d.length) || !c(d.length).equals(d)) throw Error("not a bundle archive (bad magic)");
|
|
127
|
+
for (; await s(1);) {
|
|
128
|
+
let e = c(1).readUInt8(0);
|
|
129
|
+
if (e === f || e === m) break;
|
|
130
|
+
if (e !== p) throw Error(`unknown bundle entry kind ${e}`);
|
|
131
|
+
if (!await s(4)) break;
|
|
132
|
+
let t = c(4).readUInt32BE(0);
|
|
133
|
+
if (!await s(t)) break;
|
|
134
|
+
let i = c(t).toString("utf8");
|
|
135
|
+
if (!await s(8)) break;
|
|
136
|
+
let a = await l(c(8).readBigUInt64BE(0));
|
|
137
|
+
if (a === null || n.has(i) && (r.set(i, a), n.delete(i), n.size === 0)) break;
|
|
138
|
+
}
|
|
139
|
+
return r;
|
|
140
|
+
}, w = (e, t) => S(e, {
|
|
141
|
+
dir: t,
|
|
142
|
+
blobs: T(t)
|
|
143
|
+
}), T = (t) => {
|
|
144
|
+
let r = /* @__PURE__ */ new Set();
|
|
145
|
+
return {
|
|
146
|
+
has: async (e) => r.has(e),
|
|
147
|
+
write: async (a, o, s) => {
|
|
148
|
+
let l = await import("node:fs/promises"), { pipeline: u } = await import("node:stream/promises"), { Transform: d } = await import("node:stream"), f = c(t, "assets");
|
|
149
|
+
await i(f, { recursive: !0 });
|
|
150
|
+
let p = e("sha256"), m = 0, h = new d({ transform(e, t, n) {
|
|
151
|
+
p.update(e), m += e.length, n(null, e);
|
|
152
|
+
} }), g = c(f, `.tmp-${Date.now()}-${r.size}`);
|
|
153
|
+
await u(o, h, n(g));
|
|
154
|
+
let _ = p.digest("hex");
|
|
155
|
+
await l.rename(g, c(f, _)), await l.appendFile(c(f, "index.ndjson"), `${JSON.stringify({
|
|
156
|
+
key: a,
|
|
157
|
+
sha256: _,
|
|
158
|
+
size: m,
|
|
159
|
+
contentType: s.contentType
|
|
160
|
+
})}\n`), r.add(a);
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
};
|
|
164
|
+
//#endregion
|
|
165
|
+
export { T as dirBlobSink, y as packBundle, C as peekBundle, S as unpackBundle, w as unpackBundleToDir };
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { createCipheriv as e, createDecipheriv as t, randomBytes as n, scrypt as r } from "node:crypto";
|
|
2
|
+
import { promisify as i } from "node:util";
|
|
3
|
+
//#region src/bundleCrypto.ts
|
|
4
|
+
var a = i(r), o = Buffer.from("VENC1\0\0\0"), s = 32, c = 16, l = 7, u = 16, d = 262144, f = {
|
|
5
|
+
N: 32768,
|
|
6
|
+
r: 8,
|
|
7
|
+
p: 1
|
|
8
|
+
}, p = 134217728, m = 43, h = (e, t, n) => {
|
|
9
|
+
let r = Buffer.alloc(12);
|
|
10
|
+
return e.copy(r, 0, 0, l), r.writeUInt32BE(t >>> 0, l), r.writeUInt8(+!!n, 11), r;
|
|
11
|
+
};
|
|
12
|
+
async function* g(t, r) {
|
|
13
|
+
if (!r) throw Error("encryptStream: a non-empty passphrase is required");
|
|
14
|
+
let i = n(c), g = n(l), _ = await a(r, i, s, {
|
|
15
|
+
...f,
|
|
16
|
+
maxmem: p
|
|
17
|
+
}), v = Buffer.alloc(m);
|
|
18
|
+
o.copy(v, 0), v.writeUInt32BE(f.N, 8), v.writeUInt32BE(f.r, 12), v.writeUInt32BE(f.p, 16), i.copy(v, 20), g.copy(v, 36), yield new Uint8Array(v);
|
|
19
|
+
let y = 0, b = (t, n) => {
|
|
20
|
+
let r = e("aes-256-gcm", _, h(g, y, n));
|
|
21
|
+
y === 0 && r.setAAD(v);
|
|
22
|
+
let i = Buffer.concat([r.update(t), r.final()]), a = r.getAuthTag();
|
|
23
|
+
y++;
|
|
24
|
+
let o = Buffer.alloc(5 + i.length + u);
|
|
25
|
+
return o.writeUInt8(+!!n, 0), o.writeUInt32BE(i.length + u, 1), i.copy(o, 5), a.copy(o, 5 + i.length), new Uint8Array(o);
|
|
26
|
+
}, x = Buffer.alloc(0), S = 0;
|
|
27
|
+
for await (let e of t) {
|
|
28
|
+
let t = Buffer.from(e), n = 0;
|
|
29
|
+
for (; n < t.length;) {
|
|
30
|
+
if (S === d && (yield b(x, !1), S = 0), S === 0 && t.length - n >= d) {
|
|
31
|
+
let e = t.subarray(n, n + d);
|
|
32
|
+
n += d, n < t.length ? yield b(e, !1) : (x = e, S = d);
|
|
33
|
+
continue;
|
|
34
|
+
}
|
|
35
|
+
x.length !== d && (x = Buffer.alloc(d));
|
|
36
|
+
let e = Math.min(d - S, t.length - n);
|
|
37
|
+
t.copy(x, S, n, n + e), S += e, n += e;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
yield b(x.subarray(0, S), !0);
|
|
41
|
+
}
|
|
42
|
+
async function* _(e, n) {
|
|
43
|
+
if (!n) throw Error("decryptStream: a non-empty passphrase is required");
|
|
44
|
+
let r = e[Symbol.asyncIterator](), i = Buffer.alloc(0), c = 0, l = !1, d = async (e) => {
|
|
45
|
+
let t, n = 0;
|
|
46
|
+
for (; n < e;) {
|
|
47
|
+
if (c === i.length) {
|
|
48
|
+
if (l) break;
|
|
49
|
+
let e = await r.next();
|
|
50
|
+
if (e.done) {
|
|
51
|
+
l = !0;
|
|
52
|
+
break;
|
|
53
|
+
}
|
|
54
|
+
if (i = Buffer.from(e.value), c = 0, i.length === 0) continue;
|
|
55
|
+
}
|
|
56
|
+
if (n === 0 && i.length - c >= e) {
|
|
57
|
+
let t = i.subarray(c, c + e);
|
|
58
|
+
return c += e, t;
|
|
59
|
+
}
|
|
60
|
+
t ??= Buffer.alloc(e);
|
|
61
|
+
let a = Math.min(e - n, i.length - c);
|
|
62
|
+
i.copy(t, n, c, c + a), c += a, n += a;
|
|
63
|
+
}
|
|
64
|
+
return t?.subarray(0, n) ?? Buffer.alloc(0);
|
|
65
|
+
}, f = !1;
|
|
66
|
+
try {
|
|
67
|
+
let e = await d(m);
|
|
68
|
+
if (e.length !== m) throw Error("encrypted bundle truncated (no header)");
|
|
69
|
+
if (!e.subarray(0, 8).equals(o)) throw Error("not an encrypted bundle (bad magic / wrong passphrase target)");
|
|
70
|
+
let r = e.readUInt32BE(8), i = e.readUInt32BE(12), c = e.readUInt32BE(16), l = Buffer.from(e.subarray(20, 36)), f = Buffer.from(e.subarray(36, m)), g = await a(n, l, s, {
|
|
71
|
+
N: r,
|
|
72
|
+
r: i,
|
|
73
|
+
p: c,
|
|
74
|
+
maxmem: p
|
|
75
|
+
}), _ = 0, v = !1;
|
|
76
|
+
for (;;) {
|
|
77
|
+
let n = await d(5);
|
|
78
|
+
if (n.length !== 5) {
|
|
79
|
+
if (n.length === 0 && v) return;
|
|
80
|
+
throw Error("encrypted bundle truncated (no authenticated final frame)");
|
|
81
|
+
}
|
|
82
|
+
if (v) throw Error("encrypted bundle has trailing data after the final frame");
|
|
83
|
+
let r = n.readUInt8(0);
|
|
84
|
+
if (r !== 0 && r !== 1) throw Error("encrypted bundle corrupt (invalid final flag)");
|
|
85
|
+
let i = r === 1, a = n.readUInt32BE(1);
|
|
86
|
+
if (a < u) throw Error("encrypted bundle corrupt (short frame)");
|
|
87
|
+
if (a > 262160) throw Error("encrypted bundle corrupt (oversized frame)");
|
|
88
|
+
let o = await d(a);
|
|
89
|
+
if (o.length !== a) throw Error("encrypted bundle truncated (frame body)");
|
|
90
|
+
let s = o.subarray(0, a - u), c = o.subarray(a - u), l = t("aes-256-gcm", g, h(f, _, i));
|
|
91
|
+
_ === 0 && l.setAAD(e), l.setAuthTag(c);
|
|
92
|
+
let p = Buffer.concat([l.update(s), l.final()]);
|
|
93
|
+
_++, i && (v = !0), p.length > 0 && (yield new Uint8Array(p));
|
|
94
|
+
}
|
|
95
|
+
} catch (e) {
|
|
96
|
+
throw f = !0, e;
|
|
97
|
+
} finally {
|
|
98
|
+
if (!l) try {
|
|
99
|
+
await r.return?.();
|
|
100
|
+
} catch (e) {
|
|
101
|
+
if (!f) throw e;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//#endregion
|
|
106
|
+
export { _ as decryptStream, g as encryptStream };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { BundleError as e } from "./errors.js";
|
|
2
|
+
import { restoreAssetFromCas as t, writeAssetToCas as n } from "./assets.js";
|
|
3
|
+
import { Effect as r, Stream as i } from "effect";
|
|
4
|
+
import { createReadStream as a } from "node:fs";
|
|
5
|
+
import { fromNodeReadable as o } from "@voltro/plugin-storage";
|
|
6
|
+
//#region src/casAssetPhase.ts
|
|
7
|
+
var s = {
|
|
8
|
+
done: () => !1,
|
|
9
|
+
mark: () => r.void
|
|
10
|
+
}, c = (e) => typeof e == "object" && !!e && e.status === 404, l = (t) => r.gen(function* () {
|
|
11
|
+
let a = t.progress ?? s, o = 0, l = 0, u = 0, d = 0, f = /* @__PURE__ */ new Set(), p = /* @__PURE__ */ new Set(), m = [];
|
|
12
|
+
return yield* i.runForEach(t.source.list(), (e) => (u++, a.done(e.key) || p.has(e.key) ? r.void : (p.add(e.key), t.source.open(e.key).pipe(r.map((e) => e), r.catchIf(c, () => r.sync(() => {
|
|
13
|
+
m.push(e.key);
|
|
14
|
+
})), r.flatMap((i) => i === void 0 ? r.void : n(i, t.assetsDir).pipe(r.flatMap((t) => {
|
|
15
|
+
let n = {
|
|
16
|
+
key: e.key,
|
|
17
|
+
sha256: t.sha256,
|
|
18
|
+
size: t.size,
|
|
19
|
+
contentType: t.contentType
|
|
20
|
+
};
|
|
21
|
+
return o++, l += t.size, f.has(t.sha256) || (f.add(t.sha256), d += t.size), a.mark(e.key, n);
|
|
22
|
+
}))))))).pipe(r.mapError((t) => t instanceof e ? t : new e({ reason: `asset export failed: ${String(t)}` }))), {
|
|
23
|
+
count: o,
|
|
24
|
+
totalBytes: l,
|
|
25
|
+
references: u,
|
|
26
|
+
objects: f.size,
|
|
27
|
+
objectBytes: d,
|
|
28
|
+
missing: m
|
|
29
|
+
};
|
|
30
|
+
}), u = (t) => o("bundle", () => a(t)).pipe(i.decodeText(), i.splitLines, i.filter((e) => e.trim().length > 0), i.map((e) => JSON.parse(e)), i.mapError((n) => n instanceof e ? n : new e({
|
|
31
|
+
reason: `asset index unreadable: ${String(n)}`,
|
|
32
|
+
path: t
|
|
33
|
+
}))), d = (e) => r.gen(function* () {
|
|
34
|
+
let n = e.progress ?? s, a = 0, o = /* @__PURE__ */ new Set(), c = /* @__PURE__ */ new Set();
|
|
35
|
+
return yield* i.runForEach(u(e.indexPath), (i) => n.done(i.key) || c.has(i.key) ? r.void : (c.add(i.key), t(e.sink, e.assetsDir, i)).pipe(r.andThen(r.sync(() => {
|
|
36
|
+
a++, o.add(i.sha256);
|
|
37
|
+
})), r.andThen(n.mark(i.key, i)))), {
|
|
38
|
+
count: a,
|
|
39
|
+
objects: o.size
|
|
40
|
+
};
|
|
41
|
+
});
|
|
42
|
+
//#endregion
|
|
43
|
+
export { l as captureAssetsToCas, s as noAssetProgress, u as readAssetIndex, d as restoreAssetsFromCas };
|
package/dist/compress.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import * as e from "node:zlib";
|
|
2
|
+
import { PassThrough as t } from "node:stream";
|
|
3
|
+
//#region src/compress.ts
|
|
4
|
+
var n = () => typeof e.createZstdCompress == "function", r = (e) => e === "none" ? "none" : e === "gzip" ? "gzip" : n() ? "zstd" : "gzip", i = (e) => e === "zstd" ? ".zst" : e === "gzip" ? ".gz" : "", a = (n) => n === "zstd" ? e.createZstdCompress() : n === "gzip" ? e.createGzip() : new t(), o = (n) => n === "zstd" ? e.createZstdDecompress() : n === "gzip" ? e.createGunzip() : new t();
|
|
5
|
+
//#endregion
|
|
6
|
+
export { i as compressionExt, n as hasZstd, a as makeCompressor, o as makeDecompressor, r as pickCompression };
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
//#region src/consistency.ts
|
|
2
|
+
var e = (e) => {
|
|
3
|
+
switch (e) {
|
|
4
|
+
case "postgres": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
5
|
+
case "mysql":
|
|
6
|
+
case "mariadb": return ["SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"];
|
|
7
|
+
case "mssql": return ["SET TRANSACTION ISOLATION LEVEL SNAPSHOT"];
|
|
8
|
+
case "sqlite":
|
|
9
|
+
case "turso": return [];
|
|
10
|
+
default: return [];
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
//#endregion
|
|
14
|
+
export { e as snapshotPrelude };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
//#region src/crossDialect.ts
|
|
2
|
+
var e = (e) => {
|
|
3
|
+
switch (e) {
|
|
4
|
+
case "id":
|
|
5
|
+
case "text":
|
|
6
|
+
case "enum": return e === "enum" ? "enum" : "string";
|
|
7
|
+
case "integer": return "integer";
|
|
8
|
+
case "real": return "float";
|
|
9
|
+
case "boolean": return "boolean";
|
|
10
|
+
case "timestamp": return "datetime";
|
|
11
|
+
case "date": return "date";
|
|
12
|
+
case "json": return "json";
|
|
13
|
+
case "bytes": return "binary";
|
|
14
|
+
case "reference": return "reference";
|
|
15
|
+
case "vector": return "vector";
|
|
16
|
+
case "array": return "array";
|
|
17
|
+
case "interval": return "interval";
|
|
18
|
+
case "raw": return "raw";
|
|
19
|
+
default: return "string";
|
|
20
|
+
}
|
|
21
|
+
}, t = {
|
|
22
|
+
vector: /* @__PURE__ */ new Set(["postgres"]),
|
|
23
|
+
array: /* @__PURE__ */ new Set(["postgres"]),
|
|
24
|
+
interval: /* @__PURE__ */ new Set(["postgres"])
|
|
25
|
+
}, n = (n, r, i, a) => {
|
|
26
|
+
let o = e(i);
|
|
27
|
+
if (o === "raw") return {
|
|
28
|
+
table: n,
|
|
29
|
+
column: r,
|
|
30
|
+
type: i,
|
|
31
|
+
severity: "block",
|
|
32
|
+
reason: `raw() is verbatim source-dialect SQL — it cannot be assumed valid on ${a}`
|
|
33
|
+
};
|
|
34
|
+
let s = t[o];
|
|
35
|
+
return !s || s.has(a) ? null : o === "array" || o === "interval" ? {
|
|
36
|
+
table: n,
|
|
37
|
+
column: r,
|
|
38
|
+
type: i,
|
|
39
|
+
severity: "warn",
|
|
40
|
+
reason: `${o} is native to postgres; ${a} will represent it as json/text — verify downstream reads`
|
|
41
|
+
} : {
|
|
42
|
+
table: n,
|
|
43
|
+
column: r,
|
|
44
|
+
type: i,
|
|
45
|
+
severity: "block",
|
|
46
|
+
reason: `${o} columns require a ${[...s].join("/")} target; ${a} has no compatible column type`
|
|
47
|
+
};
|
|
48
|
+
}, r = (e, t, r) => {
|
|
49
|
+
if (t === r) return [];
|
|
50
|
+
let i = [];
|
|
51
|
+
for (let t of e) for (let [e, a] of Object.entries(t.columnTypes)) {
|
|
52
|
+
let o = n(t.name, e, a, r);
|
|
53
|
+
o && i.push(o);
|
|
54
|
+
}
|
|
55
|
+
return i;
|
|
56
|
+
}, i = (e) => `[${e.severity}] ${e.table}.${e.column} (${e.type}): ${e.reason}`;
|
|
57
|
+
//#endregion
|
|
58
|
+
export { e as canonicalType, r as checkPortability, i as formatIssue };
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
//#region src/driftClassification.ts
|
|
2
|
+
var e = (e, t) => {
|
|
3
|
+
let n = new Map(t.map((e) => [e.name, e])), r = [], i = {};
|
|
4
|
+
for (let t of e) {
|
|
5
|
+
let e = n.get(t.name);
|
|
6
|
+
if (e === void 0) {
|
|
7
|
+
r.push({
|
|
8
|
+
verdict: "refuse",
|
|
9
|
+
table: t.name,
|
|
10
|
+
detail: `the bundle carries '${t.name}' and the target schema has no such table — nowhere to put ${t.name}'s rows`
|
|
11
|
+
});
|
|
12
|
+
continue;
|
|
13
|
+
}
|
|
14
|
+
let a = new Map(e.columns.map((e) => [e.name, e])), o = new Map(t.columns.map((e) => [e.name, e])), s = [];
|
|
15
|
+
for (let e of t.columns) {
|
|
16
|
+
let n = a.get(e.name);
|
|
17
|
+
n === void 0 ? (s.push(e.name), r.push({
|
|
18
|
+
verdict: "value-dropped",
|
|
19
|
+
table: t.name,
|
|
20
|
+
column: e.name,
|
|
21
|
+
detail: `'${t.name}.${e.name}' is in the bundle and not in the schema — its values are DISCARDED`
|
|
22
|
+
})) : n.type !== e.type && r.push({
|
|
23
|
+
verdict: "refuse",
|
|
24
|
+
table: t.name,
|
|
25
|
+
column: e.name,
|
|
26
|
+
detail: `'${t.name}.${e.name}' changed type (${e.type} → ${n.type}) — the bundle's values would be written through a codec that no longer matches them`
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
for (let n of e.columns) o.has(n.name) || (n.nullable || n.hasDefault ? r.push({
|
|
30
|
+
verdict: "filled-by-default",
|
|
31
|
+
table: t.name,
|
|
32
|
+
column: n.name,
|
|
33
|
+
detail: `'${t.name}.${n.name}' is in the schema and not in the bundle — every imported row gets ${n.hasDefault ? "its default" : "NULL"}`
|
|
34
|
+
}) : r.push({
|
|
35
|
+
verdict: "refuse",
|
|
36
|
+
table: t.name,
|
|
37
|
+
column: n.name,
|
|
38
|
+
detail: `'${t.name}.${n.name}' is NOT NULL with no default and the bundle has no value for it`
|
|
39
|
+
}));
|
|
40
|
+
s.length > 0 && (i[t.name] = s);
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
findings: r,
|
|
44
|
+
skipColumns: i,
|
|
45
|
+
refuse: r.some((e) => e.verdict === "refuse")
|
|
46
|
+
};
|
|
47
|
+
}, t = (e) => {
|
|
48
|
+
if (e.findings.length === 0) return [" schema matches the bundle"];
|
|
49
|
+
let t = {
|
|
50
|
+
safe: "✓",
|
|
51
|
+
"value-dropped": "⊖",
|
|
52
|
+
"filled-by-default": "⊕",
|
|
53
|
+
refuse: "✗"
|
|
54
|
+
}, n = e.findings.map((e) => ` ${t[e.verdict]} ${e.detail}`), r = (t) => e.findings.filter((e) => e.verdict === t).length;
|
|
55
|
+
return n.push(""), n.push(` discarded: ${r("value-dropped")} defaulted: ${r("filled-by-default")} refused: ${r("refuse")}`), e.refuse && (n.push(""), n.push(" REFUSED — no row was written. These are the differences that would make"), n.push(" imported rows WRONG rather than merely incomplete.")), n;
|
|
56
|
+
};
|
|
57
|
+
//#endregion
|
|
58
|
+
export { e as classifyImportDrift, t as formatImportDrift };
|