@effect-vfs/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +97 -8
- package/dist/BytePath.d.ts +21 -0
- package/dist/BytePath.d.ts.map +1 -0
- package/dist/BytePath.js +16 -0
- package/dist/Snapshot.d.ts +2 -2
- package/dist/Snapshot.js +2 -2
- package/dist/SnapshotDelta.d.ts +126 -0
- package/dist/SnapshotDelta.d.ts.map +1 -0
- package/dist/SnapshotDelta.js +112 -0
- package/dist/VirtualFileSystem.d.ts +105 -32
- package/dist/VirtualFileSystem.d.ts.map +1 -1
- package/dist/VirtualFileSystem.js +95 -45
- package/dist/internal/canonicalBase64.d.ts +7 -0
- package/dist/internal/canonicalBase64.d.ts.map +1 -0
- package/dist/internal/canonicalBase64.js +11 -0
- package/dist/internal/image.d.ts +7 -5
- package/dist/internal/image.d.ts.map +1 -1
- package/dist/internal/image.js +10 -14
- package/dist/internal/overlayChanges.d.ts.map +1 -1
- package/dist/internal/overlayChanges.js +4 -15
- package/dist/internal/snapshotDelta.d.ts +88 -0
- package/dist/internal/snapshotDelta.d.ts.map +1 -0
- package/dist/internal/snapshotDelta.js +589 -0
- package/package.json +5 -4
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/** Exact snapshot delta construction, inspection, serialization, and application. @internal */
|
|
2
|
+
import * as ByteSize from "effect/ByteSize";
|
|
3
|
+
import * as Crypto from "effect/Crypto";
|
|
4
|
+
import * as Effect from "effect/Effect";
|
|
5
|
+
import { ImageError, type Snapshot } from "../Snapshot.js";
|
|
6
|
+
import { type SnapshotDelta, SnapshotDeltaError } from "../SnapshotDelta.js";
|
|
7
|
+
/** @internal */
|
|
8
|
+
export declare const diffSnapshots: (base: Snapshot, target: Snapshot, limits: {
|
|
9
|
+
readonly maxEncodedBytes: ByteSize.ByteSize;
|
|
10
|
+
readonly maxIdentityBytes: ByteSize.ByteSize;
|
|
11
|
+
readonly maxDeltaRecords: number;
|
|
12
|
+
readonly maxDecodedDeltaBytes: ByteSize.ByteSize;
|
|
13
|
+
readonly maxBaseRecords: number;
|
|
14
|
+
readonly maxTargetRecords: number;
|
|
15
|
+
readonly maxEntries: number;
|
|
16
|
+
readonly maxOutputRecords: number;
|
|
17
|
+
readonly maxOutputBytes: ByteSize.ByteSize;
|
|
18
|
+
readonly maxInheritedRecords: number;
|
|
19
|
+
}) => Effect.Effect<SnapshotDelta, ImageError | import("effect/PlatformError").PlatformError, Crypto.Crypto>;
|
|
20
|
+
/** @internal */
|
|
21
|
+
export declare const inspectSnapshotDelta: (base: Snapshot, delta: SnapshotDelta, options: {
|
|
22
|
+
readonly includeTimestamps?: boolean;
|
|
23
|
+
}, limits: {
|
|
24
|
+
readonly maxEncodedBytes: ByteSize.ByteSize;
|
|
25
|
+
readonly maxIdentityBytes: ByteSize.ByteSize;
|
|
26
|
+
readonly maxDeltaRecords: number;
|
|
27
|
+
readonly maxDecodedDeltaBytes: ByteSize.ByteSize;
|
|
28
|
+
readonly maxBaseRecords: number;
|
|
29
|
+
readonly maxTargetRecords: number;
|
|
30
|
+
readonly maxEntries: number;
|
|
31
|
+
readonly maxOutputRecords: number;
|
|
32
|
+
readonly maxOutputBytes: ByteSize.ByteSize;
|
|
33
|
+
readonly maxInheritedRecords: number;
|
|
34
|
+
}) => Effect.Effect<readonly ({
|
|
35
|
+
readonly _tag: "Added";
|
|
36
|
+
readonly path: import("../BytePath.js").BytePath;
|
|
37
|
+
readonly kind: "directory" | "file" | "symlink";
|
|
38
|
+
} | {
|
|
39
|
+
readonly _tag: "Removed";
|
|
40
|
+
readonly path: import("../BytePath.js").BytePath;
|
|
41
|
+
readonly kind: "directory" | "file" | "symlink";
|
|
42
|
+
} | {
|
|
43
|
+
readonly _tag: "Updated";
|
|
44
|
+
readonly path: import("../BytePath.js").BytePath;
|
|
45
|
+
readonly beforeKind: "directory" | "file" | "symlink";
|
|
46
|
+
readonly afterKind: "directory" | "file" | "symlink";
|
|
47
|
+
readonly differences: readonly ("atimeNs" | "birthtimeNs" | "content" | "ctimeNs" | "gid" | "hardLinks" | "kind" | "mode" | "mtimeNs" | "target" | "uid")[];
|
|
48
|
+
})[], ImageError | import("effect/PlatformError").PlatformError | SnapshotDeltaError, Crypto.Crypto>;
|
|
49
|
+
/** @internal */
|
|
50
|
+
export declare const encodeSnapshotDelta: (delta: SnapshotDelta, limits: {
|
|
51
|
+
readonly maxEncodedBytes: ByteSize.ByteSize;
|
|
52
|
+
readonly maxIdentityBytes: ByteSize.ByteSize;
|
|
53
|
+
readonly maxDeltaRecords: number;
|
|
54
|
+
readonly maxDecodedDeltaBytes: ByteSize.ByteSize;
|
|
55
|
+
readonly maxBaseRecords: number;
|
|
56
|
+
readonly maxTargetRecords: number;
|
|
57
|
+
readonly maxEntries: number;
|
|
58
|
+
readonly maxOutputRecords: number;
|
|
59
|
+
readonly maxOutputBytes: ByteSize.ByteSize;
|
|
60
|
+
readonly maxInheritedRecords: number;
|
|
61
|
+
}) => Effect.Effect<Uint8Array<ArrayBuffer>, ImageError, never>;
|
|
62
|
+
/** @internal */
|
|
63
|
+
export declare const decodeSnapshotDelta: (input: Uint8Array<ArrayBufferLike>, limits: {
|
|
64
|
+
readonly maxEncodedBytes: ByteSize.ByteSize;
|
|
65
|
+
readonly maxIdentityBytes: ByteSize.ByteSize;
|
|
66
|
+
readonly maxDeltaRecords: number;
|
|
67
|
+
readonly maxDecodedDeltaBytes: ByteSize.ByteSize;
|
|
68
|
+
readonly maxBaseRecords: number;
|
|
69
|
+
readonly maxTargetRecords: number;
|
|
70
|
+
readonly maxEntries: number;
|
|
71
|
+
readonly maxOutputRecords: number;
|
|
72
|
+
readonly maxOutputBytes: ByteSize.ByteSize;
|
|
73
|
+
readonly maxInheritedRecords: number;
|
|
74
|
+
}) => Effect.Effect<SnapshotDelta, ImageError, never>;
|
|
75
|
+
/** @internal */
|
|
76
|
+
export declare const applySnapshotDelta: (base: Snapshot, delta: SnapshotDelta, limits: {
|
|
77
|
+
readonly maxEncodedBytes: ByteSize.ByteSize;
|
|
78
|
+
readonly maxIdentityBytes: ByteSize.ByteSize;
|
|
79
|
+
readonly maxDeltaRecords: number;
|
|
80
|
+
readonly maxDecodedDeltaBytes: ByteSize.ByteSize;
|
|
81
|
+
readonly maxBaseRecords: number;
|
|
82
|
+
readonly maxTargetRecords: number;
|
|
83
|
+
readonly maxEntries: number;
|
|
84
|
+
readonly maxOutputRecords: number;
|
|
85
|
+
readonly maxOutputBytes: ByteSize.ByteSize;
|
|
86
|
+
readonly maxInheritedRecords: number;
|
|
87
|
+
}) => Effect.Effect<Snapshot, ImageError | import("effect/PlatformError").PlatformError | SnapshotDeltaError, Crypto.Crypto>;
|
|
88
|
+
//# sourceMappingURL=snapshotDelta.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snapshotDelta.d.ts","sourceRoot":"","sources":["../../src/internal/snapshotDelta.ts"],"names":[],"mappings":"AAAA,+FAA+F;AAC/F,OAAO,KAAK,QAAQ,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,eAAe,CAAA;AAMvC,OAAO,EAAE,UAAU,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAC1D,OAAO,EAIL,KAAK,aAAa,EAClB,kBAAkB,EAKnB,MAAM,qBAAqB,CAAA;AA6d5B,gBAAgB;AAChB,eAAO,MAAM,aAAa;;;;;;;;;;;4GAgDzB,CAAA;AAwBD,gBAAgB;AAChB,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;oGAgBhC,CAAA;AACD,gBAAgB;AAChB,eAAO,MAAM,mBAAmB;;;;;;;;;;;+DAS9B,CAAA;AACF,gBAAgB;AAChB,eAAO,MAAM,mBAAmB;;;;;;;;;;;qDAkB9B,CAAA;AACF,gBAAgB;AAChB,eAAO,MAAM,kBAAkB;;;;;;;;;;;4HAI9B,CAAA"}
|
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
/** Exact snapshot delta construction, inspection, serialization, and application. @internal */
|
|
2
|
+
import * as ByteSize from "effect/ByteSize";
|
|
3
|
+
import * as Crypto from "effect/Crypto";
|
|
4
|
+
import * as Effect from "effect/Effect";
|
|
5
|
+
import * as Encoding from "effect/Encoding";
|
|
6
|
+
import * as Equal from "effect/Equal";
|
|
7
|
+
import * as Result from "effect/Result";
|
|
8
|
+
import * as Schema from "effect/Schema";
|
|
9
|
+
import { make as makeBytePath } from "../BytePath.js";
|
|
10
|
+
import { ImageError } from "../Snapshot.js";
|
|
11
|
+
import { makeSnapshotDelta, SnapshotDeltaError, snapshotDeltaValue } from "../SnapshotDelta.js";
|
|
12
|
+
import * as CanonicalBase64 from "./canonicalBase64.js";
|
|
13
|
+
import * as Image from "./image.js";
|
|
14
|
+
const Path = Schema.String;
|
|
15
|
+
const Payload = Schema.Union([
|
|
16
|
+
Schema.TaggedStruct("Inline", { bytes: Schema.String }),
|
|
17
|
+
Schema.TaggedStruct("Base", { path: Path })
|
|
18
|
+
]);
|
|
19
|
+
const Record = Schema.Union([
|
|
20
|
+
Schema.Struct({ kind: Schema.Literal("directory"), paths: Schema.Array(Path), metadata: Image.StoredMetadata }),
|
|
21
|
+
Schema.Struct({
|
|
22
|
+
kind: Schema.Literal("file"),
|
|
23
|
+
paths: Schema.Array(Path),
|
|
24
|
+
metadata: Image.StoredMetadata,
|
|
25
|
+
payload: Payload
|
|
26
|
+
}),
|
|
27
|
+
Schema.Struct({
|
|
28
|
+
kind: Schema.Literal("symlink"),
|
|
29
|
+
paths: Schema.Array(Path),
|
|
30
|
+
metadata: Image.StoredMetadata,
|
|
31
|
+
payload: Payload
|
|
32
|
+
})
|
|
33
|
+
]);
|
|
34
|
+
const Kind = Schema.Literals(["directory", "file", "symlink"]);
|
|
35
|
+
const Difference = Schema.Literals([
|
|
36
|
+
"kind",
|
|
37
|
+
"content",
|
|
38
|
+
"target",
|
|
39
|
+
"hardLinks",
|
|
40
|
+
"mode",
|
|
41
|
+
"uid",
|
|
42
|
+
"gid",
|
|
43
|
+
"atimeNs",
|
|
44
|
+
"mtimeNs",
|
|
45
|
+
"ctimeNs",
|
|
46
|
+
"birthtimeNs"
|
|
47
|
+
]);
|
|
48
|
+
const Change = Schema.Union([
|
|
49
|
+
Schema.TaggedStruct("Added", { path: Path, kind: Kind }),
|
|
50
|
+
Schema.TaggedStruct("Removed", { path: Path, kind: Kind }),
|
|
51
|
+
Schema.TaggedStruct("Updated", {
|
|
52
|
+
path: Path,
|
|
53
|
+
beforeKind: Kind,
|
|
54
|
+
afterKind: Kind,
|
|
55
|
+
differences: Schema.Array(Difference).check(Schema.isMinLength(1))
|
|
56
|
+
})
|
|
57
|
+
]);
|
|
58
|
+
const Document = Schema.Struct({
|
|
59
|
+
format: Schema.Literal("effect-vfs-delta"),
|
|
60
|
+
version: Schema.Literal(1),
|
|
61
|
+
base: Schema.Struct({ algorithm: Schema.Literal("effect-vfs-semantic-sha256-v1"), digest: Schema.String }),
|
|
62
|
+
records: Schema.Array(Record),
|
|
63
|
+
changes: Schema.Array(Change)
|
|
64
|
+
});
|
|
65
|
+
const timestampFields = new Set(["atimeNs", "mtimeNs", "ctimeNs", "birthtimeNs"]);
|
|
66
|
+
const differenceOrder = [
|
|
67
|
+
"kind",
|
|
68
|
+
"content",
|
|
69
|
+
"target",
|
|
70
|
+
"hardLinks",
|
|
71
|
+
"mode",
|
|
72
|
+
"uid",
|
|
73
|
+
"gid",
|
|
74
|
+
"atimeNs",
|
|
75
|
+
"mtimeNs",
|
|
76
|
+
"ctimeNs",
|
|
77
|
+
"birthtimeNs"
|
|
78
|
+
];
|
|
79
|
+
const encoder = new TextEncoder();
|
|
80
|
+
const Json = Schema.fromJsonString(Schema.Unknown);
|
|
81
|
+
const failure = (code, field) => new ImageError({ code, ...(field === undefined ? {} : { field }) });
|
|
82
|
+
const sameBytes = (a, b) => Equal.equals(a, b);
|
|
83
|
+
const compareBytes = (a, b) => {
|
|
84
|
+
for (let i = 0; i < Math.min(a.length, b.length); i++) {
|
|
85
|
+
const d = a[i] - b[i];
|
|
86
|
+
if (d !== 0)
|
|
87
|
+
return d;
|
|
88
|
+
}
|
|
89
|
+
return a.length - b.length;
|
|
90
|
+
};
|
|
91
|
+
const key = Encoding.encodeHex;
|
|
92
|
+
const join = (parent, name) => {
|
|
93
|
+
const out = new Uint8Array(parent.length + (parent.length === 1 ? 0 : 1) + name.length);
|
|
94
|
+
out.set(parent);
|
|
95
|
+
let offset = parent.length;
|
|
96
|
+
if (parent.length !== 1)
|
|
97
|
+
out[offset++] = 47;
|
|
98
|
+
out.set(name, offset);
|
|
99
|
+
return out;
|
|
100
|
+
};
|
|
101
|
+
const parentKey = (path) => {
|
|
102
|
+
let slash = path.length - 1;
|
|
103
|
+
while (slash > 0 && path[slash] !== 47)
|
|
104
|
+
slash--;
|
|
105
|
+
return key(slash === 0 ? new Uint8Array([47]) : path.subarray(0, slash));
|
|
106
|
+
};
|
|
107
|
+
const basename = (path) => {
|
|
108
|
+
let slash = path.length - 1;
|
|
109
|
+
while (slash > 0 && path[slash] !== 47)
|
|
110
|
+
slash--;
|
|
111
|
+
return path.subarray(slash + 1);
|
|
112
|
+
};
|
|
113
|
+
const decode64 = (s, field) => CanonicalBase64.isCanonical(s) ? Effect.succeed(Image.bytes(s)) : Effect.fail(failure("InvalidEncoding", field));
|
|
114
|
+
const safeAdd = (a, b) => {
|
|
115
|
+
const n = a + b;
|
|
116
|
+
return Number.isSafeInteger(n) ? n : undefined;
|
|
117
|
+
};
|
|
118
|
+
const encodedPayloadLength = (value) => value.length / 4 * 3 - (value.endsWith("==") ? 2 : value.endsWith("=") ? 1 : 0);
|
|
119
|
+
const exceedsByteLimit = (value, limit) => BigInt(value) > BigInt(limit);
|
|
120
|
+
const validPath = (path) => {
|
|
121
|
+
if (path.length === 0 || path[0] !== 47 || path.includes(0) || (path.length > 1 && path.at(-1) === 47))
|
|
122
|
+
return false;
|
|
123
|
+
if (path.length === 1)
|
|
124
|
+
return true;
|
|
125
|
+
let start = 1;
|
|
126
|
+
for (let i = 1; i <= path.length; i++) {
|
|
127
|
+
if (i === path.length || path[i] === 47) {
|
|
128
|
+
const length = i - start;
|
|
129
|
+
if (length < 1 || length > 255 || (length === 1 && path[start] === 46) ||
|
|
130
|
+
(length === 2 && path[start] === 46 && path[start + 1] === 46))
|
|
131
|
+
return false;
|
|
132
|
+
start = i + 1;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return true;
|
|
136
|
+
};
|
|
137
|
+
const normalize = Effect.fnUntraced(function* (snapshot, limits, role) {
|
|
138
|
+
const doc = yield* Image.inspect(snapshot);
|
|
139
|
+
if (doc.records.length > (role === "base" ? limits.maxBaseRecords : limits.maxTargetRecords)) {
|
|
140
|
+
return yield* failure("LimitExceeded", `${role}Records`);
|
|
141
|
+
}
|
|
142
|
+
const records = new Map(doc.records.map((record) => [record.id, record]));
|
|
143
|
+
const pathsById = new Map();
|
|
144
|
+
const pending = [[doc.root, new Uint8Array([47])]];
|
|
145
|
+
let entries = 0;
|
|
146
|
+
let pathBytes = 1;
|
|
147
|
+
for (let i = 0; i < pending.length; i++) {
|
|
148
|
+
const [recordId, path] = pending[i];
|
|
149
|
+
const record = records.get(recordId);
|
|
150
|
+
if (record === undefined)
|
|
151
|
+
return yield* failure("InvalidStructure", role);
|
|
152
|
+
const paths = pathsById.get(recordId);
|
|
153
|
+
if (paths === undefined)
|
|
154
|
+
pathsById.set(recordId, [path]);
|
|
155
|
+
else
|
|
156
|
+
paths.push(path);
|
|
157
|
+
if (record.kind === "directory") {
|
|
158
|
+
for (const entry of record.entries) {
|
|
159
|
+
if (++entries > limits.maxEntries)
|
|
160
|
+
return yield* failure("LimitExceeded", "entries");
|
|
161
|
+
const nameLength = encodedPayloadLength(entry.name);
|
|
162
|
+
const pathLength = safeAdd(path.length + (path.length === 1 ? 0 : 1), nameLength);
|
|
163
|
+
const nextPathBytes = pathLength === undefined ? undefined : safeAdd(pathBytes, pathLength);
|
|
164
|
+
const pathLimit = role === "base" ? limits.maxIdentityBytes : limits.maxDecodedDeltaBytes;
|
|
165
|
+
if (nextPathBytes === undefined || exceedsByteLimit(nextPathBytes, pathLimit)) {
|
|
166
|
+
return yield* failure("LimitExceeded", role === "base" ? "identityBytes" : "decodedDeltaBytes");
|
|
167
|
+
}
|
|
168
|
+
pathBytes = nextPathBytes;
|
|
169
|
+
pending.push([entry.target, join(path, Image.bytes(entry.name))]);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
const objects = [];
|
|
174
|
+
const byPath = new Map();
|
|
175
|
+
let payloadBytes = 0;
|
|
176
|
+
let basePayloadBytes = 0;
|
|
177
|
+
for (const record of doc.records) {
|
|
178
|
+
const paths = pathsById.get(record.id);
|
|
179
|
+
if (paths === undefined)
|
|
180
|
+
return yield* failure("InvalidStructure", role);
|
|
181
|
+
paths.sort(compareBytes);
|
|
182
|
+
const encodedPayload = record.kind === "directory"
|
|
183
|
+
? undefined
|
|
184
|
+
: record.kind === "file"
|
|
185
|
+
? record.data
|
|
186
|
+
: record.target;
|
|
187
|
+
if (encodedPayload !== undefined) {
|
|
188
|
+
const payloadLength = encodedPayloadLength(encodedPayload);
|
|
189
|
+
const next = safeAdd(role === "base" ? basePayloadBytes : payloadBytes, payloadLength);
|
|
190
|
+
const limit = role === "base" ? limits.maxIdentityBytes : limits.maxOutputBytes;
|
|
191
|
+
if (next === undefined || exceedsByteLimit(next, limit)) {
|
|
192
|
+
return yield* failure("LimitExceeded", role === "base" ? "identityBytes" : "outputBytes");
|
|
193
|
+
}
|
|
194
|
+
if (role === "base")
|
|
195
|
+
basePayloadBytes = next;
|
|
196
|
+
else
|
|
197
|
+
payloadBytes = next;
|
|
198
|
+
}
|
|
199
|
+
const payload = encodedPayload === undefined ? undefined : Image.bytes(encodedPayload);
|
|
200
|
+
const object = {
|
|
201
|
+
kind: record.kind,
|
|
202
|
+
metadata: record.metadata,
|
|
203
|
+
paths,
|
|
204
|
+
pathIdentity: paths.map(key).join("/"),
|
|
205
|
+
payload
|
|
206
|
+
};
|
|
207
|
+
objects.push(object);
|
|
208
|
+
for (const path of paths)
|
|
209
|
+
byPath.set(key(path), { object, path });
|
|
210
|
+
}
|
|
211
|
+
objects.sort((a, b) => compareBytes(a.paths[0], b.paths[0]));
|
|
212
|
+
return { objects, byPath };
|
|
213
|
+
});
|
|
214
|
+
const u64 = (n) => {
|
|
215
|
+
const out = new Uint8Array(8);
|
|
216
|
+
new DataView(out.buffer).setBigUint64(0, BigInt(n), false);
|
|
217
|
+
return out;
|
|
218
|
+
};
|
|
219
|
+
// This is the versioned semantic identity encoding, not a generic byte builder.
|
|
220
|
+
// Field framing, ordering, and the domain prefix are part of the persisted delta contract.
|
|
221
|
+
const identityBytes = Effect.fnUntraced(function* (view, limits) {
|
|
222
|
+
let length = 0;
|
|
223
|
+
const initialCapacity = ByteSize.isGreaterThanOrEqualTo(limits.maxIdentityBytes, ByteSize.kibibytes(1))
|
|
224
|
+
? 1024
|
|
225
|
+
: Number(ByteSize.toBigInt(limits.maxIdentityBytes));
|
|
226
|
+
let output = new Uint8Array(initialCapacity);
|
|
227
|
+
const append = (bytes) => {
|
|
228
|
+
const next = safeAdd(length, bytes.length);
|
|
229
|
+
if (next === undefined || BigInt(next) > ByteSize.toBigInt(limits.maxIdentityBytes)) {
|
|
230
|
+
return Effect.fail(failure("LimitExceeded", "identityBytes"));
|
|
231
|
+
}
|
|
232
|
+
if (next > output.length) {
|
|
233
|
+
let capacity = Math.max(1, output.length);
|
|
234
|
+
while (capacity < next)
|
|
235
|
+
capacity = Math.min(next, capacity * 2);
|
|
236
|
+
const expanded = new Uint8Array(capacity);
|
|
237
|
+
expanded.set(output);
|
|
238
|
+
output = expanded;
|
|
239
|
+
}
|
|
240
|
+
output.set(bytes, length);
|
|
241
|
+
length = next;
|
|
242
|
+
return Effect.void;
|
|
243
|
+
};
|
|
244
|
+
const frame = Effect.fnUntraced(function* (bytes) {
|
|
245
|
+
yield* append(u64(bytes.length));
|
|
246
|
+
yield* append(bytes);
|
|
247
|
+
});
|
|
248
|
+
yield* append(encoder.encode("effect-vfs-semantic-sha256-v1\0"));
|
|
249
|
+
yield* append(u64(view.objects.length));
|
|
250
|
+
for (const object of view.objects) {
|
|
251
|
+
yield* append(u64(object.paths.length));
|
|
252
|
+
for (const path of object.paths)
|
|
253
|
+
yield* frame(path);
|
|
254
|
+
yield* append(new Uint8Array([object.kind === "directory" ? 0 : object.kind === "file" ? 1 : 2]));
|
|
255
|
+
yield* append(u64(object.metadata.uid));
|
|
256
|
+
yield* append(u64(object.metadata.gid));
|
|
257
|
+
yield* append(u64(object.metadata.mode));
|
|
258
|
+
yield* frame(encoder.encode(object.metadata.atimeNs));
|
|
259
|
+
yield* frame(encoder.encode(object.metadata.mtimeNs));
|
|
260
|
+
yield* frame(encoder.encode(object.metadata.ctimeNs));
|
|
261
|
+
yield* frame(encoder.encode(object.metadata.birthtimeNs));
|
|
262
|
+
yield* frame(object.payload ?? new Uint8Array());
|
|
263
|
+
}
|
|
264
|
+
return output.slice(0, length);
|
|
265
|
+
});
|
|
266
|
+
const digest = Effect.fnUntraced(function* (view, limits) {
|
|
267
|
+
const crypto = yield* Crypto.Crypto;
|
|
268
|
+
return yield* crypto.digest("SHA-256", yield* identityBytes(view, limits));
|
|
269
|
+
});
|
|
270
|
+
const samePaths = (a, b) => a.pathIdentity === b.pathIdentity;
|
|
271
|
+
const differences = (before, after) => differenceOrder.filter((field) => {
|
|
272
|
+
if (field === "kind")
|
|
273
|
+
return before.kind !== after.kind;
|
|
274
|
+
if (field === "content") {
|
|
275
|
+
return before.kind === after.kind && after.kind === "file" && !sameBytes(before.payload, after.payload);
|
|
276
|
+
}
|
|
277
|
+
if (field === "target") {
|
|
278
|
+
return before.kind === after.kind && after.kind === "symlink" && !sameBytes(before.payload, after.payload);
|
|
279
|
+
}
|
|
280
|
+
if (field === "hardLinks")
|
|
281
|
+
return !samePaths(before, after);
|
|
282
|
+
return before.metadata[field] !== after.metadata[field];
|
|
283
|
+
});
|
|
284
|
+
const compare = (base, target) => {
|
|
285
|
+
const changes = [];
|
|
286
|
+
const add = (change, path) => changes.push({ change, path });
|
|
287
|
+
const differencesByPair = new WeakMap();
|
|
288
|
+
const cachedDifferences = (before, after) => {
|
|
289
|
+
let byAfter = differencesByPair.get(before);
|
|
290
|
+
if (byAfter === undefined) {
|
|
291
|
+
byAfter = new WeakMap();
|
|
292
|
+
differencesByPair.set(before, byAfter);
|
|
293
|
+
}
|
|
294
|
+
const cached = byAfter.get(after);
|
|
295
|
+
if (cached !== undefined)
|
|
296
|
+
return cached;
|
|
297
|
+
const value = differences(before, after);
|
|
298
|
+
byAfter.set(after, value);
|
|
299
|
+
return value;
|
|
300
|
+
};
|
|
301
|
+
for (const [pathKey, before] of base.byPath) {
|
|
302
|
+
const after = target.byPath.get(pathKey);
|
|
303
|
+
if (after === undefined) {
|
|
304
|
+
add({ _tag: "Removed", path: Image.base64(before.path), kind: before.object.kind }, before.path);
|
|
305
|
+
}
|
|
306
|
+
else {
|
|
307
|
+
const diff = cachedDifferences(before.object, after.object);
|
|
308
|
+
if (diff.length > 0) {
|
|
309
|
+
add({
|
|
310
|
+
_tag: "Updated",
|
|
311
|
+
path: Image.base64(before.path),
|
|
312
|
+
beforeKind: before.object.kind,
|
|
313
|
+
afterKind: after.object.kind,
|
|
314
|
+
differences: diff
|
|
315
|
+
}, before.path);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
for (const [pathKey, after] of target.byPath) {
|
|
320
|
+
if (!base.byPath.has(pathKey)) {
|
|
321
|
+
add({ _tag: "Added", path: Image.base64(after.path), kind: after.object.kind }, after.path);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
changes.sort((a, b) => compareBytes(a.path, b.path));
|
|
325
|
+
return changes.map(({ change }) => change);
|
|
326
|
+
};
|
|
327
|
+
const getDocument = (delta) => Effect.suspend(() => {
|
|
328
|
+
const value = snapshotDeltaValue(delta);
|
|
329
|
+
return value !== undefined && Schema.is(Document)(value)
|
|
330
|
+
? Effect.succeed(value)
|
|
331
|
+
: Effect.fail(failure("InvalidStructure", "delta"));
|
|
332
|
+
});
|
|
333
|
+
const buildImage = Effect.fnUntraced(function* (document, base, limits) {
|
|
334
|
+
const paths = new Map();
|
|
335
|
+
const records = [];
|
|
336
|
+
const entriesById = new Map();
|
|
337
|
+
let bytes = 0;
|
|
338
|
+
for (let i = 0; i < document.records.length; i++) {
|
|
339
|
+
const record = document.records[i];
|
|
340
|
+
const recordId = String(i);
|
|
341
|
+
for (const encodedPath of record.paths) {
|
|
342
|
+
const path = yield* decode64(encodedPath, "path");
|
|
343
|
+
paths.set(key(path), { id: recordId, kind: record.kind, bytes: path });
|
|
344
|
+
}
|
|
345
|
+
if (record.kind === "directory") {
|
|
346
|
+
const entries = [];
|
|
347
|
+
entriesById.set(recordId, entries);
|
|
348
|
+
records.push({ id: recordId, kind: "directory", metadata: record.metadata, entries });
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
const inherited = record.payload._tag === "Base"
|
|
352
|
+
? base?.byPath.get(key(yield* decode64(record.payload.path, "basePath")))?.object
|
|
353
|
+
: undefined;
|
|
354
|
+
const inlineLength = record.payload._tag === "Inline"
|
|
355
|
+
? encodedPayloadLength(record.payload.bytes)
|
|
356
|
+
: undefined;
|
|
357
|
+
const payloadLength = inlineLength ?? (base === undefined
|
|
358
|
+
? 0
|
|
359
|
+
: inherited?.kind === record.kind
|
|
360
|
+
? inherited.payload?.length
|
|
361
|
+
: undefined);
|
|
362
|
+
const next = payloadLength === undefined ? undefined : safeAdd(bytes, payloadLength);
|
|
363
|
+
if (next !== undefined && exceedsByteLimit(next, limits.maxOutputBytes)) {
|
|
364
|
+
return yield* failure("LimitExceeded", "outputBytes");
|
|
365
|
+
}
|
|
366
|
+
const payload = record.payload._tag === "Inline"
|
|
367
|
+
? yield* decode64(record.payload.bytes, "payload")
|
|
368
|
+
: inherited?.kind === record.kind
|
|
369
|
+
? inherited.payload
|
|
370
|
+
: undefined;
|
|
371
|
+
if (payload === undefined && base !== undefined)
|
|
372
|
+
return yield* failure("InvalidStructure", "baseReference");
|
|
373
|
+
const actual = payload ?? new Uint8Array();
|
|
374
|
+
if (next === undefined)
|
|
375
|
+
return yield* failure("LimitExceeded", "outputBytes");
|
|
376
|
+
bytes = next;
|
|
377
|
+
records.push(record.kind === "file"
|
|
378
|
+
? { id: recordId, kind: "file", metadata: record.metadata, data: Image.base64(actual) }
|
|
379
|
+
: { id: recordId, kind: "symlink", metadata: record.metadata, target: Image.base64(actual) });
|
|
380
|
+
}
|
|
381
|
+
const root = paths.get(key(new Uint8Array([47])));
|
|
382
|
+
if (root?.kind !== "directory")
|
|
383
|
+
return yield* failure("InvalidStructure", "root");
|
|
384
|
+
for (const [pathKey, child] of paths) {
|
|
385
|
+
if (pathKey !== key(new Uint8Array([47]))) {
|
|
386
|
+
const parent = paths.get(parentKey(child.bytes));
|
|
387
|
+
const entries = parent === undefined ? undefined : entriesById.get(parent.id);
|
|
388
|
+
if (entries === undefined)
|
|
389
|
+
return yield* failure("InvalidStructure", "parent");
|
|
390
|
+
entries.push({ name: Image.base64(basename(child.bytes)), target: child.id });
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
for (const entries of entriesById.values())
|
|
394
|
+
entries.sort((a, b) => a.name.localeCompare(b.name));
|
|
395
|
+
return yield* Image.capture({ format: "effect-vfs", version: 1, root: root.id, records });
|
|
396
|
+
});
|
|
397
|
+
const validate = Effect.fnUntraced(function* (document, limits) {
|
|
398
|
+
const deltaRecords = safeAdd(document.records.length, document.changes.length);
|
|
399
|
+
if (deltaRecords === undefined || deltaRecords > limits.maxDeltaRecords) {
|
|
400
|
+
return yield* failure("LimitExceeded", "deltaRecords");
|
|
401
|
+
}
|
|
402
|
+
if (document.records.length > limits.maxOutputRecords)
|
|
403
|
+
return yield* failure("LimitExceeded", "outputRecords");
|
|
404
|
+
if (!CanonicalBase64.isCanonical(document.base.digest) || CanonicalBase64.decodedLength(document.base.digest) !== 32) {
|
|
405
|
+
return yield* failure("InvalidEncoding", "digest");
|
|
406
|
+
}
|
|
407
|
+
let decoded = 32;
|
|
408
|
+
let entries = -1;
|
|
409
|
+
let inherited = 0;
|
|
410
|
+
const seen = new Set();
|
|
411
|
+
for (const record of document.records) {
|
|
412
|
+
if (record.paths.length < 1 || (record.kind === "directory" && record.paths.length !== 1)) {
|
|
413
|
+
return yield* failure("InvalidStructure", "paths");
|
|
414
|
+
}
|
|
415
|
+
for (const encodedPath of record.paths) {
|
|
416
|
+
if (!CanonicalBase64.isCanonical(encodedPath))
|
|
417
|
+
return yield* failure("InvalidEncoding", "path");
|
|
418
|
+
decoded = safeAdd(decoded, CanonicalBase64.decodedLength(encodedPath)) ?? Number.POSITIVE_INFINITY;
|
|
419
|
+
if (BigInt(decoded) > ByteSize.toBigInt(limits.maxDecodedDeltaBytes)) {
|
|
420
|
+
return yield* failure("LimitExceeded", "decodedDeltaBytes");
|
|
421
|
+
}
|
|
422
|
+
entries++;
|
|
423
|
+
if (entries > limits.maxEntries)
|
|
424
|
+
return yield* failure("LimitExceeded", "entries");
|
|
425
|
+
const path = yield* decode64(encodedPath, "path");
|
|
426
|
+
if (!validPath(path) || seen.has(key(path)))
|
|
427
|
+
return yield* failure("InvalidStructure", "path");
|
|
428
|
+
seen.add(key(path));
|
|
429
|
+
}
|
|
430
|
+
if (record.kind !== "directory") {
|
|
431
|
+
if (record.payload._tag === "Inline") {
|
|
432
|
+
if (!CanonicalBase64.isCanonical(record.payload.bytes))
|
|
433
|
+
return yield* failure("InvalidEncoding", "payload");
|
|
434
|
+
decoded = safeAdd(decoded, CanonicalBase64.decodedLength(record.payload.bytes)) ?? Number.POSITIVE_INFINITY;
|
|
435
|
+
if (BigInt(decoded) > ByteSize.toBigInt(limits.maxDecodedDeltaBytes)) {
|
|
436
|
+
return yield* failure("LimitExceeded", "decodedDeltaBytes");
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
else {
|
|
440
|
+
if (!CanonicalBase64.isCanonical(record.payload.path))
|
|
441
|
+
return yield* failure("InvalidEncoding", "basePath");
|
|
442
|
+
decoded = safeAdd(decoded, CanonicalBase64.decodedLength(record.payload.path)) ?? Number.POSITIVE_INFINITY;
|
|
443
|
+
if (BigInt(decoded) > ByteSize.toBigInt(limits.maxDecodedDeltaBytes)) {
|
|
444
|
+
return yield* failure("LimitExceeded", "decodedDeltaBytes");
|
|
445
|
+
}
|
|
446
|
+
inherited++;
|
|
447
|
+
if (inherited > limits.maxInheritedRecords) {
|
|
448
|
+
return yield* failure("LimitExceeded", "inheritedRecords");
|
|
449
|
+
}
|
|
450
|
+
const path = yield* decode64(record.payload.path, "basePath");
|
|
451
|
+
if (!validPath(path) || !record.paths.includes(record.payload.path)) {
|
|
452
|
+
return yield* failure("InvalidStructure", "basePath");
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
let previous;
|
|
458
|
+
for (const change of document.changes) {
|
|
459
|
+
if (!CanonicalBase64.isCanonical(change.path))
|
|
460
|
+
return yield* failure("InvalidEncoding", "changePath");
|
|
461
|
+
decoded = safeAdd(decoded, CanonicalBase64.decodedLength(change.path)) ?? Number.POSITIVE_INFINITY;
|
|
462
|
+
if (BigInt(decoded) > ByteSize.toBigInt(limits.maxDecodedDeltaBytes)) {
|
|
463
|
+
return yield* failure("LimitExceeded", "decodedDeltaBytes");
|
|
464
|
+
}
|
|
465
|
+
const path = yield* decode64(change.path, "changePath");
|
|
466
|
+
if (!validPath(path) || (previous !== undefined && compareBytes(previous, path) >= 0)) {
|
|
467
|
+
return yield* failure("InvalidStructure", "changes");
|
|
468
|
+
}
|
|
469
|
+
previous = path;
|
|
470
|
+
}
|
|
471
|
+
yield* buildImage(document, undefined, limits);
|
|
472
|
+
});
|
|
473
|
+
/** @internal */
|
|
474
|
+
export const diffSnapshots = Effect.fn("VirtualFileSystem.diffSnapshots")(function* (base, target, limits) {
|
|
475
|
+
const before = yield* normalize(base, limits, "base");
|
|
476
|
+
const after = yield* normalize(target, limits, "target");
|
|
477
|
+
const changes = [...compare(before, after)];
|
|
478
|
+
const deltaRecords = safeAdd(after.objects.length, changes.length);
|
|
479
|
+
if (deltaRecords === undefined || deltaRecords > limits.maxDeltaRecords) {
|
|
480
|
+
return yield* failure("LimitExceeded", "deltaRecords");
|
|
481
|
+
}
|
|
482
|
+
const payloadEquality = new WeakMap();
|
|
483
|
+
const samePayload = (source, target) => {
|
|
484
|
+
let byTarget = payloadEquality.get(source);
|
|
485
|
+
if (byTarget === undefined) {
|
|
486
|
+
byTarget = new WeakMap();
|
|
487
|
+
payloadEquality.set(source, byTarget);
|
|
488
|
+
}
|
|
489
|
+
const cached = byTarget.get(target);
|
|
490
|
+
if (cached !== undefined)
|
|
491
|
+
return cached;
|
|
492
|
+
const value = sameBytes(source.payload, target.payload);
|
|
493
|
+
byTarget.set(target, value);
|
|
494
|
+
return value;
|
|
495
|
+
};
|
|
496
|
+
const records = after.objects.map((object) => {
|
|
497
|
+
const paths = object.paths.map(Image.base64);
|
|
498
|
+
if (object.kind === "directory")
|
|
499
|
+
return { kind: object.kind, paths, metadata: object.metadata };
|
|
500
|
+
const inherited = object.paths.find((path) => {
|
|
501
|
+
const source = before.byPath.get(key(path));
|
|
502
|
+
return source?.object.kind === object.kind && samePayload(source.object, object);
|
|
503
|
+
});
|
|
504
|
+
return {
|
|
505
|
+
kind: object.kind,
|
|
506
|
+
paths,
|
|
507
|
+
metadata: object.metadata,
|
|
508
|
+
payload: inherited === undefined
|
|
509
|
+
? { _tag: "Inline", bytes: Image.base64(object.payload ?? new Uint8Array()) }
|
|
510
|
+
: { _tag: "Base", path: Image.base64(inherited) }
|
|
511
|
+
};
|
|
512
|
+
});
|
|
513
|
+
const document = {
|
|
514
|
+
format: "effect-vfs-delta",
|
|
515
|
+
version: 1,
|
|
516
|
+
base: { algorithm: "effect-vfs-semantic-sha256-v1", digest: Image.base64(yield* digest(before, limits)) },
|
|
517
|
+
records,
|
|
518
|
+
changes
|
|
519
|
+
};
|
|
520
|
+
yield* validate(document, limits);
|
|
521
|
+
return makeSnapshotDelta(document);
|
|
522
|
+
});
|
|
523
|
+
/** @internal */
|
|
524
|
+
const verify = Effect.fnUntraced(function* (base, document, limits) {
|
|
525
|
+
yield* validate(document, limits);
|
|
526
|
+
const before = yield* normalize(base, limits, "base");
|
|
527
|
+
if (!sameBytes(yield* digest(before, limits), Image.bytes(document.base.digest))) {
|
|
528
|
+
return yield* new SnapshotDeltaError({ code: "BaseMismatch" });
|
|
529
|
+
}
|
|
530
|
+
const target = yield* buildImage(document, before, limits);
|
|
531
|
+
const changes = compare(before, yield* normalize(target, limits, "target"));
|
|
532
|
+
const actualChanges = yield* Schema.encodeEffect(Json)(changes).pipe(Effect.mapError(() => failure("InvalidStructure", "changes")));
|
|
533
|
+
const expectedChanges = yield* Schema.encodeEffect(Json)(document.changes).pipe(Effect.mapError(() => failure("InvalidStructure", "changes")));
|
|
534
|
+
if (actualChanges !== expectedChanges)
|
|
535
|
+
return yield* failure("InvalidStructure", "changes");
|
|
536
|
+
return { changes, target };
|
|
537
|
+
});
|
|
538
|
+
/** @internal */
|
|
539
|
+
export const inspectSnapshotDelta = Effect.fn("VirtualFileSystem.inspectSnapshotDelta")(function* (base, delta, options, limits) {
|
|
540
|
+
const output = [];
|
|
541
|
+
const { changes } = yield* verify(base, yield* getDocument(delta), limits);
|
|
542
|
+
for (const change of changes) {
|
|
543
|
+
const path = makeBytePath(new Uint8Array(yield* decode64(change.path, "changePath")));
|
|
544
|
+
if (change._tag !== "Updated")
|
|
545
|
+
output.push(Object.freeze({ ...change, path }));
|
|
546
|
+
else {
|
|
547
|
+
const fields = change.differences.filter((field) => options.includeTimestamps === true || !timestampFields.has(field));
|
|
548
|
+
if (fields.length > 0)
|
|
549
|
+
output.push(Object.freeze({ ...change, path, differences: Object.freeze([...fields]) }));
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
return Object.freeze(output);
|
|
553
|
+
});
|
|
554
|
+
/** @internal */
|
|
555
|
+
export const encodeSnapshotDelta = Effect.fnUntraced(function* (delta, limits) {
|
|
556
|
+
const document = yield* getDocument(delta);
|
|
557
|
+
yield* validate(document, limits);
|
|
558
|
+
const text = yield* Schema.encodeEffect(Json)(document).pipe(Effect.mapError(() => failure("InvalidStructure")));
|
|
559
|
+
const bytes = encoder.encode(text);
|
|
560
|
+
if (BigInt(bytes.length) > ByteSize.toBigInt(limits.maxEncodedBytes)) {
|
|
561
|
+
return yield* failure("LimitExceeded", "encodedBytes");
|
|
562
|
+
}
|
|
563
|
+
return bytes;
|
|
564
|
+
});
|
|
565
|
+
/** @internal */
|
|
566
|
+
export const decodeSnapshotDelta = Effect.fnUntraced(function* (input, limits) {
|
|
567
|
+
if (!(input instanceof Uint8Array) || !(input.buffer instanceof ArrayBuffer))
|
|
568
|
+
return yield* failure("InvalidEncoding");
|
|
569
|
+
if (BigInt(input.byteLength) > ByteSize.toBigInt(limits.maxEncodedBytes)) {
|
|
570
|
+
return yield* failure("LimitExceeded", "encodedBytes");
|
|
571
|
+
}
|
|
572
|
+
const text = yield* Effect.try({
|
|
573
|
+
try: () => new TextDecoder("utf-8", { fatal: true }).decode(new Uint8Array(input)),
|
|
574
|
+
catch: () => failure("InvalidEncoding")
|
|
575
|
+
});
|
|
576
|
+
const value = yield* Schema.decodeEffect(Json)(text).pipe(Effect.mapError(() => failure("InvalidEncoding")));
|
|
577
|
+
if (typeof value === "object" && value !== null && "format" in value && value.format === "effect-vfs-delta" &&
|
|
578
|
+
"version" in value && value.version !== 1)
|
|
579
|
+
return yield* failure("UnsupportedVersion");
|
|
580
|
+
const parsed = Schema.decodeUnknownResult(Document, { onExcessProperty: "error" })(value);
|
|
581
|
+
if (Result.isFailure(parsed))
|
|
582
|
+
return yield* failure("InvalidStructure");
|
|
583
|
+
yield* validate(parsed.success, limits);
|
|
584
|
+
return makeSnapshotDelta(parsed.success);
|
|
585
|
+
});
|
|
586
|
+
/** @internal */
|
|
587
|
+
export const applySnapshotDelta = Effect.fn("VirtualFileSystem.applySnapshotDelta")(function* (base, delta, limits) {
|
|
588
|
+
return (yield* verify(base, yield* getDocument(delta), limits)).target;
|
|
589
|
+
});
|