@amalgm/shell 0.1.163 → 0.1.164-main.34182745507
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PURPOSE.md +68 -6
- package/README.md +58 -11
- package/config/releases/trust/runtime/main.root.json +84 -0
- package/config/releases/trust/runtime/preview.root.json +84 -0
- package/dist/boot.js +28 -20
- package/dist/boot.js.map +1 -1
- package/dist/cli.js +7 -1
- package/dist/cli.js.map +1 -1
- package/dist/content-cache-index.js +2 -2
- package/dist/content-cache-index.js.map +1 -1
- package/dist/detection/runtime.js +2 -2
- package/dist/detection/runtime.js.map +1 -1
- package/dist/entity-apply-host.js +2 -2
- package/dist/entity-apply-host.js.map +1 -1
- package/dist/entity-apply-store.d.ts +2 -1
- package/dist/entity-apply-store.js +52 -37
- package/dist/entity-apply-store.js.map +1 -1
- package/dist/installation/lease.js +2 -2
- package/dist/installation/lease.js.map +1 -1
- package/dist/installation/ownership.js +2 -2
- package/dist/installation/ownership.js.map +1 -1
- package/dist/machine-host.d.ts +1 -0
- package/dist/machine-host.js +56 -31
- package/dist/machine-host.js.map +1 -1
- package/dist/process-lifecycle.d.ts +1 -3
- package/dist/process-lifecycle.js +10 -0
- package/dist/process-lifecycle.js.map +1 -1
- package/dist/pty-host.d.ts +1 -1
- package/dist/pty-host.js +70 -21
- package/dist/pty-host.js.map +1 -1
- package/dist/service/cli.js +50 -9
- package/dist/service/cli.js.map +1 -1
- package/dist/service/control.d.ts +10 -1
- package/dist/service/control.js +1 -1
- package/dist/service/control.js.map +1 -1
- package/dist/service/keepalive.d.ts +10 -3
- package/dist/service/keepalive.js +267 -29
- package/dist/service/keepalive.js.map +1 -1
- package/dist/service/recovery.d.ts +27 -0
- package/dist/service/recovery.js +58 -0
- package/dist/service/recovery.js.map +1 -0
- package/dist/service/systemd-unit.d.ts +7 -0
- package/dist/service/systemd-unit.js +43 -0
- package/dist/service/systemd-unit.js.map +1 -0
- package/dist/service/systemd.d.ts +1 -5
- package/dist/service/systemd.js +9 -31
- package/dist/service/systemd.js.map +1 -1
- package/dist/service/windows-action.d.ts +9 -0
- package/dist/service/windows-action.js +55 -0
- package/dist/service/windows-action.js.map +1 -0
- package/dist/service/windows-task.d.ts +1 -8
- package/dist/service/windows-task.js +4 -40
- package/dist/service/windows-task.js.map +1 -1
- package/dist/service/windows.js +3 -3
- package/dist/service/windows.js.map +1 -1
- package/dist/sqlite.d.ts +4 -0
- package/dist/sqlite.js +16 -0
- package/dist/sqlite.js.map +1 -0
- package/dist/updates/artifacts.d.ts +40 -0
- package/dist/updates/artifacts.js +299 -0
- package/dist/updates/artifacts.js.map +1 -0
- package/dist/updates/bootstrap.d.ts +48 -0
- package/dist/updates/bootstrap.js +384 -0
- package/dist/updates/bootstrap.js.map +1 -0
- package/dist/updates/controller.d.ts +66 -0
- package/dist/updates/controller.js +322 -0
- package/dist/updates/controller.js.map +1 -0
- package/dist/updates/isolated-command-lifetime.d.ts +1 -0
- package/dist/updates/isolated-command-lifetime.js +63 -0
- package/dist/updates/isolated-command-lifetime.js.map +1 -0
- package/dist/updates/store.d.ts +41 -0
- package/dist/updates/store.js +80 -0
- package/dist/updates/store.js.map +1 -0
- package/dist/updates/trust.d.ts +20 -0
- package/dist/updates/trust.js +138 -0
- package/dist/updates/trust.js.map +1 -0
- package/dist/user-ground-host.d.ts +1 -1
- package/dist/user-ground-host.js +3 -3
- package/dist/user-ground-host.js.map +1 -1
- package/dist/watching/host.js +4 -2
- package/dist/watching/host.js.map +1 -1
- package/dist/workspace-add-staging.js +2 -2
- package/dist/workspace-add-staging.js.map +1 -1
- package/examinations/fixtures/machine-control-plane.js +57 -0
- package/examinations/fixtures/private-wire-authority.js +1065 -0
- package/examinations/native-service.mjs +92 -0
- package/examinations/network-view.mjs +202 -0
- package/examinations/runtime-upgrade.native-test.mjs +198 -0
- package/npm-shrinkwrap.json +2104 -191
- package/package.json +94 -13
|
@@ -0,0 +1,1065 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { createCipheriv, createHash, createHmac, randomBytes } from "node:crypto";
|
|
3
|
+
import { createServer } from "node:http";
|
|
4
|
+
import { inflateRawSync } from "node:zlib";
|
|
5
|
+
|
|
6
|
+
import { WebSocket, WebSocketServer } from "ws";
|
|
7
|
+
import {
|
|
8
|
+
acceptEntityRecord,
|
|
9
|
+
decodeInlineCargo,
|
|
10
|
+
decodeLocalEntityRecord,
|
|
11
|
+
encodeEntityRecord,
|
|
12
|
+
parseGraphHead,
|
|
13
|
+
parseGraphPage,
|
|
14
|
+
proveRootPagination,
|
|
15
|
+
recordsByRoot,
|
|
16
|
+
sameRootPages,
|
|
17
|
+
GRAPH_HEAD_KIND,
|
|
18
|
+
GRAPH_HEAD_OPERATION_KIND,
|
|
19
|
+
PROTECTED_ENVELOPE_FORMAT,
|
|
20
|
+
PROTECTED_ENVELOPE_MAGIC,
|
|
21
|
+
rootReplacementFromRecords,
|
|
22
|
+
snapshotFromRecords,
|
|
23
|
+
stableJson,
|
|
24
|
+
} from "@amalgm/live";
|
|
25
|
+
import { decodeBinaryFrame } from "../../dist/wire-binary.js";
|
|
26
|
+
|
|
27
|
+
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
28
|
+
|
|
29
|
+
/** The fake plays both the gateway (key handoff, URL signing) and content
|
|
30
|
+
* storage (HTTP GET of sealed envelopes) with the production key law:
|
|
31
|
+
* per-resource key = HMAC(master, resource:id:contract). */
|
|
32
|
+
const CONTENT_MASTER_KEY = Buffer.from("aa".repeat(32), "hex");
|
|
33
|
+
const RESOURCE_CONTRACT = "amalgm-entity-registry@3";
|
|
34
|
+
|
|
35
|
+
function resourceContentKey(resourceId) {
|
|
36
|
+
return createHmac("sha256", CONTENT_MASTER_KEY)
|
|
37
|
+
.update(`resource:${resourceId}:${RESOURCE_CONTRACT}`)
|
|
38
|
+
.digest();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Seal exactly the envelope the production gateway stores: AES-256-GCM with
|
|
42
|
+
* the AAD binding resource, contract, and purpose. Shell must refuse anything
|
|
43
|
+
* sealed under a different binding. */
|
|
44
|
+
function sealContentEnvelope(resourceId, data) {
|
|
45
|
+
const iv = randomBytes(12);
|
|
46
|
+
const cipher = createCipheriv("aes-256-gcm", resourceContentKey(resourceId), iv);
|
|
47
|
+
cipher.setAAD(Buffer.from(
|
|
48
|
+
`${PROTECTED_ENVELOPE_FORMAT}:${resourceId}:${RESOURCE_CONTRACT}:entity-content`,
|
|
49
|
+
"utf8",
|
|
50
|
+
));
|
|
51
|
+
const sealed = Buffer.concat([cipher.update(data), cipher.final()]);
|
|
52
|
+
return Buffer.concat([
|
|
53
|
+
Buffer.from(PROTECTED_ENVELOPE_MAGIC, "ascii"),
|
|
54
|
+
iv,
|
|
55
|
+
cipher.getAuthTag(),
|
|
56
|
+
sealed,
|
|
57
|
+
]);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/** Shared test authority for the exact private-resource/content/mutation
|
|
61
|
+
* frames used by the production host. It also serves the direct content
|
|
62
|
+
* plane: refs answer where each sealed envelope lives — a byte range of an
|
|
63
|
+
* immutable cargo pack, or a standalone object URL — and this server answers
|
|
64
|
+
* the bytes the way content storage would, Range slices included. Additional
|
|
65
|
+
* HTTP behavior may be composed on the same server by supplying a request
|
|
66
|
+
* listener. */
|
|
67
|
+
export function createPrivateWireAuthority({
|
|
68
|
+
token = "test-tunnel-token",
|
|
69
|
+
protocolVersion = 17,
|
|
70
|
+
onRequest,
|
|
71
|
+
contentPutDelayMs = 0,
|
|
72
|
+
contentPutFailures = 0,
|
|
73
|
+
contentPutFailureRetryable = true,
|
|
74
|
+
contentPutGate,
|
|
75
|
+
} = {}) {
|
|
76
|
+
const contentFetches = [];
|
|
77
|
+
let activeContentFetchGate = null;
|
|
78
|
+
const serveDirectContent = async (request, response) => {
|
|
79
|
+
const [, , resourceId, kind, sha] = request.url.split("/").map(decodeURIComponent);
|
|
80
|
+
contentFetches.push({ resourceId, kind, sha256: sha });
|
|
81
|
+
const admitted = activeContentFetchGate
|
|
82
|
+
? activeContentFetchGate({ resourceId, kind, sha256: sha })
|
|
83
|
+
: Promise.resolve();
|
|
84
|
+
try {
|
|
85
|
+
await admitted;
|
|
86
|
+
} catch {
|
|
87
|
+
response.statusCode = 503;
|
|
88
|
+
response.end();
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
const entry = kind === "manifest"
|
|
92
|
+
? findContent(content, { resource_id: resourceId, content_hash: sha, kind: "manifest" })
|
|
93
|
+
: findContent(content, { resource_id: resourceId, kind: "chunk", sha256: sha });
|
|
94
|
+
if (!entry) {
|
|
95
|
+
response.statusCode = 404;
|
|
96
|
+
response.end();
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
response.statusCode = 200;
|
|
100
|
+
response.end(sealContentEnvelope(resourceId, entry.data));
|
|
101
|
+
};
|
|
102
|
+
/** Immutable pack serving: one stored byte string per cargo segment, sliced
|
|
103
|
+
* by Range exactly as content storage would. Fetch logs and the fetch gate
|
|
104
|
+
* see the pack read, not its members — that is what storage observes. */
|
|
105
|
+
const serveDirectPack = async (request, response) => {
|
|
106
|
+
const [, , resourceId, packId] = request.url.split("/").map(decodeURIComponent);
|
|
107
|
+
const range = request.headers.range;
|
|
108
|
+
contentFetches.push({ resourceId, kind: "pack", sha256: packId, range });
|
|
109
|
+
const admitted = activeContentFetchGate
|
|
110
|
+
? activeContentFetchGate({ resourceId, kind: "pack", sha256: packId })
|
|
111
|
+
: Promise.resolve();
|
|
112
|
+
try {
|
|
113
|
+
await admitted;
|
|
114
|
+
} catch {
|
|
115
|
+
response.statusCode = 503;
|
|
116
|
+
response.end();
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const pack = packs.get(packId);
|
|
120
|
+
if (!pack) {
|
|
121
|
+
response.statusCode = 404;
|
|
122
|
+
response.end();
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
const bounds = range ? /^bytes=(\d+)-(\d+)$/.exec(range) : null;
|
|
126
|
+
if (bounds) {
|
|
127
|
+
response.statusCode = 206;
|
|
128
|
+
response.end(pack.subarray(Number(bounds[1]), Number(bounds[2]) + 1));
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
response.statusCode = 200;
|
|
132
|
+
response.end(pack);
|
|
133
|
+
};
|
|
134
|
+
const server = createServer((request, response) => {
|
|
135
|
+
if (request.url?.startsWith("/direct-content/")) {
|
|
136
|
+
void serveDirectContent(request, response);
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
if (request.url?.startsWith("/direct-pack/")) {
|
|
140
|
+
void serveDirectPack(request, response);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (onRequest) {
|
|
144
|
+
onRequest(request, response);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
response.statusCode = 404;
|
|
148
|
+
response.end();
|
|
149
|
+
});
|
|
150
|
+
const sockets = new WebSocketServer({ server });
|
|
151
|
+
let current = null;
|
|
152
|
+
const content = new Map();
|
|
153
|
+
// The pack law: each cargo segment is durable as ONE immutable byte string
|
|
154
|
+
// (chunk envelopes in artifact order, manifest envelopes at the tail) and
|
|
155
|
+
// every member keeps its identity as a byte range in the location index.
|
|
156
|
+
const packs = new Map();
|
|
157
|
+
const packLocations = new Map();
|
|
158
|
+
let contentPutsInFlight = 0;
|
|
159
|
+
let maxContentPutsInFlight = 0;
|
|
160
|
+
let contentPutAttempts = 0;
|
|
161
|
+
// Every put attempt's content hash, and the hashes the committed graph
|
|
162
|
+
// claims as pages. Pages ride the same content plane as file cargo, so
|
|
163
|
+
// separating the two is the only way to ask whether the FILES moved.
|
|
164
|
+
const putAttemptHashes = [];
|
|
165
|
+
const graphPageHashes = new Set();
|
|
166
|
+
const contentInventories = [];
|
|
167
|
+
const contentSealQueries = [];
|
|
168
|
+
let onContentSealQuery = null;
|
|
169
|
+
const contentPutBatches = [];
|
|
170
|
+
const contentCargoSegments = [];
|
|
171
|
+
const contentKeyRequests = [];
|
|
172
|
+
const contentRefs = [];
|
|
173
|
+
let binaryContentPuts = 0;
|
|
174
|
+
let inlineCargoSubmits = 0;
|
|
175
|
+
// Arrival-order ledger for the transfer ordering laws: every content
|
|
176
|
+
// frame must reach the authority before the graph head that would make
|
|
177
|
+
// its records visible. Head submits are named so tests can anchor on them.
|
|
178
|
+
const frameOrder = [];
|
|
179
|
+
let remainingContentPutFailures = contentPutFailures;
|
|
180
|
+
let contentFailureRetryable = contentPutFailureRetryable;
|
|
181
|
+
let rejectMutationSubmits = false;
|
|
182
|
+
let remainingMutationSubmitFailures = 0;
|
|
183
|
+
let dropMutationAcks = 0;
|
|
184
|
+
const committedMutationAcks = new Map();
|
|
185
|
+
let deliverySequence = 0;
|
|
186
|
+
const entityRecords = [];
|
|
187
|
+
const entityHeads = new Map();
|
|
188
|
+
const entityDedup = new Map();
|
|
189
|
+
const graphMutations = [];
|
|
190
|
+
const entitySubscriptions = new Map();
|
|
191
|
+
let entitySubscribeRequests = 0;
|
|
192
|
+
sockets.on("connection", (socket, request) => {
|
|
193
|
+
assert.equal(request.headers.authorization, `Bearer ${token}`);
|
|
194
|
+
entitySubscriptions.set(socket, new Set());
|
|
195
|
+
socket.once("close", () => {
|
|
196
|
+
entitySubscriptions.delete(socket);
|
|
197
|
+
});
|
|
198
|
+
socket.on("message", (data, isBinary) => {
|
|
199
|
+
const frame = isBinary
|
|
200
|
+
? (() => {
|
|
201
|
+
const decoded = decodeBinaryFrame(data);
|
|
202
|
+
return { ...decoded.frame, data_bin: decoded.bytes };
|
|
203
|
+
})()
|
|
204
|
+
: JSON.parse(data.toString("utf8"));
|
|
205
|
+
if (frame.type === "private.entity-content.put"
|
|
206
|
+
|| frame.type === "private.entity-content.put-blocks"
|
|
207
|
+
|| frame.type === "private.entity-content.put-cargo") {
|
|
208
|
+
frameOrder.push(frame.type);
|
|
209
|
+
} else if (frame.type === "shared.mutation.submit") {
|
|
210
|
+
frameOrder.push(frame.envelope?.operationKind === GRAPH_HEAD_OPERATION_KIND
|
|
211
|
+
? "graph-head.submit"
|
|
212
|
+
: frame.type);
|
|
213
|
+
}
|
|
214
|
+
if (frame.type === "hello") {
|
|
215
|
+
socket.send(JSON.stringify({ type: "hello_ack", protocol_version: protocolVersion }));
|
|
216
|
+
} else if (frame.type === "private.resource.lookup") {
|
|
217
|
+
socket.send(JSON.stringify(current ? ready(frame.request_id, current, true) : {
|
|
218
|
+
type: "private.resource.missing",
|
|
219
|
+
request_id: frame.request_id,
|
|
220
|
+
resource_id: frame.resource_id,
|
|
221
|
+
contract: frame.contract,
|
|
222
|
+
}));
|
|
223
|
+
} else if (frame.type === "private.resource.ensure") {
|
|
224
|
+
const duplicate = current !== null;
|
|
225
|
+
if (!current) current = {
|
|
226
|
+
resourceId: frame.resource_id,
|
|
227
|
+
checksum: frame.snapshot_checksum,
|
|
228
|
+
snapshotB64: frame.snapshot_b64,
|
|
229
|
+
rootRecords: recordsByRoot(JSON.parse(
|
|
230
|
+
Buffer.from(frame.snapshot_b64, "base64").toString("utf8"),
|
|
231
|
+
).records),
|
|
232
|
+
version: 0,
|
|
233
|
+
deliveryWatermark: 0,
|
|
234
|
+
authorityEpoch: 1,
|
|
235
|
+
};
|
|
236
|
+
socket.send(JSON.stringify(ready(frame.request_id, current, duplicate)));
|
|
237
|
+
} else if (frame.type === "private.entity-content.inventory") {
|
|
238
|
+
contentInventories.push({ ...frame });
|
|
239
|
+
const manifestKey = contentKey({
|
|
240
|
+
...frame, kind: "manifest", part_index: 0, sha256: "",
|
|
241
|
+
});
|
|
242
|
+
const complete = content.has(manifestKey);
|
|
243
|
+
const missing = complete ? [] : frame.chunks.flatMap((chunk, index) => (
|
|
244
|
+
content.has(contentKey({
|
|
245
|
+
...frame, kind: "chunk", part_index: index, sha256: chunk.sha256,
|
|
246
|
+
})) ? [] : [index]
|
|
247
|
+
));
|
|
248
|
+
socket.send(JSON.stringify({
|
|
249
|
+
type: "private.entity-content.inventory-result",
|
|
250
|
+
request_id: frame.request_id,
|
|
251
|
+
content_hash: frame.content_hash,
|
|
252
|
+
complete,
|
|
253
|
+
missing,
|
|
254
|
+
}));
|
|
255
|
+
} else if (frame.type === "private.entity-content.put") {
|
|
256
|
+
// The seal frame: a manifest may only bind blocks the authority
|
|
257
|
+
// already holds, so a partial artifact can never become visible.
|
|
258
|
+
assert.equal(isBinary, true, "immutable content travels as a binary WebSocket frame");
|
|
259
|
+
assert.equal(frame.data_bin instanceof Uint8Array, true);
|
|
260
|
+
binaryContentPuts += 1;
|
|
261
|
+
contentPutAttempts += 1;
|
|
262
|
+
putAttemptHashes.push(String(frame.sha256));
|
|
263
|
+
if (remainingContentPutFailures > 0) {
|
|
264
|
+
remainingContentPutFailures -= 1;
|
|
265
|
+
socket.send(JSON.stringify({
|
|
266
|
+
type: "shared.error",
|
|
267
|
+
request_id: frame.request_id,
|
|
268
|
+
code: "shared_realtime_internal_error",
|
|
269
|
+
message: "transient object authority failure",
|
|
270
|
+
status: 503,
|
|
271
|
+
retryable: contentFailureRetryable,
|
|
272
|
+
}));
|
|
273
|
+
return;
|
|
274
|
+
}
|
|
275
|
+
contentPutsInFlight += 1;
|
|
276
|
+
maxContentPutsInFlight = Math.max(maxContentPutsInFlight, contentPutsInFlight);
|
|
277
|
+
const store = () => {
|
|
278
|
+
const data = Buffer.from(frame.data_bin);
|
|
279
|
+
assert.equal(sha256(data), frame.sha256, "a manifest seal proves its own digest");
|
|
280
|
+
const manifest = JSON.parse(data.toString("utf8"));
|
|
281
|
+
assert.equal(manifest.contentHash, frame.content_hash);
|
|
282
|
+
const absent = manifest.chunks.find((chunk) => !findContent(content, {
|
|
283
|
+
resource_id: frame.resource_id,
|
|
284
|
+
kind: "chunk",
|
|
285
|
+
sha256: chunk.sha256,
|
|
286
|
+
}));
|
|
287
|
+
if (absent) {
|
|
288
|
+
socket.send(JSON.stringify({
|
|
289
|
+
type: "shared.error",
|
|
290
|
+
request_id: frame.request_id,
|
|
291
|
+
code: "entity_content_incomplete",
|
|
292
|
+
message: `manifest names absent block ${absent.sha256}`,
|
|
293
|
+
status: 409,
|
|
294
|
+
retryable: true,
|
|
295
|
+
}));
|
|
296
|
+
contentPutsInFlight -= 1;
|
|
297
|
+
return;
|
|
298
|
+
}
|
|
299
|
+
content.set(contentKey(frame), {
|
|
300
|
+
sha256: frame.sha256,
|
|
301
|
+
bytes: frame.bytes,
|
|
302
|
+
data,
|
|
303
|
+
});
|
|
304
|
+
socket.send(JSON.stringify({
|
|
305
|
+
type: "private.entity-content.stored",
|
|
306
|
+
request_id: frame.request_id,
|
|
307
|
+
content_hash: frame.content_hash,
|
|
308
|
+
bytes: frame.bytes,
|
|
309
|
+
verify_ms: 2,
|
|
310
|
+
store_ms: 3,
|
|
311
|
+
seal_ms: 4,
|
|
312
|
+
}));
|
|
313
|
+
contentPutsInFlight -= 1;
|
|
314
|
+
};
|
|
315
|
+
const admitted = contentPutGate ? contentPutGate() : Promise.resolve();
|
|
316
|
+
void admitted.then(() => {
|
|
317
|
+
if (contentPutDelayMs > 0) setTimeout(store, contentPutDelayMs);
|
|
318
|
+
else store();
|
|
319
|
+
});
|
|
320
|
+
} else if (frame.type === "private.entity-content.put-blocks") {
|
|
321
|
+
// Provisional blocks addressed by their own digests: durable at
|
|
322
|
+
// once, bound to no artifact until some manifest seals over them.
|
|
323
|
+
assert.equal(isBinary, true, "block waves use the binary WebSocket rail");
|
|
324
|
+
assert.equal(frame.data_bin instanceof Uint8Array, true);
|
|
325
|
+
binaryContentPuts += 1;
|
|
326
|
+
contentPutBatches.push({ ...frame, data_bin: undefined });
|
|
327
|
+
contentPutAttempts += frame.blocks.length;
|
|
328
|
+
putAttemptHashes.push(...frame.blocks.map((block) => String(block.sha256)));
|
|
329
|
+
if (remainingContentPutFailures > 0) {
|
|
330
|
+
remainingContentPutFailures -= 1;
|
|
331
|
+
socket.send(JSON.stringify({
|
|
332
|
+
type: "shared.error",
|
|
333
|
+
request_id: frame.request_id,
|
|
334
|
+
code: "shared_realtime_internal_error",
|
|
335
|
+
message: "transient object authority failure",
|
|
336
|
+
status: 503,
|
|
337
|
+
retryable: contentFailureRetryable,
|
|
338
|
+
}));
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
contentPutsInFlight += 1;
|
|
342
|
+
maxContentPutsInFlight = Math.max(maxContentPutsInFlight, contentPutsInFlight);
|
|
343
|
+
const decodedBatch = decodeWireBatch(frame);
|
|
344
|
+
const store = () => {
|
|
345
|
+
let offset = 0;
|
|
346
|
+
for (const block of frame.blocks) {
|
|
347
|
+
const bytes = Buffer.from(decodedBatch.subarray(offset, offset + block.bytes));
|
|
348
|
+
assert.equal(sha256(bytes), block.sha256, "a block wave is refused whole when a digest lies");
|
|
349
|
+
content.set(contentKey({ ...frame, kind: "chunk", sha256: block.sha256 }), {
|
|
350
|
+
sha256: block.sha256,
|
|
351
|
+
bytes: block.bytes,
|
|
352
|
+
data: bytes,
|
|
353
|
+
});
|
|
354
|
+
offset += block.bytes;
|
|
355
|
+
}
|
|
356
|
+
assert.equal(offset, frame.bytes, "block bytes cover the declared payload");
|
|
357
|
+
socket.send(JSON.stringify({
|
|
358
|
+
type: "private.entity-content.stored-blocks",
|
|
359
|
+
request_id: frame.request_id,
|
|
360
|
+
blocks: frame.blocks.length,
|
|
361
|
+
bytes: frame.bytes,
|
|
362
|
+
verify_ms: 2,
|
|
363
|
+
store_ms: 3,
|
|
364
|
+
}));
|
|
365
|
+
contentPutsInFlight -= 1;
|
|
366
|
+
};
|
|
367
|
+
const admitted = contentPutGate ? contentPutGate() : Promise.resolve();
|
|
368
|
+
void admitted.then(() => {
|
|
369
|
+
if (contentPutDelayMs > 0) setTimeout(store, contentPutDelayMs);
|
|
370
|
+
else store();
|
|
371
|
+
});
|
|
372
|
+
} else if (frame.type === "private.entity-content.put-cargo") {
|
|
373
|
+
assert.equal(isBinary, true, "cargo segments use the binary WebSocket rail");
|
|
374
|
+
assert.equal(frame.data_bin instanceof Uint8Array, true);
|
|
375
|
+
binaryContentPuts += 1;
|
|
376
|
+
contentCargoSegments.push({ ...frame, data_bin: undefined });
|
|
377
|
+
// One attempt per object the segment makes durable: every chunk plus
|
|
378
|
+
// each artifact's manifest — parity with the per-frame put rails.
|
|
379
|
+
contentPutAttempts += frame.artifacts
|
|
380
|
+
.reduce((total, artifact) => total + artifact.chunks.length + 1, 0);
|
|
381
|
+
for (const artifact of frame.artifacts) {
|
|
382
|
+
putAttemptHashes.push(String(artifact.content_hash));
|
|
383
|
+
putAttemptHashes.push(...artifact.chunks.map((chunk) => String(chunk.sha256)));
|
|
384
|
+
}
|
|
385
|
+
if (remainingContentPutFailures > 0) {
|
|
386
|
+
remainingContentPutFailures -= 1;
|
|
387
|
+
socket.send(JSON.stringify({
|
|
388
|
+
type: "shared.error",
|
|
389
|
+
request_id: frame.request_id,
|
|
390
|
+
code: "shared_realtime_internal_error",
|
|
391
|
+
message: "transient object authority failure",
|
|
392
|
+
status: 503,
|
|
393
|
+
retryable: contentFailureRetryable,
|
|
394
|
+
}));
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
contentPutsInFlight += 1;
|
|
398
|
+
maxContentPutsInFlight = Math.max(maxContentPutsInFlight, contentPutsInFlight);
|
|
399
|
+
const decodedCargo = decodeWireBatch(frame);
|
|
400
|
+
const store = () => {
|
|
401
|
+
const packEntries = [];
|
|
402
|
+
let offset = 0;
|
|
403
|
+
for (const artifact of frame.artifacts) {
|
|
404
|
+
for (const chunk of artifact.chunks) {
|
|
405
|
+
const bytes = Buffer.from(decodedCargo.subarray(offset, offset + chunk.bytes));
|
|
406
|
+
content.set(contentKey({ ...frame, kind: "chunk", sha256: chunk.sha256 }), {
|
|
407
|
+
sha256: chunk.sha256,
|
|
408
|
+
bytes: chunk.bytes,
|
|
409
|
+
data: bytes,
|
|
410
|
+
});
|
|
411
|
+
packEntries.push({
|
|
412
|
+
kind: "block",
|
|
413
|
+
sha256: chunk.sha256,
|
|
414
|
+
envelope: sealContentEnvelope(frame.resource_id, bytes),
|
|
415
|
+
});
|
|
416
|
+
offset += chunk.bytes;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
for (const artifact of frame.artifacts) {
|
|
420
|
+
const manifest = Buffer.from(JSON.stringify({
|
|
421
|
+
bytes: artifact.bytes,
|
|
422
|
+
chunks: artifact.chunks.map((chunk) => ({ bytes: chunk.bytes, sha256: chunk.sha256 })),
|
|
423
|
+
contentHash: artifact.content_hash,
|
|
424
|
+
contract: "amalgm-entity-content@4",
|
|
425
|
+
}), "utf8");
|
|
426
|
+
content.set(contentKey({ ...frame, kind: "manifest", content_hash: artifact.content_hash }), {
|
|
427
|
+
bytes: manifest.length,
|
|
428
|
+
data: manifest,
|
|
429
|
+
});
|
|
430
|
+
packEntries.push({
|
|
431
|
+
kind: "manifest",
|
|
432
|
+
sha256: artifact.content_hash,
|
|
433
|
+
envelope: sealContentEnvelope(frame.resource_id, manifest),
|
|
434
|
+
});
|
|
435
|
+
}
|
|
436
|
+
assert.equal(offset, frame.bytes, "cargo chunk bytes cover the declared payload");
|
|
437
|
+
// One immutable pack per segment; locations insert first-wins,
|
|
438
|
+
// exactly like the production ignore-duplicates index.
|
|
439
|
+
const packId = `pack-${packs.size}`;
|
|
440
|
+
let packOffset = 0;
|
|
441
|
+
for (const entry of packEntries) {
|
|
442
|
+
const key = `${frame.resource_id}:${entry.kind}:${entry.sha256}`;
|
|
443
|
+
if (!packLocations.has(key)) {
|
|
444
|
+
packLocations.set(key, { pack: packId, offset: packOffset, length: entry.envelope.length });
|
|
445
|
+
}
|
|
446
|
+
packOffset += entry.envelope.length;
|
|
447
|
+
}
|
|
448
|
+
packs.set(packId, Buffer.concat(packEntries.map((entry) => entry.envelope)));
|
|
449
|
+
// Distinct fixed timings so tests can prove the host folds every
|
|
450
|
+
// acknowledgement's gateway legs into its ledger, per segment.
|
|
451
|
+
socket.send(JSON.stringify({
|
|
452
|
+
type: "private.entity-content.stored-cargo",
|
|
453
|
+
request_id: frame.request_id,
|
|
454
|
+
sealed: frame.artifacts.map((artifact) => artifact.content_hash),
|
|
455
|
+
bytes: frame.bytes,
|
|
456
|
+
verify_ms: 2,
|
|
457
|
+
store_ms: 3,
|
|
458
|
+
seal_ms: 4,
|
|
459
|
+
}));
|
|
460
|
+
contentPutsInFlight -= 1;
|
|
461
|
+
};
|
|
462
|
+
const admitted = contentPutGate ? contentPutGate() : Promise.resolve();
|
|
463
|
+
void admitted.then(() => {
|
|
464
|
+
if (contentPutDelayMs > 0) setTimeout(store, contentPutDelayMs);
|
|
465
|
+
else store();
|
|
466
|
+
});
|
|
467
|
+
} else if (frame.type === "private.entity-content.sealed") {
|
|
468
|
+
// Seal evidence under the current contract: a manifest in this
|
|
469
|
+
// store. The upload planner trusts this answer alone — never graph
|
|
470
|
+
// membership — so a cutover wipe forces re-upload.
|
|
471
|
+
contentSealQueries.push({ ...frame });
|
|
472
|
+
// The seal query is the first upload frame — after capture, before
|
|
473
|
+
// any source byte is read. Tests hook it to move the ground inside
|
|
474
|
+
// exactly that window.
|
|
475
|
+
onContentSealQuery?.(frame);
|
|
476
|
+
socket.send(JSON.stringify({
|
|
477
|
+
type: "private.entity-content.sealed-result",
|
|
478
|
+
request_id: frame.request_id,
|
|
479
|
+
sealed: frame.content_hashes.filter((hash) => findContent(content, {
|
|
480
|
+
resource_id: frame.resource_id,
|
|
481
|
+
content_hash: hash,
|
|
482
|
+
kind: "manifest",
|
|
483
|
+
})),
|
|
484
|
+
// Placement, per the production axiom: a manifest location row
|
|
485
|
+
// proves the artifact's whole packed closure, because put-cargo
|
|
486
|
+
// records every location before sealing.
|
|
487
|
+
packed: frame.content_hashes.filter((hash) =>
|
|
488
|
+
packLocations.has(`${frame.resource_id}:manifest:${hash}`)),
|
|
489
|
+
}));
|
|
490
|
+
} else if (frame.type === "private.entity-content.key") {
|
|
491
|
+
contentKeyRequests.push({ ...frame });
|
|
492
|
+
socket.send(JSON.stringify({
|
|
493
|
+
type: "private.entity-content.key-material",
|
|
494
|
+
request_id: frame.request_id,
|
|
495
|
+
resource_id: frame.resource_id,
|
|
496
|
+
contract: RESOURCE_CONTRACT,
|
|
497
|
+
key_hex: resourceContentKey(frame.resource_id).toString("hex"),
|
|
498
|
+
}));
|
|
499
|
+
} else if (frame.type === "private.entity-content.refs") {
|
|
500
|
+
// Where each object's sealed envelope lives: packed objects answer
|
|
501
|
+
// their pack's URL (signed once per batch, however many members)
|
|
502
|
+
// plus the byte range; objects with no location row answer a
|
|
503
|
+
// standalone whole-object URL.
|
|
504
|
+
contentRefs.push({ ...frame });
|
|
505
|
+
const port = server.address().port;
|
|
506
|
+
const urls = [];
|
|
507
|
+
const urlIndex = new Map();
|
|
508
|
+
const indexOf = (path) => {
|
|
509
|
+
let index = urlIndex.get(path);
|
|
510
|
+
if (index === undefined) {
|
|
511
|
+
index = urls.length;
|
|
512
|
+
urlIndex.set(path, index);
|
|
513
|
+
urls.push(`http://127.0.0.1:${port}${path}`);
|
|
514
|
+
}
|
|
515
|
+
return index;
|
|
516
|
+
};
|
|
517
|
+
const refs = frame.objects.map((object) => {
|
|
518
|
+
const packed = packLocations.get(`${frame.resource_id}:${object.kind}:${object.sha256}`);
|
|
519
|
+
if (packed) {
|
|
520
|
+
return {
|
|
521
|
+
url: indexOf(`/direct-pack/${encodeURIComponent(frame.resource_id)}/${packed.pack}`),
|
|
522
|
+
offset: packed.offset,
|
|
523
|
+
length: packed.length,
|
|
524
|
+
};
|
|
525
|
+
}
|
|
526
|
+
return {
|
|
527
|
+
url: indexOf([
|
|
528
|
+
"/direct-content",
|
|
529
|
+
encodeURIComponent(frame.resource_id),
|
|
530
|
+
object.kind,
|
|
531
|
+
object.sha256,
|
|
532
|
+
].join("/")),
|
|
533
|
+
};
|
|
534
|
+
});
|
|
535
|
+
socket.send(JSON.stringify({
|
|
536
|
+
type: "private.entity-content.refs-result",
|
|
537
|
+
request_id: frame.request_id,
|
|
538
|
+
resource_id: frame.resource_id,
|
|
539
|
+
urls,
|
|
540
|
+
refs,
|
|
541
|
+
expires_in_seconds: 300,
|
|
542
|
+
}));
|
|
543
|
+
} else if (frame.type === "shared.mutation.submit") {
|
|
544
|
+
if (rejectMutationSubmits) {
|
|
545
|
+
socket.send(JSON.stringify({
|
|
546
|
+
type: "shared.error",
|
|
547
|
+
request_id: frame.request_id,
|
|
548
|
+
code: "shared_realtime_internal_error",
|
|
549
|
+
message: "synthetic graph commit failure",
|
|
550
|
+
status: 503,
|
|
551
|
+
retryable: false,
|
|
552
|
+
}));
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
if (remainingMutationSubmitFailures > 0) {
|
|
556
|
+
remainingMutationSubmitFailures -= 1;
|
|
557
|
+
socket.send(JSON.stringify({
|
|
558
|
+
type: "shared.error",
|
|
559
|
+
request_id: frame.request_id,
|
|
560
|
+
code: "shared_database_serialization_failure",
|
|
561
|
+
message: "synthetic transient graph commit failure",
|
|
562
|
+
status: 503,
|
|
563
|
+
retryable: true,
|
|
564
|
+
}));
|
|
565
|
+
return;
|
|
566
|
+
}
|
|
567
|
+
const envelope = frame.envelope;
|
|
568
|
+
// Every operation travels inline now: a graph head names pages instead
|
|
569
|
+
// of carrying records, so it is small by construction.
|
|
570
|
+
assert.ok(
|
|
571
|
+
Buffer.byteLength(JSON.stringify(envelope.operation), "utf8") <= 8 * 1024 * 1024,
|
|
572
|
+
"an inline operation obeys the per-message law",
|
|
573
|
+
);
|
|
574
|
+
// The production commit RPC is idempotent on mutation_id: a replayed
|
|
575
|
+
// known mutation returns its original acknowledgement as a duplicate
|
|
576
|
+
// instead of committing twice.
|
|
577
|
+
const committed = committedMutationAcks.get(envelope.mutationId);
|
|
578
|
+
if (committed) {
|
|
579
|
+
socket.send(JSON.stringify({
|
|
580
|
+
...committed,
|
|
581
|
+
request_id: frame.request_id,
|
|
582
|
+
duplicate: true,
|
|
583
|
+
}));
|
|
584
|
+
return;
|
|
585
|
+
}
|
|
586
|
+
assert.equal(envelope.baseVersion, current.version);
|
|
587
|
+
assert.equal(envelope.authorityEpoch, current.authorityEpoch);
|
|
588
|
+
assert.equal(envelope.deliveryWatermark, deliverySequence,
|
|
589
|
+
"a graph publication names the complete current delivery prefix");
|
|
590
|
+
graphMutations.push(envelope);
|
|
591
|
+
const base = Buffer.from(current.snapshotB64, "base64");
|
|
592
|
+
const baseIsHead = base.toString("utf8", 0, 128).includes(`"kind":"${GRAPH_HEAD_KIND}"`);
|
|
593
|
+
let snapshotBytes;
|
|
594
|
+
let rootRecords;
|
|
595
|
+
if (envelope.operationKind === GRAPH_HEAD_OPERATION_KIND) {
|
|
596
|
+
const committed = commitGraphHead(envelope, base, baseIsHead, current.rootRecords, content, sha256);
|
|
597
|
+
for (const root of committed.head.roots) {
|
|
598
|
+
for (const page of root.pages) graphPageHashes.add(String(page.sha256));
|
|
599
|
+
}
|
|
600
|
+
snapshotBytes = Buffer.from(stableJson(committed.head), "utf8");
|
|
601
|
+
rootRecords = committed.rootRecords;
|
|
602
|
+
} else if (envelope.operationKind === "entity.roots.replace") {
|
|
603
|
+
// The transition tail only: once a head is committed, the records
|
|
604
|
+
// shape can never return, so a late root replacement is refused
|
|
605
|
+
// rather than folded into a graph it cannot describe.
|
|
606
|
+
assert.equal(baseIsHead, false,
|
|
607
|
+
"a root replacement may not follow a committed graph head");
|
|
608
|
+
const replacement = rootReplacementFromRecords(
|
|
609
|
+
envelope.operation.rootIds,
|
|
610
|
+
envelope.operation.records,
|
|
611
|
+
);
|
|
612
|
+
const prior = snapshotFromRecords(JSON.parse(base.toString("utf8")).records);
|
|
613
|
+
const records = new Map(prior.records.map((record) => [record.uuid, record]));
|
|
614
|
+
const roots = new Set(replacement.rootIds);
|
|
615
|
+
const rootMemo = new Map();
|
|
616
|
+
const rootOf = (record) => {
|
|
617
|
+
if (rootMemo.has(record.uuid)) return rootMemo.get(record.uuid);
|
|
618
|
+
const root = record.parentUUID === null ? record.uuid : rootOf(records.get(record.parentUUID));
|
|
619
|
+
rootMemo.set(record.uuid, root);
|
|
620
|
+
return root;
|
|
621
|
+
};
|
|
622
|
+
const priorRoots = prior.records.map((record) => [record.uuid, rootOf(record)]);
|
|
623
|
+
for (const [uuid, root] of priorRoots) {
|
|
624
|
+
if (roots.has(root)) records.delete(uuid);
|
|
625
|
+
}
|
|
626
|
+
for (const record of replacement.records) records.set(record.uuid, record);
|
|
627
|
+
const folded = snapshotFromRecords([...records.values()]);
|
|
628
|
+
snapshotBytes = Buffer.from(JSON.stringify(folded));
|
|
629
|
+
rootRecords = recordsByRoot(folded.records);
|
|
630
|
+
} else {
|
|
631
|
+
assert.fail(`unsupported fake entity graph operation ${envelope.operationKind}`);
|
|
632
|
+
}
|
|
633
|
+
current = {
|
|
634
|
+
...current,
|
|
635
|
+
snapshotB64: snapshotBytes.toString("base64"),
|
|
636
|
+
checksum: sha256(snapshotBytes),
|
|
637
|
+
rootRecords,
|
|
638
|
+
version: current.version + 1,
|
|
639
|
+
deliveryWatermark: envelope.deliveryWatermark,
|
|
640
|
+
};
|
|
641
|
+
const ack = {
|
|
642
|
+
type: "shared.mutation.ack",
|
|
643
|
+
request_id: frame.request_id,
|
|
644
|
+
resource_id: current.resourceId,
|
|
645
|
+
mutation_id: envelope.mutationId,
|
|
646
|
+
version: current.version,
|
|
647
|
+
authority_epoch: current.authorityEpoch,
|
|
648
|
+
committed_at: new Date().toISOString(),
|
|
649
|
+
delivery_watermark: current.deliveryWatermark,
|
|
650
|
+
duplicate: false,
|
|
651
|
+
};
|
|
652
|
+
committedMutationAcks.set(envelope.mutationId, ack);
|
|
653
|
+
if (dropMutationAcks > 0) {
|
|
654
|
+
// The commit landed; the acknowledgement is lost with the socket.
|
|
655
|
+
dropMutationAcks -= 1;
|
|
656
|
+
socket.close(1011, "synthetic acknowledgement loss");
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
socket.send(JSON.stringify(ack));
|
|
660
|
+
} else if (frame.type === "entity.record.submit") {
|
|
661
|
+
const submitted = decodeLocalEntityRecord(frame.record_json);
|
|
662
|
+
assert.equal(submitted.authorityId, current.resourceId);
|
|
663
|
+
if (frame.cargo !== undefined) {
|
|
664
|
+
inlineCargoSubmits += 1;
|
|
665
|
+
const cargo = decodeInlineCargo(
|
|
666
|
+
frame.cargo,
|
|
667
|
+
frame.data_bin,
|
|
668
|
+
new Set(submitted.change.cargo.map((item) => item.contentHash)),
|
|
669
|
+
sha256,
|
|
670
|
+
);
|
|
671
|
+
for (const item of cargo) {
|
|
672
|
+
item.blocks.forEach((block, index) => {
|
|
673
|
+
content.set(contentKey({
|
|
674
|
+
resource_id: submitted.authorityId,
|
|
675
|
+
kind: "chunk",
|
|
676
|
+
sha256: item.manifest.chunks[index].sha256,
|
|
677
|
+
}), {
|
|
678
|
+
sha256: item.manifest.chunks[index].sha256,
|
|
679
|
+
bytes: block.byteLength,
|
|
680
|
+
data: Buffer.from(block),
|
|
681
|
+
});
|
|
682
|
+
});
|
|
683
|
+
const manifestBytes = Buffer.from(stableJson(item.manifest), "utf8");
|
|
684
|
+
content.set(contentKey({
|
|
685
|
+
resource_id: submitted.authorityId,
|
|
686
|
+
content_hash: item.manifest.contentHash,
|
|
687
|
+
kind: "manifest",
|
|
688
|
+
}), {
|
|
689
|
+
sha256: sha256(manifestBytes),
|
|
690
|
+
bytes: manifestBytes.byteLength,
|
|
691
|
+
data: manifestBytes,
|
|
692
|
+
});
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
// Cargo-before-record, exactly as the production authority enforces
|
|
696
|
+
// it: every cargo head must be sealed before the record is accepted.
|
|
697
|
+
const unsealed = submitted.change.cargo.find((item) => !findContent(content, {
|
|
698
|
+
resource_id: submitted.authorityId,
|
|
699
|
+
content_hash: item.contentHash,
|
|
700
|
+
kind: "manifest",
|
|
701
|
+
}));
|
|
702
|
+
if (unsealed) {
|
|
703
|
+
socket.send(JSON.stringify({
|
|
704
|
+
type: "shared.error",
|
|
705
|
+
request_id: frame.request_id,
|
|
706
|
+
code: "entity_record_cargo_missing",
|
|
707
|
+
message: "entity record cargo is incomplete",
|
|
708
|
+
status: 409,
|
|
709
|
+
retryable: true,
|
|
710
|
+
}));
|
|
711
|
+
return;
|
|
712
|
+
}
|
|
713
|
+
const key = `${submitted.entityId}\u0000${submitted.mutationId}`;
|
|
714
|
+
const decision = acceptEntityRecord({
|
|
715
|
+
submitted,
|
|
716
|
+
existing: entityDedup.get(key) ?? null,
|
|
717
|
+
headSequence: entityHeads.get(submitted.entityId) ?? 0,
|
|
718
|
+
actorId: "test-user",
|
|
719
|
+
originDeviceId: "test-computer",
|
|
720
|
+
acceptedAt: Date.now(),
|
|
721
|
+
});
|
|
722
|
+
if (decision.kind === "append") {
|
|
723
|
+
deliverySequence += 1;
|
|
724
|
+
entityHeads.set(submitted.entityId, decision.record.globalSequence);
|
|
725
|
+
entityDedup.set(key, decision.record);
|
|
726
|
+
entityRecords.push({ delivery_sequence: deliverySequence, record: decision.record });
|
|
727
|
+
for (const [peer, authorities] of entitySubscriptions) {
|
|
728
|
+
if (authorities.has(submitted.authorityId) && peer.readyState === WebSocket.OPEN) {
|
|
729
|
+
peer.send(JSON.stringify({
|
|
730
|
+
type: "entity.record.available",
|
|
731
|
+
authority_id: submitted.authorityId,
|
|
732
|
+
}));
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
socket.send(JSON.stringify({
|
|
737
|
+
type: "entity.record.ack",
|
|
738
|
+
request_id: frame.request_id,
|
|
739
|
+
record_json: encodeEntityRecord(decision.record),
|
|
740
|
+
duplicate: decision.kind === "duplicate",
|
|
741
|
+
}));
|
|
742
|
+
} else if (frame.type === "entity.record.subscribe") {
|
|
743
|
+
entitySubscribeRequests += 1;
|
|
744
|
+
entitySubscriptions.get(socket).add(frame.authority_id);
|
|
745
|
+
const records = entityRecords
|
|
746
|
+
.filter((item) => item.record.authorityId === frame.authority_id
|
|
747
|
+
&& item.delivery_sequence > frame.after_delivery_sequence)
|
|
748
|
+
.slice(0, frame.limit)
|
|
749
|
+
.map((item) => ({
|
|
750
|
+
delivery_sequence: item.delivery_sequence,
|
|
751
|
+
record_json: encodeEntityRecord(item.record),
|
|
752
|
+
}));
|
|
753
|
+
socket.send(JSON.stringify({
|
|
754
|
+
type: "entity.record.subscribed",
|
|
755
|
+
request_id: frame.request_id,
|
|
756
|
+
records,
|
|
757
|
+
}));
|
|
758
|
+
} else if (frame.type === "entity.record.backfill") {
|
|
759
|
+
const records = entityRecords
|
|
760
|
+
.filter((item) => item.record.authorityId === frame.authority_id
|
|
761
|
+
&& item.record.entityId === frame.entity_id
|
|
762
|
+
&& item.record.globalSequence > frame.after_global_sequence
|
|
763
|
+
&& item.record.globalSequence <= frame.through_global_sequence)
|
|
764
|
+
.sort((left, right) => left.record.globalSequence - right.record.globalSequence)
|
|
765
|
+
.map((item) => ({
|
|
766
|
+
delivery_sequence: item.delivery_sequence,
|
|
767
|
+
record_json: encodeEntityRecord(item.record),
|
|
768
|
+
}));
|
|
769
|
+
socket.send(JSON.stringify({
|
|
770
|
+
type: "entity.record.backfill-result",
|
|
771
|
+
request_id: frame.request_id,
|
|
772
|
+
records,
|
|
773
|
+
}));
|
|
774
|
+
}
|
|
775
|
+
});
|
|
776
|
+
});
|
|
777
|
+
return {
|
|
778
|
+
server,
|
|
779
|
+
sockets,
|
|
780
|
+
get current() { return current; },
|
|
781
|
+
/** Every accepted entity record, in acceptance order. */
|
|
782
|
+
get entityRecords() { return entityRecords.map((item) => item.record); },
|
|
783
|
+
get contentCount() { return content.size; },
|
|
784
|
+
get contentBytes() {
|
|
785
|
+
return [...content.values()].reduce((total, value) => total + value.data.length, 0);
|
|
786
|
+
},
|
|
787
|
+
contentArtifact(contentHash) {
|
|
788
|
+
if (!current) return null;
|
|
789
|
+
const manifest = findContent(content, {
|
|
790
|
+
resource_id: current.resourceId,
|
|
791
|
+
content_hash: contentHash,
|
|
792
|
+
kind: "manifest",
|
|
793
|
+
});
|
|
794
|
+
if (!manifest) return null;
|
|
795
|
+
const declaration = JSON.parse(manifest.data.toString("utf8"));
|
|
796
|
+
return Buffer.concat(declaration.chunks.map((chunk) => {
|
|
797
|
+
const found = findContent(content, {
|
|
798
|
+
resource_id: current.resourceId,
|
|
799
|
+
content_hash: contentHash,
|
|
800
|
+
kind: "chunk",
|
|
801
|
+
sha256: chunk.sha256,
|
|
802
|
+
});
|
|
803
|
+
if (!found) throw new Error(`missing fake authority block ${chunk.sha256}`);
|
|
804
|
+
return found.data;
|
|
805
|
+
}));
|
|
806
|
+
},
|
|
807
|
+
get contentPutAttempts() { return contentPutAttempts; },
|
|
808
|
+
/** Put attempts that moved file cargo. A graph that changed always seals
|
|
809
|
+
* new pages, so only this number answers whether the FILES moved. */
|
|
810
|
+
get cargoPutAttempts() {
|
|
811
|
+
return putAttemptHashes.filter((sha256) => !graphPageHashes.has(sha256)).length;
|
|
812
|
+
},
|
|
813
|
+
failNextContentPuts(count = 1, retryable = false) {
|
|
814
|
+
remainingContentPutFailures = count;
|
|
815
|
+
contentFailureRetryable = retryable;
|
|
816
|
+
},
|
|
817
|
+
setContentFetchGate(gate) {
|
|
818
|
+
activeContentFetchGate = gate ?? null;
|
|
819
|
+
},
|
|
820
|
+
rejectMutationSubmits(reject) {
|
|
821
|
+
rejectMutationSubmits = reject;
|
|
822
|
+
},
|
|
823
|
+
interceptContentSealQueries(handler) {
|
|
824
|
+
onContentSealQuery = handler;
|
|
825
|
+
},
|
|
826
|
+
failNextMutationSubmits(count = 1) {
|
|
827
|
+
remainingMutationSubmitFailures = count;
|
|
828
|
+
},
|
|
829
|
+
dropNextMutationAcks(count = 1) {
|
|
830
|
+
dropMutationAcks = count;
|
|
831
|
+
},
|
|
832
|
+
get contentInventories() { return [...contentInventories]; },
|
|
833
|
+
get contentSealQueries() { return [...contentSealQueries]; },
|
|
834
|
+
/** Simulate a content-contract cutover: the cloud graph keeps every
|
|
835
|
+
* record, but no cargo remains sealed under the current contract. The
|
|
836
|
+
* graph's own pages survive — they are what the authority stores, not
|
|
837
|
+
* cargo it seals on a machine's behalf. */
|
|
838
|
+
wipeSealedContent() {
|
|
839
|
+
for (const key of content.keys()) {
|
|
840
|
+
const [, second, third] = key.split(":");
|
|
841
|
+
if (graphPageHashes.has(second) || graphPageHashes.has(third)) continue;
|
|
842
|
+
content.delete(key);
|
|
843
|
+
}
|
|
844
|
+
// The pack index answers downloads; a wiped seal must not leave a
|
|
845
|
+
// location row resolving cargo the contract no longer holds.
|
|
846
|
+
for (const key of packLocations.keys()) {
|
|
847
|
+
const [, , sha] = key.split(":");
|
|
848
|
+
if (graphPageHashes.has(sha)) continue;
|
|
849
|
+
packLocations.delete(key);
|
|
850
|
+
}
|
|
851
|
+
},
|
|
852
|
+
/** Fabricate legacy standalone placement: the artifact is sealed — its
|
|
853
|
+
* manifest and block exist and downloads serve them as standalone
|
|
854
|
+
* objects — but no pack location rows exist, so the seal inventory
|
|
855
|
+
* answers sealed-but-not-packed. This is exactly what pre-pack uploads
|
|
856
|
+
* and inline Send cargo left behind. Small artifacts are single-chunk
|
|
857
|
+
* by construction (one chunk holds anything up to the chunk cap). */
|
|
858
|
+
adoptStandaloneArtifact(bytes) {
|
|
859
|
+
if (!current) throw new Error("no resource is registered yet");
|
|
860
|
+
const data = Buffer.from(bytes);
|
|
861
|
+
const contentHash = sha256(data);
|
|
862
|
+
content.set(contentKey({
|
|
863
|
+
resource_id: current.resourceId,
|
|
864
|
+
kind: "chunk",
|
|
865
|
+
sha256: contentHash,
|
|
866
|
+
}), { sha256: contentHash, bytes: data.length, data });
|
|
867
|
+
const manifest = Buffer.from(JSON.stringify({
|
|
868
|
+
bytes: data.length,
|
|
869
|
+
chunks: [{ bytes: data.length, sha256: contentHash }],
|
|
870
|
+
contentHash,
|
|
871
|
+
contract: "amalgm-entity-content@4",
|
|
872
|
+
}), "utf8");
|
|
873
|
+
content.set(contentKey({
|
|
874
|
+
resource_id: current.resourceId,
|
|
875
|
+
kind: "manifest",
|
|
876
|
+
content_hash: contentHash,
|
|
877
|
+
}), { bytes: manifest.length, data: manifest });
|
|
878
|
+
return contentHash;
|
|
879
|
+
},
|
|
880
|
+
/** Remove only physical location rows for one sealed artifact. Its
|
|
881
|
+
* immutable manifest and blocks remain readable as standalone objects. */
|
|
882
|
+
makeArtifactStandalone(contentHash) {
|
|
883
|
+
if (!current) throw new Error("no resource is registered yet");
|
|
884
|
+
const manifest = findContent(content, {
|
|
885
|
+
resource_id: current.resourceId,
|
|
886
|
+
content_hash: contentHash,
|
|
887
|
+
kind: "manifest",
|
|
888
|
+
});
|
|
889
|
+
if (!manifest) throw new Error(`no sealed manifest ${contentHash}`);
|
|
890
|
+
const declaration = JSON.parse(manifest.data.toString("utf8"));
|
|
891
|
+
packLocations.delete(`${current.resourceId}:manifest:${contentHash}`);
|
|
892
|
+
for (const chunk of declaration.chunks) {
|
|
893
|
+
packLocations.delete(`${current.resourceId}:block:${chunk.sha256}`);
|
|
894
|
+
}
|
|
895
|
+
},
|
|
896
|
+
/** Move one sealed artifact's whole closure into a new immutable pack.
|
|
897
|
+
* This models a legal standalone-to-pack placement convergence while a
|
|
898
|
+
* reader still holds refs from the earlier physical plan. */
|
|
899
|
+
moveArtifactToPack(contentHash) {
|
|
900
|
+
if (!current) throw new Error("no resource is registered yet");
|
|
901
|
+
const manifest = findContent(content, {
|
|
902
|
+
resource_id: current.resourceId,
|
|
903
|
+
content_hash: contentHash,
|
|
904
|
+
kind: "manifest",
|
|
905
|
+
});
|
|
906
|
+
if (!manifest) throw new Error(`no sealed manifest ${contentHash}`);
|
|
907
|
+
const declaration = JSON.parse(manifest.data.toString("utf8"));
|
|
908
|
+
const entries = declaration.chunks.map((chunk) => {
|
|
909
|
+
const found = findContent(content, {
|
|
910
|
+
resource_id: current.resourceId,
|
|
911
|
+
kind: "chunk",
|
|
912
|
+
sha256: chunk.sha256,
|
|
913
|
+
});
|
|
914
|
+
if (!found) throw new Error(`no sealed block ${chunk.sha256}`);
|
|
915
|
+
return {
|
|
916
|
+
kind: "block",
|
|
917
|
+
sha256: chunk.sha256,
|
|
918
|
+
envelope: sealContentEnvelope(current.resourceId, found.data),
|
|
919
|
+
};
|
|
920
|
+
});
|
|
921
|
+
entries.push({
|
|
922
|
+
kind: "manifest",
|
|
923
|
+
sha256: contentHash,
|
|
924
|
+
envelope: sealContentEnvelope(current.resourceId, manifest.data),
|
|
925
|
+
});
|
|
926
|
+
const packId = `moved-pack-${packs.size}`;
|
|
927
|
+
let offset = 0;
|
|
928
|
+
for (const entry of entries) {
|
|
929
|
+
packLocations.set(`${current.resourceId}:${entry.kind}:${entry.sha256}`, {
|
|
930
|
+
pack: packId,
|
|
931
|
+
offset,
|
|
932
|
+
length: entry.envelope.length,
|
|
933
|
+
});
|
|
934
|
+
offset += entry.envelope.length;
|
|
935
|
+
}
|
|
936
|
+
packs.set(packId, Buffer.concat(entries.map((entry) => entry.envelope)));
|
|
937
|
+
return packId;
|
|
938
|
+
},
|
|
939
|
+
get contentPutBatches() { return [...contentPutBatches]; },
|
|
940
|
+
get contentCargoSegments() { return [...contentCargoSegments]; },
|
|
941
|
+
get contentKeyRequests() { return [...contentKeyRequests]; },
|
|
942
|
+
get contentRefs() { return [...contentRefs]; },
|
|
943
|
+
get contentFetches() { return [...contentFetches]; },
|
|
944
|
+
get contentPacks() { return new Map(packs); },
|
|
945
|
+
get binaryContentPuts() { return binaryContentPuts; },
|
|
946
|
+
get inlineCargoSubmits() { return inlineCargoSubmits; },
|
|
947
|
+
/** Content and head frames in the exact order they arrived. */
|
|
948
|
+
get frameOrder() { return [...frameOrder]; },
|
|
949
|
+
get maxContentPutsInFlight() { return maxContentPutsInFlight; },
|
|
950
|
+
get entityRecords() {
|
|
951
|
+
return entityRecords.map((item) => ({
|
|
952
|
+
deliverySequence: item.delivery_sequence,
|
|
953
|
+
record: item.record,
|
|
954
|
+
}));
|
|
955
|
+
},
|
|
956
|
+
get entitySubscribeRequests() { return entitySubscribeRequests; },
|
|
957
|
+
get graphMutations() { return [...graphMutations]; },
|
|
958
|
+
/** Every record of the committed registry graph. A head commit names its
|
|
959
|
+
* roots as immutable pages, so this is what the authority actually holds
|
|
960
|
+
* after assembling them — the honest replacement for reading records out
|
|
961
|
+
* of an operation that no longer carries any. */
|
|
962
|
+
get graphRecords() {
|
|
963
|
+
return current ? [...current.rootRecords.values()].flat() : [];
|
|
964
|
+
},
|
|
965
|
+
};
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
function ready(requestId, current, duplicate) {
|
|
969
|
+
return {
|
|
970
|
+
type: "private.resource.ready",
|
|
971
|
+
request_id: requestId,
|
|
972
|
+
resource_id: current.resourceId,
|
|
973
|
+
authority_epoch: current.authorityEpoch,
|
|
974
|
+
head_version: current.version,
|
|
975
|
+
snapshot_version: current.version,
|
|
976
|
+
snapshot_checksum: current.checksum,
|
|
977
|
+
delivery_watermark: current.deliveryWatermark,
|
|
978
|
+
snapshot_b64: current.snapshotB64,
|
|
979
|
+
contract: "amalgm-entity-registry@3",
|
|
980
|
+
schema_version: 1,
|
|
981
|
+
duplicate,
|
|
982
|
+
};
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/** Prove one graph-head commit the way the gateway does, and answer the head
|
|
986
|
+
* that becomes the stored snapshot. Every root the head keeps unchanged must
|
|
987
|
+
* carry the committed head's exact page list — the structural proof the
|
|
988
|
+
* publisher folded on the base it claims — every base root the head drops
|
|
989
|
+
* must be declared replaced, and every replaced root's pages must be sealed,
|
|
990
|
+
* readable, and re-prove their own identity and subtree laws. A records-shape
|
|
991
|
+
* base carries no page identities, so the first head must replace every root. */
|
|
992
|
+
function commitGraphHead(envelope, base, baseIsHead, baseRecords, content, sha256Hex) {
|
|
993
|
+
const { head, replacedRootIds } = envelope.operation;
|
|
994
|
+
const replaced = new Set(replacedRootIds);
|
|
995
|
+
const basePages = new Map();
|
|
996
|
+
if (baseIsHead) {
|
|
997
|
+
for (const root of parseGraphHead(base).roots) basePages.set(root.rootId, root.pages);
|
|
998
|
+
} else {
|
|
999
|
+
for (const record of JSON.parse(base.toString("utf8")).records) {
|
|
1000
|
+
if (record.parentUUID === null) basePages.set(record.uuid, null);
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
const headRoots = new Set(head.roots.map((root) => root.rootId));
|
|
1004
|
+
for (const rootId of basePages.keys()) {
|
|
1005
|
+
assert.ok(headRoots.has(rootId) || replaced.has(rootId),
|
|
1006
|
+
`graph head drops root ${rootId} without replacing it`);
|
|
1007
|
+
}
|
|
1008
|
+
const rootRecords = new Map();
|
|
1009
|
+
for (const root of head.roots) {
|
|
1010
|
+
if (!replaced.has(root.rootId)) {
|
|
1011
|
+
const pages = basePages.get(root.rootId);
|
|
1012
|
+
assert.ok(pages != null,
|
|
1013
|
+
`graph head root ${root.rootId} must be replaced: the base does not carry its pages`);
|
|
1014
|
+
assert.ok(sameRootPages(pages, root.pages),
|
|
1015
|
+
`graph head root ${root.rootId} pages do not match the committed head`);
|
|
1016
|
+
rootRecords.set(root.rootId, baseRecords.get(root.rootId) ?? []);
|
|
1017
|
+
continue;
|
|
1018
|
+
}
|
|
1019
|
+
const bytes = root.pages.map((page) => {
|
|
1020
|
+
assert.ok(
|
|
1021
|
+
findContent(content, {
|
|
1022
|
+
resource_id: envelope.resourceId,
|
|
1023
|
+
content_hash: page.sha256,
|
|
1024
|
+
kind: "manifest",
|
|
1025
|
+
}),
|
|
1026
|
+
`graph page ${page.sha256} is not sealed`,
|
|
1027
|
+
);
|
|
1028
|
+
const block = findContent(content, {
|
|
1029
|
+
resource_id: envelope.resourceId, kind: "chunk", sha256: page.sha256,
|
|
1030
|
+
});
|
|
1031
|
+
assert.ok(block, `graph page ${page.sha256} is sealed but unreadable`);
|
|
1032
|
+
return block.data;
|
|
1033
|
+
});
|
|
1034
|
+
const assembled = root.pages.flatMap((page, index) =>
|
|
1035
|
+
parseGraphPage(page, bytes[index], sha256Hex));
|
|
1036
|
+
rootRecords.set(
|
|
1037
|
+
root.rootId,
|
|
1038
|
+
proveRootPagination(root.rootId, assembled, root.pages, sha256Hex),
|
|
1039
|
+
);
|
|
1040
|
+
}
|
|
1041
|
+
return { head, rootRecords };
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
function contentKey(frame) {
|
|
1045
|
+
if (frame.kind === "chunk") {
|
|
1046
|
+
return [frame.resource_id, "block", frame.sha256].join(":");
|
|
1047
|
+
}
|
|
1048
|
+
return [frame.resource_id, frame.content_hash, "manifest"].join(":");
|
|
1049
|
+
}
|
|
1050
|
+
|
|
1051
|
+
function findContent(content, frame) {
|
|
1052
|
+
return content.get(contentKey(frame));
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
function decodeWireBatch(frame) {
|
|
1056
|
+
const wireBytes = Buffer.from(frame.data_bin);
|
|
1057
|
+
assert.equal(wireBytes.byteLength, frame.wire_bytes ?? frame.bytes);
|
|
1058
|
+
const decoded = frame.content_encoding === undefined
|
|
1059
|
+
? wireBytes
|
|
1060
|
+
: frame.content_encoding === "deflate-raw"
|
|
1061
|
+
? inflateRawSync(wireBytes, { maxOutputLength: frame.bytes })
|
|
1062
|
+
: assert.fail(`unknown content encoding ${frame.content_encoding}`);
|
|
1063
|
+
assert.equal(decoded.byteLength, frame.bytes);
|
|
1064
|
+
return decoded;
|
|
1065
|
+
}
|