amalgm 0.1.246 → 0.1.247
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/lib/cli.js +92 -4
- package/lib/shared-realtime-tunnel.js +82 -19
- package/package.json +1 -1
- package/runtime/scripts/amalgm-mcp/automations/deliveries.js +276 -0
- package/runtime/scripts/amalgm-mcp/automations/rest.js +51 -0
- package/runtime/scripts/amalgm-mcp/automations/scheduler.js +11 -67
- package/runtime/scripts/amalgm-mcp/automations/store.js +67 -20
- package/runtime/scripts/amalgm-mcp/events/ingress.js +69 -215
- package/runtime/scripts/amalgm-mcp/events/processor.js +189 -0
- package/runtime/scripts/amalgm-mcp/lib/email-embeds.js +41 -0
- package/runtime/scripts/amalgm-mcp/lib/email-md.js +107 -1
- package/runtime/scripts/amalgm-mcp/lib/generated/email-render.json +10 -2
- package/runtime/scripts/amalgm-mcp/observer/README.md +8 -6
- package/runtime/scripts/amalgm-mcp/registration/entity-cloud.js +262 -22
- package/runtime/scripts/amalgm-mcp/registration/entity-content.js +86 -0
- package/runtime/scripts/amalgm-mcp/registration/index.js +4 -1
- package/runtime/scripts/amalgm-mcp/registration/service.js +102 -0
- package/runtime/scripts/amalgm-mcp/registration/tree.js +19 -5
- package/runtime/scripts/amalgm-mcp/registry/index.js +16 -1
- package/runtime/scripts/amalgm-mcp/registry/store.js +58 -7
- package/runtime/scripts/amalgm-mcp/server/routes/automations.js +8 -0
- package/runtime/scripts/amalgm-mcp/server/routes/entities.js +56 -0
- package/runtime/scripts/amalgm-mcp/server/routes/workspace.js +14 -0
- package/runtime/scripts/amalgm-mcp/tests/adapters.test.js +1 -1
- package/runtime/scripts/amalgm-mcp/tests/doorbell.matrix.watch.test.js +2 -2
- package/runtime/scripts/amalgm-mcp/tests/email-render-goldens/citations.html +5 -5
- package/runtime/scripts/amalgm-mcp/tests/email-render-goldens/links.html +3 -3
- package/runtime/scripts/amalgm-mcp/tests/email-render-goldens/notification-shell.html +1 -1
- package/runtime/scripts/amalgm-mcp/tests/email-render-goldens/share-invite.html +13 -0
- package/runtime/scripts/amalgm-mcp/tests/email-render-goldens/text.html +1 -1
- package/runtime/scripts/amalgm-mcp/tests/email-render.test.js +17 -1
- package/runtime/scripts/amalgm-mcp/tests/entity-cloud.test.js +189 -0
- package/runtime/scripts/amalgm-mcp/tests/entity-content.test.js +126 -0
- package/runtime/scripts/amalgm-mcp/tests/entity-materialization.test.js +118 -0
- package/runtime/scripts/amalgm-mcp/tests/entity.registry.test.js +126 -11
- package/runtime/scripts/amalgm-mcp/tests/entity.rig.js +20 -6
- package/runtime/scripts/amalgm-mcp/tests/fake-watch.js +8 -0
- package/runtime/scripts/amalgm-mcp/tests/observer.rig.js +8 -1
- package/runtime/scripts/amalgm-mcp/tests/registration.service.test.js +630 -102
- package/runtime/scripts/amalgm-mcp/tests/registration.test.js +45 -10
- package/runtime/scripts/amalgm-mcp/tests/workspace-materialize.test.js +54 -0
- package/runtime/scripts/amalgm-mcp/workspace/materialize.js +292 -0
- package/runtime/scripts/chat-core/contract.js +21 -3
- package/runtime/scripts/chat-core/engine.js +8 -0
- package/runtime/scripts/chat-core/server.js +11 -1
- package/runtime/scripts/chat-core/tests/workspace-preflight.test.js +116 -0
- package/runtime/scripts/chat-core/workspace-preflight.js +89 -0
- package/runtime/scripts/local-gateway.js +0 -1
|
@@ -167,6 +167,121 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
167
167
|
);
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
/** The catalog is the cloud graph imported at one verified snapshot head.
|
|
171
|
+
* It deliberately lives outside `entities`: a catalog record has no local
|
|
172
|
+
* address, evidence, or watcher claim until `amalgm add` binds it. */
|
|
173
|
+
function replaceCatalog({ resourceId, authorityEpoch, headVersion, records }) {
|
|
174
|
+
const normalized = snapshotFromRecords(records).records;
|
|
175
|
+
db.transaction(() => {
|
|
176
|
+
db.prepare('DELETE FROM entity_cloud_catalog WHERE resource_id = ?').run(resourceId);
|
|
177
|
+
const insert = db.prepare(`
|
|
178
|
+
INSERT INTO entity_cloud_catalog (
|
|
179
|
+
resource_id, entity_uuid, record_json, snapshot_epoch,
|
|
180
|
+
snapshot_version, updated_at
|
|
181
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
182
|
+
`);
|
|
183
|
+
for (const record of normalized) {
|
|
184
|
+
insert.run(resourceId, record.uuid, stableJson(record), authorityEpoch, headVersion, now());
|
|
185
|
+
}
|
|
186
|
+
// This marker is intentionally separate from the rows it describes.
|
|
187
|
+
// A later mutation may update an individual catalog row so live state
|
|
188
|
+
// can be inspected, but that must never make the whole catalog look
|
|
189
|
+
// like a newly compacted snapshot that is safe to materialize.
|
|
190
|
+
db.prepare(`
|
|
191
|
+
INSERT INTO entity_cloud_catalog_state (
|
|
192
|
+
singleton, resource_id, authority_epoch, snapshot_version, updated_at
|
|
193
|
+
) VALUES (1, ?, ?, ?, ?)
|
|
194
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
195
|
+
resource_id = excluded.resource_id,
|
|
196
|
+
authority_epoch = excluded.authority_epoch,
|
|
197
|
+
snapshot_version = excluded.snapshot_version,
|
|
198
|
+
updated_at = excluded.updated_at
|
|
199
|
+
`).run(resourceId, authorityEpoch, headVersion, now());
|
|
200
|
+
})();
|
|
201
|
+
return normalized;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function catalogRecords(resourceId) {
|
|
205
|
+
return db.prepare(`
|
|
206
|
+
SELECT record_json FROM entity_cloud_catalog
|
|
207
|
+
WHERE resource_id = ?
|
|
208
|
+
ORDER BY entity_uuid
|
|
209
|
+
`).all(resourceId).map((row) => assertRecord(JSON.parse(row.record_json)));
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function upsertCatalogRecord({ resourceId, authorityEpoch, headVersion, record }) {
|
|
213
|
+
const normalized = assertRecord(record);
|
|
214
|
+
db.prepare(`
|
|
215
|
+
INSERT INTO entity_cloud_catalog (
|
|
216
|
+
resource_id, entity_uuid, record_json, snapshot_epoch,
|
|
217
|
+
snapshot_version, updated_at
|
|
218
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
219
|
+
ON CONFLICT(resource_id, entity_uuid) DO UPDATE SET
|
|
220
|
+
record_json = excluded.record_json,
|
|
221
|
+
snapshot_epoch = excluded.snapshot_epoch,
|
|
222
|
+
snapshot_version = excluded.snapshot_version,
|
|
223
|
+
updated_at = excluded.updated_at
|
|
224
|
+
`).run(resourceId, normalized.uuid, stableJson(normalized), authorityEpoch, headVersion, now());
|
|
225
|
+
return normalized;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
function catalogTree({ resourceId, uuid }) {
|
|
229
|
+
const records = catalogRecords(resourceId);
|
|
230
|
+
const byUuid = new Map(records.map((record) => [record.uuid, record]));
|
|
231
|
+
const root = byUuid.get(String(uuid || '').toLowerCase());
|
|
232
|
+
if (!root) throw new Error(`cloud entity ${uuid} is not available on this machine`);
|
|
233
|
+
if (root.parentUUID !== null) {
|
|
234
|
+
throw new Error('amalgm add currently materializes a cloud tree root; select the parentless workspace, repository, file, or link UUID');
|
|
235
|
+
}
|
|
236
|
+
const children = new Map();
|
|
237
|
+
for (const record of records) {
|
|
238
|
+
if (record.parentUUID === null) continue;
|
|
239
|
+
const group = children.get(record.parentUUID) || [];
|
|
240
|
+
group.push(record);
|
|
241
|
+
children.set(record.parentUUID, group);
|
|
242
|
+
}
|
|
243
|
+
const tree = [];
|
|
244
|
+
const visit = (record) => {
|
|
245
|
+
tree.push(record);
|
|
246
|
+
for (const child of (children.get(record.uuid) || []).sort((left, right) => left.name.localeCompare(right.name))) {
|
|
247
|
+
visit(child);
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
visit(root);
|
|
251
|
+
return tree;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function currentCatalog() {
|
|
255
|
+
const current = replica();
|
|
256
|
+
if (!current || current.state !== 'active') {
|
|
257
|
+
throw new Error('entity cloud registry is not ready yet; wait for the signed-in runtime to finish its cloud handshake');
|
|
258
|
+
}
|
|
259
|
+
const snapshot = db.prepare(`
|
|
260
|
+
SELECT resource_id, authority_epoch, snapshot_version
|
|
261
|
+
FROM entity_cloud_catalog_state WHERE singleton = 1
|
|
262
|
+
`).get();
|
|
263
|
+
if (!snapshot || snapshot.resource_id !== current.resourceId
|
|
264
|
+
|| Number(snapshot.authority_epoch) !== current.authorityEpoch
|
|
265
|
+
|| Number(snapshot.snapshot_version) !== current.headVersion) {
|
|
266
|
+
throw new Error('entity cloud catalog is not pinned to the active cloud head yet; retry after the next snapshot handshake');
|
|
267
|
+
}
|
|
268
|
+
return current;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function materialization(row) {
|
|
272
|
+
if (!row) return null;
|
|
273
|
+
return {
|
|
274
|
+
jobId: row.job_id,
|
|
275
|
+
resourceId: row.resource_id,
|
|
276
|
+
entityUuid: row.entity_uuid,
|
|
277
|
+
targetPath: row.target_path,
|
|
278
|
+
authorityEpoch: Number(row.authority_epoch),
|
|
279
|
+
headVersion: Number(row.head_version),
|
|
280
|
+
state: row.state,
|
|
281
|
+
error: row.error,
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
|
|
170
285
|
/** Invoked by registry/index.js INSIDE its identity transaction. */
|
|
171
286
|
function journal({ record }) {
|
|
172
287
|
const current = replica();
|
|
@@ -177,6 +292,12 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
177
292
|
// content drain creates the filesystem upload cursor; doing that here
|
|
178
293
|
// would let a rolled-back identity write upload unreachable bytes.
|
|
179
294
|
queueContent(record, current.resourceId);
|
|
295
|
+
upsertCatalogRecord({
|
|
296
|
+
resourceId: current.resourceId,
|
|
297
|
+
authorityEpoch: current.authorityEpoch,
|
|
298
|
+
headVersion: current.headVersion,
|
|
299
|
+
record,
|
|
300
|
+
});
|
|
180
301
|
const mutationId = `ent_${crypto.randomBytes(18).toString('base64url')}`;
|
|
181
302
|
const operation = { kind: 'entity.upsert', record: assertRecord(record) };
|
|
182
303
|
const createdAt = now();
|
|
@@ -239,20 +360,30 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
239
360
|
if (bootstrapping && !sameRecord(local, snapshot)) {
|
|
240
361
|
throw new Error('first cloud entity snapshot does not match the locally bootstrapped identity graph');
|
|
241
362
|
}
|
|
242
|
-
if (!bootstrapping
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
363
|
+
if (!bootstrapping) {
|
|
364
|
+
const cloudByUuid = new Map(snapshot.records.map((record) => [record.uuid, record]));
|
|
365
|
+
for (const record of local.records) {
|
|
366
|
+
if (!sameRecord(record, cloudByUuid.get(record.uuid))) {
|
|
367
|
+
throw new Error('cloud entity graph conflicts with locally bound ground; resolve the local mutation before importing a new cloud head');
|
|
368
|
+
}
|
|
369
|
+
}
|
|
247
370
|
}
|
|
248
371
|
if (!Number.isInteger(Number(authorityEpoch)) || Number(authorityEpoch) < 1
|
|
249
372
|
|| !Number.isInteger(Number(headVersion)) || Number(headVersion) < 0) {
|
|
250
373
|
throw new Error('cloud entity registry authority is invalid');
|
|
251
374
|
}
|
|
252
|
-
//
|
|
253
|
-
// upload.
|
|
254
|
-
//
|
|
255
|
-
|
|
375
|
+
// A creator's first snapshot has no tail mutation to trigger content
|
|
376
|
+
// upload. A joining device must NOT queue remote bytes as uploads; it
|
|
377
|
+
// downloads them later through a materialization job instead.
|
|
378
|
+
if (bootstrapping) {
|
|
379
|
+
for (const record of snapshot.records) queueContent(record, resourceId, { queue: true });
|
|
380
|
+
}
|
|
381
|
+
replaceCatalog({
|
|
382
|
+
resourceId,
|
|
383
|
+
authorityEpoch: Number(authorityEpoch),
|
|
384
|
+
headVersion: Number(headVersion),
|
|
385
|
+
records: snapshot.records,
|
|
386
|
+
});
|
|
256
387
|
saveReplica({
|
|
257
388
|
resourceId,
|
|
258
389
|
deviceId: currentIdentity.deviceId,
|
|
@@ -261,7 +392,109 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
261
392
|
state: 'active',
|
|
262
393
|
snapshotCutoff: null,
|
|
263
394
|
});
|
|
264
|
-
return {
|
|
395
|
+
return {
|
|
396
|
+
resourceId,
|
|
397
|
+
records: snapshot.records.length,
|
|
398
|
+
headVersion: Number(headVersion),
|
|
399
|
+
snapshotVersion: Number(headVersion),
|
|
400
|
+
};
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* Start an `amalgm add` job from exactly one compacted cloud snapshot. The
|
|
405
|
+
* job deliberately contains no mutation cursor: historical mutations were
|
|
406
|
+
* folded into that snapshot by the authority before this point.
|
|
407
|
+
*/
|
|
408
|
+
function startMaterialization({ uuid, targetPath }) {
|
|
409
|
+
const current = currentCatalog();
|
|
410
|
+
if (typeof targetPath !== 'string' || !targetPath) throw new Error('materialization requires a destination path');
|
|
411
|
+
const tree = catalogTree({ resourceId: current.resourceId, uuid });
|
|
412
|
+
if (tree[0].status !== 'active') throw new Error(`cloud entity ${uuid} is not active`);
|
|
413
|
+
const unsupported = tree.find((record) => ['repo.git', 'link', 'reference'].includes(record.type));
|
|
414
|
+
if (unsupported) {
|
|
415
|
+
throw new Error(`cloud entity ${unsupported.uuid} is ${unsupported.type}; this first materializer supports ordinary workspace and file trees only`);
|
|
416
|
+
}
|
|
417
|
+
const prior = db.prepare(`
|
|
418
|
+
SELECT * FROM entity_materializations
|
|
419
|
+
WHERE resource_id = ? AND entity_uuid = ? AND target_path = ?
|
|
420
|
+
AND state IN ('waiting_for_content', 'materializing', 'complete')
|
|
421
|
+
ORDER BY created_at DESC LIMIT 1
|
|
422
|
+
`).get(current.resourceId, tree[0].uuid, targetPath);
|
|
423
|
+
if (prior) return materialization(prior);
|
|
424
|
+
const jobId = `mat_${crypto.randomBytes(18).toString('base64url')}`;
|
|
425
|
+
const createdAt = now();
|
|
426
|
+
db.prepare(`
|
|
427
|
+
INSERT INTO entity_materializations (
|
|
428
|
+
job_id, resource_id, entity_uuid, target_path, authority_epoch,
|
|
429
|
+
head_version, state, created_at, updated_at
|
|
430
|
+
) VALUES (?, ?, ?, ?, ?, ?, 'waiting_for_content', ?, ?)
|
|
431
|
+
`).run(jobId, current.resourceId, tree[0].uuid, targetPath, current.authorityEpoch, current.headVersion, createdAt, createdAt);
|
|
432
|
+
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function pendingMaterializations() {
|
|
436
|
+
return db.prepare(`
|
|
437
|
+
SELECT * FROM entity_materializations
|
|
438
|
+
WHERE state IN ('waiting_for_content', 'materializing')
|
|
439
|
+
ORDER BY created_at ASC
|
|
440
|
+
`).all().map(materialization);
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function claimDownload() {
|
|
444
|
+
const current = replica();
|
|
445
|
+
if (!current || !content) return null;
|
|
446
|
+
for (const job of pendingMaterializations().filter((entry) => entry.state === 'waiting_for_content')) {
|
|
447
|
+
if (job.resourceId !== current.resourceId) continue;
|
|
448
|
+
const tree = catalogTree({ resourceId: job.resourceId, uuid: job.entityUuid });
|
|
449
|
+
for (const record of tree) {
|
|
450
|
+
if (!['file.text', 'file.binary'].includes(record.type) || record.status !== 'active') continue;
|
|
451
|
+
if (!contentHead(record)) throw new Error(`cloud file ${record.uuid} has no immutable content head`);
|
|
452
|
+
const part = content.missingPart({ contentHash: record.payloadVersion });
|
|
453
|
+
if (!part) continue;
|
|
454
|
+
return {
|
|
455
|
+
resourceId: job.resourceId,
|
|
456
|
+
contentHash: record.payloadVersion,
|
|
457
|
+
kind: part.kind,
|
|
458
|
+
partIndex: part.partIndex,
|
|
459
|
+
sha256: part.sha256,
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function receiveDownload({ resourceId, contentHash, kind, partIndex, sha256, dataBase64 }) {
|
|
467
|
+
const current = replica();
|
|
468
|
+
if (!current || current.resourceId !== resourceId || !content) {
|
|
469
|
+
throw new Error(`unknown entity content resource ${resourceId}`);
|
|
470
|
+
}
|
|
471
|
+
return content.receive({ contentHash, kind, partIndex, sha256, dataBase64 });
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
function completeMaterialization({ jobId }) {
|
|
475
|
+
const row = db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId);
|
|
476
|
+
if (!row) throw new Error(`unknown entity materialization ${jobId}`);
|
|
477
|
+
db.prepare(`
|
|
478
|
+
UPDATE entity_materializations
|
|
479
|
+
SET state = 'complete', error = NULL, updated_at = ?
|
|
480
|
+
WHERE job_id = ?
|
|
481
|
+
`).run(now(), jobId);
|
|
482
|
+
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
function failMaterialization({ jobId, message }) {
|
|
486
|
+
const row = db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId);
|
|
487
|
+
if (!row) throw new Error(`unknown entity materialization ${jobId}`);
|
|
488
|
+
db.prepare(`
|
|
489
|
+
UPDATE entity_materializations
|
|
490
|
+
SET state = 'failed', error = ?, updated_at = ?
|
|
491
|
+
WHERE job_id = ?
|
|
492
|
+
`).run(String(message || 'cloud materialization failed').slice(0, 4000), now(), jobId);
|
|
493
|
+
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function materializationStatus(jobId) {
|
|
497
|
+
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
265
498
|
}
|
|
266
499
|
|
|
267
500
|
function claim({ limit = 16, leaseMs = 30_000 } = {}) {
|
|
@@ -411,20 +644,19 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
411
644
|
committedAt,
|
|
412
645
|
});
|
|
413
646
|
}
|
|
414
|
-
//
|
|
415
|
-
//
|
|
416
|
-
// accepting a later unknown root here would violate the registry's
|
|
417
|
-
// watched-root loss law. A duplicate record is safe and advances the
|
|
418
|
-
// local cloud cursor; a distinct record is deliberately surfaced.
|
|
647
|
+
// Remote mutations update the cloud-only catalog. They never create a
|
|
648
|
+
// local watcher; `amalgm add` alone turns catalog identity into ground.
|
|
419
649
|
let local;
|
|
420
|
-
try {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
throw new Error('remote entity mutation needs cloud-to-local materialization before this device can apply it');
|
|
424
|
-
}
|
|
425
|
-
if (!sameRecord(local, record)) {
|
|
426
|
-
throw new Error('remote entity mutation conflicts with this device; materialize or reconcile the cloud entity first');
|
|
650
|
+
try { local = registry.syncRecord(record.uuid); } catch { local = null; }
|
|
651
|
+
if (local && !sameRecord(local, record)) {
|
|
652
|
+
throw new Error('remote entity mutation conflicts with this device; refresh the compacted cloud head before applying it locally');
|
|
427
653
|
}
|
|
654
|
+
upsertCatalogRecord({
|
|
655
|
+
resourceId: current.resourceId,
|
|
656
|
+
authorityEpoch: current.authorityEpoch,
|
|
657
|
+
headVersion: Number(version),
|
|
658
|
+
record,
|
|
659
|
+
});
|
|
428
660
|
if (Number(version) > current.headVersion) saveReplica({ ...current, headVersion: Number(version) });
|
|
429
661
|
return { duplicate: true, version: Number(version) };
|
|
430
662
|
}
|
|
@@ -442,6 +674,14 @@ function createEntityCloud({ db, userId, deviceId, content = null }) {
|
|
|
442
674
|
claimContent,
|
|
443
675
|
acknowledgeContent,
|
|
444
676
|
failContent,
|
|
677
|
+
startMaterialization,
|
|
678
|
+
pendingMaterializations,
|
|
679
|
+
claimDownload,
|
|
680
|
+
receiveDownload,
|
|
681
|
+
completeMaterialization,
|
|
682
|
+
failMaterialization,
|
|
683
|
+
materializationStatus,
|
|
684
|
+
catalogTree,
|
|
445
685
|
replica,
|
|
446
686
|
};
|
|
447
687
|
}
|
|
@@ -330,6 +330,88 @@ function createEntityContent({ dir }) {
|
|
|
330
330
|
return manifest(contentHash) !== null;
|
|
331
331
|
}
|
|
332
332
|
|
|
333
|
+
/**
|
|
334
|
+
* A received cloud part is admitted by the same immutable manifest/chunk
|
|
335
|
+
* rules as a locally captured part. A manifest arrives first; it names the
|
|
336
|
+
* exact chunk checksums the later requests are allowed to write.
|
|
337
|
+
*/
|
|
338
|
+
function receive({ contentHash, kind, partIndex, sha256, dataBase64 }) {
|
|
339
|
+
assertHash(contentHash);
|
|
340
|
+
assertHash(sha256, 'part checksum');
|
|
341
|
+
if (!['manifest', 'chunk'].includes(kind) || !Number.isSafeInteger(partIndex) || partIndex < 0) {
|
|
342
|
+
throw new Error('remote entity content part is invalid');
|
|
343
|
+
}
|
|
344
|
+
const bytes = Buffer.from(String(dataBase64 || ''), 'base64');
|
|
345
|
+
if (digest(bytes) !== sha256) throw new Error('remote entity content checksum mismatch');
|
|
346
|
+
ensureDirs();
|
|
347
|
+
if (kind === 'manifest') {
|
|
348
|
+
if (partIndex !== 0) throw new Error('remote entity content manifest has an invalid part index');
|
|
349
|
+
let remote;
|
|
350
|
+
try {
|
|
351
|
+
remote = JSON.parse(bytes.toString('utf8'));
|
|
352
|
+
} catch {
|
|
353
|
+
throw new Error('remote entity content manifest is not JSON');
|
|
354
|
+
}
|
|
355
|
+
const value = validateManifest(remote, contentHash);
|
|
356
|
+
const current = manifest(contentHash);
|
|
357
|
+
if (current && stableManifest(current) !== stableManifest(value)) {
|
|
358
|
+
throw new Error(`local entity content manifest ${contentHash} conflicts with cloud`);
|
|
359
|
+
}
|
|
360
|
+
if (!current) writeJson(manifestFile(contentHash), value);
|
|
361
|
+
return { kind, contentHash, complete: complete({ contentHash }) };
|
|
362
|
+
}
|
|
363
|
+
const current = manifest(contentHash);
|
|
364
|
+
if (!current) throw new Error(`remote entity content chunk ${contentHash} arrived before its manifest`);
|
|
365
|
+
const expected = current.chunks[partIndex];
|
|
366
|
+
if (!expected || expected.sha256 !== sha256 || expected.bytes !== bytes.length) {
|
|
367
|
+
throw new Error(`remote entity content chunk ${contentHash}/${partIndex} does not match its manifest`);
|
|
368
|
+
}
|
|
369
|
+
writeChunk(sha256, bytes);
|
|
370
|
+
return { kind, contentHash, complete: complete({ contentHash }) };
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
function stableManifest(value) {
|
|
374
|
+
return JSON.stringify({
|
|
375
|
+
contract: value.contract,
|
|
376
|
+
contentHash: value.contentHash,
|
|
377
|
+
bytes: value.bytes,
|
|
378
|
+
chunks: value.chunks,
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
function missingPart({ contentHash }) {
|
|
383
|
+
const current = manifest(contentHash);
|
|
384
|
+
if (!current) return { kind: 'manifest', partIndex: 0, sha256: null };
|
|
385
|
+
for (let partIndex = 0; partIndex < current.chunks.length; partIndex += 1) {
|
|
386
|
+
const chunk = current.chunks[partIndex];
|
|
387
|
+
try {
|
|
388
|
+
const bytes = fs.readFileSync(chunkFile(chunk.sha256));
|
|
389
|
+
if (bytes.length === chunk.bytes && digest(bytes) === chunk.sha256) continue;
|
|
390
|
+
} catch (error) {
|
|
391
|
+
if (error.code !== 'ENOENT') throw error;
|
|
392
|
+
}
|
|
393
|
+
return { kind: 'chunk', partIndex, sha256: chunk.sha256 };
|
|
394
|
+
}
|
|
395
|
+
return null;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
function complete({ contentHash }) {
|
|
399
|
+
return missingPart({ contentHash }) === null;
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function bytes({ contentHash }) {
|
|
403
|
+
const current = manifest(contentHash);
|
|
404
|
+
if (!current || !complete({ contentHash })) {
|
|
405
|
+
throw new Error(`entity content ${contentHash} is not complete locally`);
|
|
406
|
+
}
|
|
407
|
+
const parts = current.chunks.map((chunk) => fs.readFileSync(chunkFile(chunk.sha256)));
|
|
408
|
+
const full = Buffer.concat(parts, current.bytes);
|
|
409
|
+
if (full.length !== current.bytes || digest(full) !== contentHash) {
|
|
410
|
+
throw new Error(`entity content ${contentHash} does not match its declared head`);
|
|
411
|
+
}
|
|
412
|
+
return full;
|
|
413
|
+
}
|
|
414
|
+
|
|
333
415
|
return {
|
|
334
416
|
captureFile,
|
|
335
417
|
ensureUpload,
|
|
@@ -339,6 +421,10 @@ function createEntityContent({ dir }) {
|
|
|
339
421
|
fail,
|
|
340
422
|
uploaded,
|
|
341
423
|
has,
|
|
424
|
+
receive,
|
|
425
|
+
missingPart,
|
|
426
|
+
complete,
|
|
427
|
+
bytes,
|
|
342
428
|
};
|
|
343
429
|
}
|
|
344
430
|
|
|
@@ -478,6 +478,7 @@ function createRegistration(options) {
|
|
|
478
478
|
*/
|
|
479
479
|
function add(targetPath, options = {}) {
|
|
480
480
|
const identityAt = options.identityAt ?? null;
|
|
481
|
+
const suppressJournal = options.suppressJournal === true;
|
|
481
482
|
if (identityAt !== null && typeof identityAt !== 'function') {
|
|
482
483
|
throw new Error('add identityAt must be a function when supplied');
|
|
483
484
|
}
|
|
@@ -514,7 +515,7 @@ function createRegistration(options) {
|
|
|
514
515
|
const root = observer.enrollRoot(ground, { scan: false, watch: false });
|
|
515
516
|
try {
|
|
516
517
|
assertWitnessed({ observer, registry, memory });
|
|
517
|
-
registerTree({
|
|
518
|
+
const register = () => registerTree({
|
|
518
519
|
root,
|
|
519
520
|
registry,
|
|
520
521
|
memory,
|
|
@@ -523,6 +524,8 @@ function createRegistration(options) {
|
|
|
523
524
|
repoStates,
|
|
524
525
|
identityAt,
|
|
525
526
|
});
|
|
527
|
+
if (suppressJournal) registry.withoutJournal(register);
|
|
528
|
+
else register();
|
|
526
529
|
} catch (error) {
|
|
527
530
|
// The observer root is only an address until structural registration
|
|
528
531
|
// commits. Do not leave an unregistered address behind after a refused
|
|
@@ -38,6 +38,8 @@
|
|
|
38
38
|
* every declared safe root (docs/user-cloud-bootstrap.md).
|
|
39
39
|
*/
|
|
40
40
|
|
|
41
|
+
const crypto = require('crypto');
|
|
42
|
+
const fs = require('fs');
|
|
41
43
|
const path = require('path');
|
|
42
44
|
|
|
43
45
|
const Database = require('better-sqlite3');
|
|
@@ -257,6 +259,99 @@ function createRegistrationService({
|
|
|
257
259
|
};
|
|
258
260
|
}
|
|
259
261
|
|
|
262
|
+
/** Materialize one already-pinned cloud tree into a fresh destination, then
|
|
263
|
+
* bind its cloud UUID graph to Watch. The destination is never merged with
|
|
264
|
+
* user ground: bytes are assembled in a sibling staging path and renamed
|
|
265
|
+
* only after every immutable head verifies locally. */
|
|
266
|
+
function materializeCloudTree(job) {
|
|
267
|
+
const tree = entityCloud.catalogTree({ resourceId: job.resourceId, uuid: job.entityUuid });
|
|
268
|
+
const root = tree[0];
|
|
269
|
+
const active = tree.filter((record) => record.status === 'active');
|
|
270
|
+
const records = new Map(active.map((record) => [record.uuid, record]));
|
|
271
|
+
const children = new Map();
|
|
272
|
+
for (const record of active) {
|
|
273
|
+
if (record.parentUUID === null) continue;
|
|
274
|
+
const group = children.get(record.parentUUID) || [];
|
|
275
|
+
group.push(record);
|
|
276
|
+
children.set(record.parentUUID, group);
|
|
277
|
+
}
|
|
278
|
+
for (const record of active) {
|
|
279
|
+
if (['file.text', 'file.binary'].includes(record.type)
|
|
280
|
+
&& !entityContent.complete({ contentHash: record.payloadVersion })) {
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
if (!['workspace', 'file.text', 'file.binary'].includes(root.type)) {
|
|
285
|
+
throw new Error(`cloud root ${root.uuid} is ${root.type}; this materializer supports workspace and file trees only`);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const parent = fs.realpathSync(job.targetPath);
|
|
289
|
+
const parentStat = fs.statSync(parent);
|
|
290
|
+
if (!parentStat.isDirectory()) throw new Error(`add destination is not a directory: ${parent}`);
|
|
291
|
+
const destination = path.join(parent, root.name);
|
|
292
|
+
if (fs.existsSync(destination)) throw new Error(`refusing to add ${root.name}: destination already exists at ${destination}`);
|
|
293
|
+
const staging = path.join(parent, `.${root.name}.amalgm-materializing-${crypto.randomBytes(10).toString('hex')}`);
|
|
294
|
+
let promoted = false;
|
|
295
|
+
|
|
296
|
+
function write(record, target) {
|
|
297
|
+
if (record.type === 'workspace' || record.type === 'folder') {
|
|
298
|
+
fs.mkdirSync(target, { mode: 0o700 });
|
|
299
|
+
for (const child of (children.get(record.uuid) || []).sort((left, right) => left.name.localeCompare(right.name))) {
|
|
300
|
+
write(child, path.join(target, child.name));
|
|
301
|
+
}
|
|
302
|
+
return;
|
|
303
|
+
}
|
|
304
|
+
if (record.type === 'file.text' || record.type === 'file.binary') {
|
|
305
|
+
fs.writeFileSync(target, entityContent.bytes({ contentHash: record.payloadVersion }), { mode: 0o600, flag: 'wx' });
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
throw new Error(`cloud entity ${record.uuid} is ${record.type}; this materializer cannot write it yet`);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function identityAt(relativePath) {
|
|
312
|
+
if (!relativePath) return root;
|
|
313
|
+
let cursor = root;
|
|
314
|
+
for (const name of relativePath.split('/')) {
|
|
315
|
+
cursor = (children.get(cursor.uuid) || []).find((record) => record.name === name) || null;
|
|
316
|
+
if (!cursor) return null;
|
|
317
|
+
}
|
|
318
|
+
return records.get(cursor.uuid) || null;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
try {
|
|
322
|
+
write(root, staging);
|
|
323
|
+
fs.renameSync(staging, destination);
|
|
324
|
+
promoted = true;
|
|
325
|
+
const bound = boundary.current.add(destination, { identityAt, suppressJournal: true });
|
|
326
|
+
if (bound.record.uuid !== root.uuid) {
|
|
327
|
+
throw new Error(`materialized cloud root ${root.uuid} bound to unexpected local UUID ${bound.record.uuid}`);
|
|
328
|
+
}
|
|
329
|
+
return { ...bound, destination };
|
|
330
|
+
} catch (error) {
|
|
331
|
+
// The staging and, if binding failed, destination contain only bytes
|
|
332
|
+
// produced from the verified immutable snapshot in this invocation.
|
|
333
|
+
// They never existed before this command, so removing them cannot erase
|
|
334
|
+
// user ground or hide a merge.
|
|
335
|
+
try {
|
|
336
|
+
if (fs.existsSync(staging)) fs.rmSync(staging, { recursive: true, force: true });
|
|
337
|
+
if (promoted && fs.existsSync(destination)) fs.rmSync(destination, { recursive: true, force: true });
|
|
338
|
+
} catch {}
|
|
339
|
+
throw error;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function advanceMaterializations() {
|
|
344
|
+
for (const job of entityCloud.pendingMaterializations()) {
|
|
345
|
+
if (job.state !== 'waiting_for_content') continue;
|
|
346
|
+
try {
|
|
347
|
+
const result = materializeCloudTree(job);
|
|
348
|
+
if (result) entityCloud.completeMaterialization({ jobId: job.jobId });
|
|
349
|
+
} catch (error) {
|
|
350
|
+
entityCloud.failMaterialization({ jobId: job.jobId, message: error.message });
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
|
|
260
355
|
return {
|
|
261
356
|
dir,
|
|
262
357
|
homeDir,
|
|
@@ -272,6 +367,13 @@ function createRegistrationService({
|
|
|
272
367
|
entityCloud.assertRegistrationReady();
|
|
273
368
|
return boundary.current.addMany(targetPaths);
|
|
274
369
|
},
|
|
370
|
+
/** `amalgm add <cloud-root-uuid> [parent-directory]` — cloud to local. */
|
|
371
|
+
add: async ({ uuid, targetPath }) => {
|
|
372
|
+
const job = entityCloud.startMaterialization({ uuid, targetPath });
|
|
373
|
+
advanceMaterializations();
|
|
374
|
+
return entityCloud.materializationStatus(job.jobId);
|
|
375
|
+
},
|
|
376
|
+
advanceMaterializations,
|
|
275
377
|
bootstrapUserHome,
|
|
276
378
|
entityCloud,
|
|
277
379
|
entityContent,
|
|
@@ -67,7 +67,7 @@ function registerTree({ root, registry, memory, shouldEnroll, captureRepo, repoS
|
|
|
67
67
|
|
|
68
68
|
function seeded(relativePath, { type, parentUUID, name, root: isRoot }) {
|
|
69
69
|
const identity = identityAt?.(relativePath) || null;
|
|
70
|
-
if (!identity) return { type, uuid: undefined };
|
|
70
|
+
if (!identity) return { type, uuid: undefined, payloadVersion: undefined };
|
|
71
71
|
if (identity.name !== name || identity.parentUUID !== parentUUID || identity.status !== 'active') {
|
|
72
72
|
throw new Error(`cloud identity at ${relativePath || '.'} does not match the local declared tree`);
|
|
73
73
|
}
|
|
@@ -82,7 +82,15 @@ function registerTree({ root, registry, memory, shouldEnroll, captureRepo, repoS
|
|
|
82
82
|
if (isRoot && !['workspace', 'repo.git', 'file.text', 'file.binary', 'link'].includes(identity.type)) {
|
|
83
83
|
throw new Error(`cloud identity at ${relativePath || '.'} is not a valid tree root`);
|
|
84
84
|
}
|
|
85
|
-
return {
|
|
85
|
+
return {
|
|
86
|
+
type: identity.type,
|
|
87
|
+
uuid: identity.uuid,
|
|
88
|
+
// Structural registration normally leaves leaf payloads for Detect.
|
|
89
|
+
// A cloud materializer has already verified immutable bytes before it
|
|
90
|
+
// binds this tree, so retaining that head prevents an import from
|
|
91
|
+
// inventing a temporary local mutation.
|
|
92
|
+
payloadVersion: identity.payloadVersion,
|
|
93
|
+
};
|
|
86
94
|
}
|
|
87
95
|
|
|
88
96
|
return registry.transaction(() => {
|
|
@@ -93,7 +101,7 @@ function registerTree({ root, registry, memory, shouldEnroll, captureRepo, repoS
|
|
|
93
101
|
const record = registry.enrollRoot({
|
|
94
102
|
type: rootIdentity.type,
|
|
95
103
|
name: path.basename(root.path),
|
|
96
|
-
payloadVersion: initialState?.stateId ?? null,
|
|
104
|
+
payloadVersion: initialState?.stateId ?? rootIdentity.payloadVersion ?? null,
|
|
97
105
|
uuid: rootIdentity.uuid,
|
|
98
106
|
});
|
|
99
107
|
memory.bind(root.rootId, root.rootId, record.uuid);
|
|
@@ -157,7 +165,7 @@ function registerDirectory(dirPath, parentUUID, {
|
|
|
157
165
|
type: identity.type,
|
|
158
166
|
parentUUID,
|
|
159
167
|
name: entry.name,
|
|
160
|
-
payloadVersion: state?.stateId ?? null,
|
|
168
|
+
payloadVersion: state?.stateId ?? identity.payloadVersion ?? null,
|
|
161
169
|
uuid: identity.uuid,
|
|
162
170
|
});
|
|
163
171
|
if (state) repoStates.put(record.uuid, state);
|
|
@@ -170,7 +178,13 @@ function registerDirectory(dirPath, parentUUID, {
|
|
|
170
178
|
const identity = seeded(entryRelativePath, {
|
|
171
179
|
type: 'file.text', parentUUID, name: entry.name, root: false,
|
|
172
180
|
});
|
|
173
|
-
registry.create({
|
|
181
|
+
registry.create({
|
|
182
|
+
type: identity.type,
|
|
183
|
+
parentUUID,
|
|
184
|
+
name: entry.name,
|
|
185
|
+
payloadVersion: identity.payloadVersion ?? null,
|
|
186
|
+
uuid: identity.uuid,
|
|
187
|
+
});
|
|
174
188
|
}
|
|
175
189
|
}
|
|
176
190
|
}
|
|
@@ -182,6 +182,7 @@ function createRegistry(options) {
|
|
|
182
182
|
// attachment must always compare, pending bell or not.
|
|
183
183
|
let bellPending = false;
|
|
184
184
|
const deferred = []; // bells born inside an open transaction: they ring at commit, or a rollback silences them
|
|
185
|
+
let journalSuppression = 0;
|
|
185
186
|
|
|
186
187
|
function ring(event) {
|
|
187
188
|
try {
|
|
@@ -311,7 +312,9 @@ function createRegistry(options) {
|
|
|
311
312
|
// before notification. A failed journal write rolls the entity mutation
|
|
312
313
|
// back; a later process can never observe local truth without its cloud
|
|
313
314
|
// delivery intent.
|
|
314
|
-
if (journal
|
|
315
|
+
if (journal && journalSuppression === 0) {
|
|
316
|
+
journal({ record: syncRecord(next), previous: previous ? syncRecord(previous) : null });
|
|
317
|
+
}
|
|
315
318
|
}
|
|
316
319
|
|
|
317
320
|
function normalizeCloudRecord(value) {
|
|
@@ -475,6 +478,18 @@ function createRegistry(options) {
|
|
|
475
478
|
*/
|
|
476
479
|
transaction,
|
|
477
480
|
|
|
481
|
+
/** Bind already-authoritative cloud identity to materialized local ground
|
|
482
|
+
* without turning that import into a new outbound mutation tail. */
|
|
483
|
+
withoutJournal(fn) {
|
|
484
|
+
if (typeof fn !== 'function') throw new Error('withoutJournal requires a function');
|
|
485
|
+
journalSuppression += 1;
|
|
486
|
+
try {
|
|
487
|
+
return fn();
|
|
488
|
+
} finally {
|
|
489
|
+
journalSuppression -= 1;
|
|
490
|
+
}
|
|
491
|
+
},
|
|
492
|
+
|
|
478
493
|
/** Always requests comparison — it never trusts the in-memory flag,
|
|
479
494
|
* because a restart loses that memory. Returns whether the bell rang. */
|
|
480
495
|
flush() {
|