amalgm 0.1.257 → 0.1.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.js +72 -3
- package/lib/layout.js +30 -9
- package/lib/shared-realtime-tunnel.js +12 -14
- package/package.json +2 -2
- package/runtime/scripts/amalgm-mcp/lib/layout.js +32 -9
- package/runtime/scripts/amalgm-mcp/registration/entity-cloud.js +186 -32
- package/runtime/scripts/amalgm-mcp/registration/entity-content.js +33 -6
- package/runtime/scripts/amalgm-mcp/registration/index.js +21 -0
- package/runtime/scripts/amalgm-mcp/registration/service.js +102 -11
- package/runtime/scripts/amalgm-mcp/registry/index.js +1 -1
- package/runtime/scripts/amalgm-mcp/server/routes/entities.js +6 -0
- package/runtime/scripts/amalgm-mcp/tests/entity-cloud.test.js +67 -8
- package/runtime/scripts/amalgm-mcp/tests/entity-materialization.test.js +3 -3
- package/runtime/scripts/amalgm-mcp/tests/registration.service.test.js +45 -3
- package/runtime/scripts/amalgm-mcp/tests/system-catalog.test.js +3 -6
- package/runtime/scripts/amalgm-mcp/tests/toolbox-artifacts.test.js +88 -0
- package/runtime/scripts/amalgm-mcp/tests/toolbox-cloud-registration.test.js +91 -0
- package/runtime/scripts/amalgm-mcp/toolbox/artifacts.js +225 -0
- package/runtime/scripts/amalgm-mcp/toolbox/store.js +59 -10
- package/runtime/scripts/amalgm-mcp/toolbox/system-catalog.js +14 -5
|
@@ -11,12 +11,13 @@
|
|
|
11
11
|
|
|
12
12
|
const crypto = require('crypto');
|
|
13
13
|
|
|
14
|
-
// `@1`
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
// home; later user ground enters
|
|
19
|
-
|
|
14
|
+
// `@1` could project arbitrary pre-cloud registrations. `@2` omitted the
|
|
15
|
+
// declared built-in tool entities, so its toolbox root could not prove the
|
|
16
|
+
// snapshot membership it named. A resource contract names immutable authority
|
|
17
|
+
// semantics: both remain quarantined rather than reinterpreted or overwritten.
|
|
18
|
+
// `@3` begins with the complete verified user home; later user ground enters
|
|
19
|
+
// exclusively through `amalgm register`.
|
|
20
|
+
const ENTITY_CLOUD_CONTRACT = 'amalgm-entity-registry@3';
|
|
20
21
|
const ENTITY_CLOUD_SCHEMA_VERSION = 1;
|
|
21
22
|
const BOOTSTRAP_RECONCILIATION_STATE = 'reconciling-bootstrap';
|
|
22
23
|
const SNAPSHOT_REPLACE_OPERATION = 'entity.snapshot.replace';
|
|
@@ -191,6 +192,56 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
191
192
|
};
|
|
192
193
|
}
|
|
193
194
|
|
|
195
|
+
// The catalog mirrors the compacted cloud head as long as every head
|
|
196
|
+
// advance since the last installed snapshot was applied to it in order:
|
|
197
|
+
// journal() upserts this device's own records, ingest() upserts remote
|
|
198
|
+
// ones, and the gateway's fold regenerates only derived container
|
|
199
|
+
// versions, which every catalog consumer already tolerates. A version
|
|
200
|
+
// gap, a remote snapshot replacement, or a restart drops this back to
|
|
201
|
+
// whatever the durable pin proves, and the next snapshot download
|
|
202
|
+
// re-establishes it.
|
|
203
|
+
let catalogLive = null;
|
|
204
|
+
// Mutation rows whose content-upload cursors already exist on disk.
|
|
205
|
+
const contentQueuedSequences = new Set();
|
|
206
|
+
|
|
207
|
+
function catalogPinnedAtHead(current) {
|
|
208
|
+
if (!current) return false;
|
|
209
|
+
const pin = db.prepare(`
|
|
210
|
+
SELECT resource_id, authority_epoch, snapshot_version
|
|
211
|
+
FROM entity_cloud_catalog_state WHERE singleton = 1
|
|
212
|
+
`).get();
|
|
213
|
+
return Boolean(pin && pin.resource_id === current.resourceId
|
|
214
|
+
&& Number(pin.authority_epoch) === current.authorityEpoch
|
|
215
|
+
&& Number(pin.snapshot_version) === current.headVersion);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function unsettledMutationCount(resourceId) {
|
|
219
|
+
return Number(db.prepare(`
|
|
220
|
+
SELECT COUNT(*) AS count FROM entity_cloud_mutations
|
|
221
|
+
WHERE resource_id = ? AND mutation_state <> 'synced'
|
|
222
|
+
`).get(resourceId).count);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function catalogIsLive(current) {
|
|
226
|
+
if (catalogLive === null) {
|
|
227
|
+
catalogLive = catalogPinnedAtHead(current) && unsettledMutationCount(current.resourceId) === 0;
|
|
228
|
+
}
|
|
229
|
+
return catalogLive;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function pinCatalogState({ resourceId, authorityEpoch, headVersion }) {
|
|
233
|
+
db.prepare(`
|
|
234
|
+
INSERT INTO entity_cloud_catalog_state (
|
|
235
|
+
singleton, resource_id, authority_epoch, snapshot_version, updated_at
|
|
236
|
+
) VALUES (1, ?, ?, ?, ?)
|
|
237
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
238
|
+
resource_id = excluded.resource_id,
|
|
239
|
+
authority_epoch = excluded.authority_epoch,
|
|
240
|
+
snapshot_version = excluded.snapshot_version,
|
|
241
|
+
updated_at = excluded.updated_at
|
|
242
|
+
`).run(resourceId, authorityEpoch, headVersion, now());
|
|
243
|
+
}
|
|
244
|
+
|
|
194
245
|
function saveReplica(value) {
|
|
195
246
|
db.prepare(`
|
|
196
247
|
INSERT INTO entity_cloud_replica (
|
|
@@ -236,16 +287,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
236
287
|
// A later mutation may update an individual catalog row so live state
|
|
237
288
|
// can be inspected, but that must never make the whole catalog look
|
|
238
289
|
// like a newly compacted snapshot that is safe to materialize.
|
|
239
|
-
|
|
240
|
-
INSERT INTO entity_cloud_catalog_state (
|
|
241
|
-
singleton, resource_id, authority_epoch, snapshot_version, updated_at
|
|
242
|
-
) VALUES (1, ?, ?, ?, ?)
|
|
243
|
-
ON CONFLICT(singleton) DO UPDATE SET
|
|
244
|
-
resource_id = excluded.resource_id,
|
|
245
|
-
authority_epoch = excluded.authority_epoch,
|
|
246
|
-
snapshot_version = excluded.snapshot_version,
|
|
247
|
-
updated_at = excluded.updated_at
|
|
248
|
-
`).run(resourceId, authorityEpoch, headVersion, now());
|
|
290
|
+
pinCatalogState({ resourceId, authorityEpoch, headVersion });
|
|
249
291
|
})();
|
|
250
292
|
return normalized;
|
|
251
293
|
}
|
|
@@ -523,6 +565,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
523
565
|
});
|
|
524
566
|
enqueueBootstrapReconciliation(trustedBootstrap, next);
|
|
525
567
|
})();
|
|
568
|
+
catalogLive = false;
|
|
526
569
|
return {
|
|
527
570
|
resourceId,
|
|
528
571
|
records: snapshot.records.length,
|
|
@@ -557,6 +600,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
557
600
|
state: complete ? 'active' : BOOTSTRAP_RECONCILIATION_STATE,
|
|
558
601
|
snapshotCutoff: null,
|
|
559
602
|
});
|
|
603
|
+
catalogLive = complete;
|
|
560
604
|
return {
|
|
561
605
|
resourceId,
|
|
562
606
|
records: snapshot.records.length,
|
|
@@ -604,6 +648,10 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
604
648
|
state: 'active',
|
|
605
649
|
snapshotCutoff: null,
|
|
606
650
|
});
|
|
651
|
+
// A snapshot installed over unsettled local mutations replaced their
|
|
652
|
+
// catalog upserts; liveness returns with the install that finds an
|
|
653
|
+
// empty outbox.
|
|
654
|
+
catalogLive = unsettledMutationCount(resourceId) === 0;
|
|
607
655
|
return {
|
|
608
656
|
resourceId,
|
|
609
657
|
records: snapshot.records.length,
|
|
@@ -710,6 +758,58 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
710
758
|
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
711
759
|
}
|
|
712
760
|
|
|
761
|
+
/** Read-only cloud-replica truth for `amalgm status`. This deliberately
|
|
762
|
+
* reports a stale catalog as stale instead of presenting its records as a
|
|
763
|
+
* head that is safe to materialize. Status is an observer of authority,
|
|
764
|
+
* never a repair or a hidden snapshot handshake. */
|
|
765
|
+
function status() {
|
|
766
|
+
const current = replica();
|
|
767
|
+
const resourceId = current?.resourceId || null;
|
|
768
|
+
const mutationCounts = new Map();
|
|
769
|
+
if (resourceId) {
|
|
770
|
+
for (const row of db.prepare(`
|
|
771
|
+
SELECT mutation_state, COUNT(*) AS count
|
|
772
|
+
FROM entity_cloud_mutations
|
|
773
|
+
WHERE resource_id = ?
|
|
774
|
+
GROUP BY mutation_state
|
|
775
|
+
`).all(resourceId)) {
|
|
776
|
+
mutationCounts.set(row.mutation_state, Number(row.count));
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
const materializationCounts = new Map();
|
|
780
|
+
if (resourceId) {
|
|
781
|
+
for (const row of db.prepare(`
|
|
782
|
+
SELECT state, COUNT(*) AS count
|
|
783
|
+
FROM entity_materializations
|
|
784
|
+
WHERE resource_id = ?
|
|
785
|
+
GROUP BY state
|
|
786
|
+
`).all(resourceId)) {
|
|
787
|
+
materializationCounts.set(row.state, Number(row.count));
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
const pending = (mutationCounts.get('pending') || 0) + (mutationCounts.get('syncing') || 0);
|
|
791
|
+
const pinned = current ? catalogPinnedAtHead(current) : false;
|
|
792
|
+
return {
|
|
793
|
+
identity: identity(),
|
|
794
|
+
replica: current,
|
|
795
|
+
catalog: {
|
|
796
|
+
pinned,
|
|
797
|
+
current: Boolean(current?.state === 'active' && pinned && pending === 0),
|
|
798
|
+
records: resourceId ? catalogRecords(resourceId) : [],
|
|
799
|
+
},
|
|
800
|
+
outbox: {
|
|
801
|
+
pending: mutationCounts.get('pending') || 0,
|
|
802
|
+
syncing: mutationCounts.get('syncing') || 0,
|
|
803
|
+
rejected: mutationCounts.get('rejected') || 0,
|
|
804
|
+
},
|
|
805
|
+
materializations: {
|
|
806
|
+
waitingForContent: materializationCounts.get('waiting_for_content') || 0,
|
|
807
|
+
materializing: materializationCounts.get('materializing') || 0,
|
|
808
|
+
failed: materializationCounts.get('failed') || 0,
|
|
809
|
+
},
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
|
|
713
813
|
/** A completed add is a live binding, not a one-time download. A remote
|
|
714
814
|
* record beneath that binding moves the durable job back to its content
|
|
715
815
|
* gate. The service owns the later filesystem application; this module
|
|
@@ -810,17 +910,21 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
810
910
|
// Later registrations journal their record transactionally in SQLite.
|
|
811
911
|
// Derive its content-upload cursor only after that commit exists, so a
|
|
812
912
|
// failed transaction cannot publish a private object no entity names.
|
|
913
|
+
// Each mutation's operation is immutable, so it is scanned once per
|
|
914
|
+
// process; a restart simply rescans the still-unsettled rows.
|
|
813
915
|
const rows = db.prepare(`
|
|
814
|
-
SELECT operation_json FROM entity_cloud_mutations
|
|
916
|
+
SELECT sequence, operation_json FROM entity_cloud_mutations
|
|
815
917
|
WHERE resource_id = ? AND mutation_state IN ('pending', 'syncing')
|
|
816
918
|
ORDER BY sequence ASC
|
|
817
919
|
`).all(current.resourceId);
|
|
818
920
|
for (const row of rows) {
|
|
921
|
+
if (contentQueuedSequences.has(row.sequence)) continue;
|
|
819
922
|
const operation = JSON.parse(row.operation_json);
|
|
820
923
|
const records = operation?.kind === SNAPSHOT_REPLACE_OPERATION
|
|
821
924
|
? operation?.snapshot?.records
|
|
822
925
|
: [operation?.record];
|
|
823
926
|
for (const record of records || []) queueContent(record, current.resourceId, { queue: true });
|
|
927
|
+
contentQueuedSequences.add(row.sequence);
|
|
824
928
|
}
|
|
825
929
|
return content.claim({ resourceId: current.resourceId, leaseMs });
|
|
826
930
|
}
|
|
@@ -850,31 +954,59 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
850
954
|
.get(resourceId, mutationId);
|
|
851
955
|
if (!row) throw new Error(`unknown entity cloud mutation ${mutationId}`);
|
|
852
956
|
if (row.mutation_state === 'synced') {
|
|
853
|
-
const unsettled = db.prepare('SELECT COUNT(*) AS count FROM entity_cloud_mutations WHERE resource_id = ? AND mutation_state <> \'synced\'')
|
|
854
|
-
.get(resourceId);
|
|
855
957
|
return {
|
|
856
958
|
duplicate: true,
|
|
857
959
|
version: Number(row.official_version),
|
|
858
|
-
requiresSnapshot: current.state
|
|
960
|
+
requiresSnapshot: current.state === BOOTSTRAP_RECONCILIATION_STATE
|
|
961
|
+
? unsettledMutationCount(resourceId) === 0
|
|
962
|
+
: !catalogIsLive(current),
|
|
859
963
|
};
|
|
860
964
|
}
|
|
861
965
|
if (!['pending', 'syncing'].includes(row.mutation_state)) throw new Error(`entity cloud mutation ${mutationId} cannot sync from ${row.mutation_state}`);
|
|
966
|
+
const live = current.state === BOOTSTRAP_RECONCILIATION_STATE ? false : catalogIsLive(current);
|
|
862
967
|
db.prepare(`
|
|
863
968
|
UPDATE entity_cloud_mutations
|
|
864
969
|
SET mutation_state = 'synced', official_version = ?, cloud_committed_at = ?,
|
|
865
970
|
next_attempt_at = NULL, last_error = NULL, updated_at = ?
|
|
866
971
|
WHERE sequence = ?
|
|
867
972
|
`).run(Number(version), String(committedAt || now()), now(), row.sequence);
|
|
868
|
-
|
|
973
|
+
const previousHead = current.headVersion;
|
|
974
|
+
if (Number(version) > previousHead) {
|
|
869
975
|
saveReplica({ ...current, headVersion: Number(version) });
|
|
870
976
|
}
|
|
871
|
-
const unsettled =
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
977
|
+
const unsettled = unsettledMutationCount(resourceId);
|
|
978
|
+
if (current.state === BOOTSTRAP_RECONCILIATION_STATE) {
|
|
979
|
+
return { duplicate: false, version: Number(version), requiresSnapshot: unsettled === 0 };
|
|
980
|
+
}
|
|
981
|
+
// Local convergence instead of a snapshot re-download per ack. This
|
|
982
|
+
// device journaled these exact records into the catalog when the
|
|
983
|
+
// mutation was created, so an in-order acknowledgement leaves the
|
|
984
|
+
// catalog equal to the compacted head by construction. Anything out
|
|
985
|
+
// of order gives liveness up until the next installed snapshot.
|
|
986
|
+
let operation = null;
|
|
987
|
+
try { operation = JSON.parse(row.operation_json); } catch {}
|
|
988
|
+
if (operation?.kind === SNAPSHOT_REPLACE_OPERATION) {
|
|
989
|
+
if (unsettled === 0 && Array.isArray(operation?.snapshot?.records)) {
|
|
990
|
+
// The replacement operation IS the complete graph at this version.
|
|
991
|
+
replaceCatalog({
|
|
992
|
+
resourceId,
|
|
993
|
+
authorityEpoch: current.authorityEpoch,
|
|
994
|
+
headVersion: Number(version),
|
|
995
|
+
records: operation.snapshot.records,
|
|
996
|
+
});
|
|
997
|
+
catalogLive = true;
|
|
998
|
+
} else {
|
|
999
|
+
catalogLive = false;
|
|
1000
|
+
}
|
|
1001
|
+
} else if (!operation || Number(version) !== previousHead + 1) {
|
|
1002
|
+
catalogLive = false;
|
|
1003
|
+
} else {
|
|
1004
|
+
catalogLive = live;
|
|
1005
|
+
if (catalogLive && unsettled === 0) {
|
|
1006
|
+
pinCatalogState({ resourceId, authorityEpoch: current.authorityEpoch, headVersion: Number(version) });
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
return { duplicate: false, version: Number(version), requiresSnapshot: !catalogLive };
|
|
878
1010
|
})();
|
|
879
1011
|
}
|
|
880
1012
|
|
|
@@ -912,10 +1044,17 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
912
1044
|
if (envelope?.operationKind === SNAPSHOT_REPLACE_OPERATION
|
|
913
1045
|
&& envelope?.operation?.kind === SNAPSHOT_REPLACE_OPERATION) {
|
|
914
1046
|
// The complete replacement is imported only from the active snapshot
|
|
915
|
-
// the gateway
|
|
916
|
-
//
|
|
1047
|
+
// the gateway compacts at lookup. Consuming its tail would reintroduce
|
|
1048
|
+
// the very partial-graph state this operation prevents.
|
|
917
1049
|
if (Number(version) > current.headVersion) saveReplica({ ...current, headVersion: Number(version) });
|
|
918
|
-
|
|
1050
|
+
catalogLive = false;
|
|
1051
|
+
return {
|
|
1052
|
+
duplicate: true,
|
|
1053
|
+
version: Number(version),
|
|
1054
|
+
snapshotRequired: true,
|
|
1055
|
+
requiresSnapshot: true,
|
|
1056
|
+
materializations: [],
|
|
1057
|
+
};
|
|
919
1058
|
}
|
|
920
1059
|
if (envelope?.operationKind !== 'entity.upsert' || envelope?.operation?.kind !== 'entity.upsert') {
|
|
921
1060
|
throw new Error('cloud entity mutation does not belong to this registry');
|
|
@@ -951,10 +1090,24 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
951
1090
|
headVersion: Number(version),
|
|
952
1091
|
record,
|
|
953
1092
|
});
|
|
954
|
-
|
|
1093
|
+
// A remote record applied in official order keeps the catalog at the
|
|
1094
|
+
// compacted head, so no snapshot re-download is needed for it either.
|
|
1095
|
+
const live = catalogIsLive(current);
|
|
1096
|
+
const previousHead = current.headVersion;
|
|
1097
|
+
if (Number(version) > previousHead) saveReplica({ ...current, headVersion: Number(version) });
|
|
1098
|
+
if (Number(version) !== previousHead + 1) {
|
|
1099
|
+
catalogLive = false;
|
|
1100
|
+
} else if (live && unsettledMutationCount(current.resourceId) === 0) {
|
|
1101
|
+
pinCatalogState({
|
|
1102
|
+
resourceId: current.resourceId,
|
|
1103
|
+
authorityEpoch: current.authorityEpoch,
|
|
1104
|
+
headVersion: Number(version),
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
955
1107
|
return {
|
|
956
1108
|
duplicate: true,
|
|
957
1109
|
version: Number(version),
|
|
1110
|
+
requiresSnapshot: !catalogLive,
|
|
958
1111
|
materializations: refreshMaterializations({ resourceId: current.resourceId, record, version }),
|
|
959
1112
|
};
|
|
960
1113
|
}
|
|
@@ -980,6 +1133,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
980
1133
|
completeMaterialization,
|
|
981
1134
|
failMaterialization,
|
|
982
1135
|
materializationStatus,
|
|
1136
|
+
status,
|
|
983
1137
|
hasUnsettledLocalMutation,
|
|
984
1138
|
catalogTree,
|
|
985
1139
|
replica,
|
|
@@ -15,7 +15,10 @@ const fs = require('fs');
|
|
|
15
15
|
const path = require('path');
|
|
16
16
|
|
|
17
17
|
const CONTENT_CONTRACT = 'amalgm-entity-content@1';
|
|
18
|
-
|
|
18
|
+
// One chunk is one WAN round trip. The gateway accepts parts up to this same
|
|
19
|
+
// bound (ENTITY_CONTENT_PART_BYTES); older 256KB chunks remain valid because
|
|
20
|
+
// the manifest rule is `bytes <= CHUNK_BYTES`, never an exact size.
|
|
21
|
+
const CHUNK_BYTES = 4 * 1024 * 1024;
|
|
19
22
|
const HASH = /^[0-9a-f]{64}$/;
|
|
20
23
|
|
|
21
24
|
function digest(bytes) {
|
|
@@ -41,6 +44,10 @@ function createEntityContent({ dir }) {
|
|
|
41
44
|
const chunksDir = path.join(root, 'chunks');
|
|
42
45
|
const manifestsDir = path.join(root, 'manifests');
|
|
43
46
|
const uploadsDir = path.join(root, 'uploads');
|
|
47
|
+
// Unfinished uploads per resource, so claim() does not re-read the whole
|
|
48
|
+
// ledger directory for every part it sends. The on-disk ledger stays the
|
|
49
|
+
// durable truth; this set is rebuilt from it once per process.
|
|
50
|
+
const pendingUploads = new Map();
|
|
44
51
|
|
|
45
52
|
function ensureDirs() {
|
|
46
53
|
for (const target of [chunksDir, manifestsDir, uploadsDir]) {
|
|
@@ -236,14 +243,27 @@ function createEntityContent({ dir }) {
|
|
|
236
243
|
}
|
|
237
244
|
}
|
|
238
245
|
|
|
246
|
+
function pendingSet(resourceId) {
|
|
247
|
+
let pending = pendingUploads.get(resourceId);
|
|
248
|
+
if (!pending) {
|
|
249
|
+
pending = new Set(uploadEntries(resourceId));
|
|
250
|
+
pendingUploads.set(resourceId, pending);
|
|
251
|
+
}
|
|
252
|
+
return pending;
|
|
253
|
+
}
|
|
254
|
+
|
|
239
255
|
function ensureUpload({ resourceId, contentHash }) {
|
|
240
256
|
if (typeof resourceId !== 'string' || !resourceId) throw new Error('entity content upload requires a resource id');
|
|
241
257
|
const content = manifest(contentHash);
|
|
242
258
|
if (!content) throw new Error(`content ${contentHash} was not captured before its entity record committed`);
|
|
243
259
|
const current = readUpload(resourceId, contentHash, content);
|
|
244
|
-
if (current)
|
|
260
|
+
if (current) {
|
|
261
|
+
if (!current.manifestUploaded) pendingSet(resourceId).add(contentHash);
|
|
262
|
+
return current;
|
|
263
|
+
}
|
|
245
264
|
const created = initialUpload(resourceId, contentHash);
|
|
246
265
|
saveUpload(created);
|
|
266
|
+
pendingSet(resourceId).add(contentHash);
|
|
247
267
|
return created;
|
|
248
268
|
}
|
|
249
269
|
|
|
@@ -269,11 +289,15 @@ function createEntityContent({ dir }) {
|
|
|
269
289
|
function claim({ resourceId, leaseMs = 30_000 }) {
|
|
270
290
|
ensureDirs();
|
|
271
291
|
const at = Date.now();
|
|
272
|
-
|
|
292
|
+
const pending = pendingSet(resourceId);
|
|
293
|
+
for (const contentHash of pending) {
|
|
273
294
|
const content = manifest(contentHash);
|
|
274
295
|
if (!content) throw new Error(`entity content ${contentHash} disappeared from the local cache`);
|
|
275
296
|
const upload = readUpload(resourceId, contentHash, content);
|
|
276
|
-
if (!upload || upload.manifestUploaded)
|
|
297
|
+
if (!upload || upload.manifestUploaded) {
|
|
298
|
+
pending.delete(contentHash);
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
277
301
|
if (upload.leaseUntil && Date.parse(upload.leaseUntil) > at) continue;
|
|
278
302
|
const next = { ...upload, leaseUntil: new Date(at + Math.max(1_000, Number(leaseMs) || 30_000)).toISOString(), updatedAt: now() };
|
|
279
303
|
if (next.nextPart < content.chunks.length) {
|
|
@@ -333,6 +357,7 @@ function createEntityContent({ dir }) {
|
|
|
333
357
|
current.leaseUntil = null;
|
|
334
358
|
current.updatedAt = now();
|
|
335
359
|
saveUpload(current);
|
|
360
|
+
if (current.manifestUploaded) pendingUploads.get(resourceId)?.delete(contentHash);
|
|
336
361
|
return { duplicate: false, uploaded: current.manifestUploaded };
|
|
337
362
|
}
|
|
338
363
|
|
|
@@ -412,9 +437,11 @@ function createEntityContent({ dir }) {
|
|
|
412
437
|
if (!current) return { kind: 'manifest', partIndex: 0, sha256: null };
|
|
413
438
|
for (let partIndex = 0; partIndex < current.chunks.length; partIndex += 1) {
|
|
414
439
|
const chunk = current.chunks[partIndex];
|
|
440
|
+
// Presence + size only: every chunk file was hash-verified before it
|
|
441
|
+
// was written (capture or receive), writes are write-once, and final
|
|
442
|
+
// assembly (`bytes`) re-verifies the whole content head.
|
|
415
443
|
try {
|
|
416
|
-
|
|
417
|
-
if (bytes.length === chunk.bytes && digest(bytes) === chunk.sha256) continue;
|
|
444
|
+
if (fs.statSync(chunkFile(chunk.sha256)).size === chunk.bytes) continue;
|
|
418
445
|
} catch (error) {
|
|
419
446
|
if (error.code !== 'ENOENT') throw error;
|
|
420
447
|
}
|
|
@@ -596,6 +596,26 @@ function createRegistration(options) {
|
|
|
596
596
|
return normalized.map(({ targetPath, identityAt }) => add(targetPath, { identityAt }));
|
|
597
597
|
}
|
|
598
598
|
|
|
599
|
+
/** Reconcile one already-watched tree after the platform has materialized
|
|
600
|
+
* a newly declared exact default inside it. This is deliberately not part
|
|
601
|
+
* of ordinary re-registration: the caller must have already named the
|
|
602
|
+
* exact platform node it needs, and the normal Detect → Record path still
|
|
603
|
+
* decides the resulting record. */
|
|
604
|
+
function reconcileDeclared(targetPath) {
|
|
605
|
+
let ground;
|
|
606
|
+
try {
|
|
607
|
+
ground = fs.lstatSync(targetPath).isSymbolicLink() ? path.resolve(targetPath) : fs.realpathSync(targetPath);
|
|
608
|
+
} catch {
|
|
609
|
+
throw new Error(`declared user-home root is unavailable: ${targetPath}`);
|
|
610
|
+
}
|
|
611
|
+
const covering = treeRootCovering(ground);
|
|
612
|
+
if (!covering) throw new Error(`declared user-home root was not registered: ${targetPath}`);
|
|
613
|
+
observer.scanNow(covering.rootId);
|
|
614
|
+
flushIfDirty();
|
|
615
|
+
requireLiveWatchers(covering.path);
|
|
616
|
+
return { rootPath: covering.path, record: rootRecordOf(covering.rootId) };
|
|
617
|
+
}
|
|
618
|
+
|
|
599
619
|
/** The boundary's whole convergence pass, one act: loss convicted
|
|
600
620
|
* first (loudly — a conviction is never a retry), edges followed,
|
|
601
621
|
* lenses reconciled, renderings projected. Startup, the explicit
|
|
@@ -626,6 +646,7 @@ function createRegistration(options) {
|
|
|
626
646
|
registry,
|
|
627
647
|
add,
|
|
628
648
|
addMany,
|
|
649
|
+
reconcileDeclared,
|
|
629
650
|
|
|
630
651
|
/**
|
|
631
652
|
* `amalgm register <existing-path> --as <address>` — the managed
|
|
@@ -47,6 +47,10 @@ const Database = require('better-sqlite3');
|
|
|
47
47
|
const { STORAGE_DIR } = require('../config');
|
|
48
48
|
const { ensureDir } = require('../lib/storage');
|
|
49
49
|
const { ensureUserHome } = require('../lib/layout');
|
|
50
|
+
const {
|
|
51
|
+
materializeSystemToolArtifacts,
|
|
52
|
+
migrateLegacyToolboxCatalog,
|
|
53
|
+
} = require('../toolbox/artifacts');
|
|
50
54
|
const { createObserver } = require('../observer');
|
|
51
55
|
const { shouldEnroll } = require('../workspace/tree/inclusion');
|
|
52
56
|
const { classifyFile } = require('./classify');
|
|
@@ -58,8 +62,8 @@ const { transportBytes, localState, unpack } = require('./repo-states');
|
|
|
58
62
|
const { machineId, userId } = require('../workspace/identity');
|
|
59
63
|
|
|
60
64
|
const ENTITIES_DIR = path.join(STORAGE_DIR, 'entities');
|
|
61
|
-
const USER_HOME_CLOUD_CONTRACT = 'amalgm-user-home@
|
|
62
|
-
const USER_HOME_CLOUD_SCHEMA_VERSION =
|
|
65
|
+
const USER_HOME_CLOUD_CONTRACT = 'amalgm-user-home@2';
|
|
66
|
+
const USER_HOME_CLOUD_SCHEMA_VERSION = 2;
|
|
63
67
|
const ATOMIC_REGISTRATION_THRESHOLD = 500;
|
|
64
68
|
|
|
65
69
|
let live = null;
|
|
@@ -260,6 +264,19 @@ function createRegistrationService({
|
|
|
260
264
|
*/
|
|
261
265
|
async function bootstrapUserHome({ identitySnapshot = null } = {}) {
|
|
262
266
|
const declaration = ensureUserHome(homeDir);
|
|
267
|
+
// Materialize every built-in tool before the shared registration boundary
|
|
268
|
+
// first sees `toolbox/`. Each file is an exact managed child; user tools
|
|
269
|
+
// remain ordinary sibling files. A legacy aggregate is moved outside the
|
|
270
|
+
// watched tree only after per-tool replacements exist.
|
|
271
|
+
const toolbox = declaration.roots.find((entry) => entry.key === 'toolbox');
|
|
272
|
+
if (!toolbox) throw new Error('declared user-home is missing its toolbox root');
|
|
273
|
+
const migrated = migrateLegacyToolboxCatalog({ toolboxDir: toolbox.path });
|
|
274
|
+
const systemTools = materializeSystemToolArtifacts({ toolboxDir: toolbox.path });
|
|
275
|
+
declaration.materializedExactPaths = [
|
|
276
|
+
...(declaration.materializedExactPaths || []),
|
|
277
|
+
...(migrated.artifactPaths || []).map((target) => path.relative(homeDir, target).split(path.sep).join('/')),
|
|
278
|
+
...(systemTools.changedRelativePaths || []),
|
|
279
|
+
];
|
|
263
280
|
const seeds = identitySeeds(identitySnapshot, declaration);
|
|
264
281
|
const addedRoots = boundary.current.addMany(declaration.roots.map((entry) => ({
|
|
265
282
|
targetPath: entry.path,
|
|
@@ -274,25 +291,50 @@ function createRegistrationService({
|
|
|
274
291
|
};
|
|
275
292
|
});
|
|
276
293
|
|
|
277
|
-
const
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const recordAt = (relativePath) => {
|
|
294
|
+
const recordAt = (rootKey, relativePath, { required = true } = {}) => {
|
|
295
|
+
const root = roots.find((entry) => entry.key === rootKey);
|
|
296
|
+
if (!root) throw new Error(`declared user-home root was not registered: ${rootKey}`);
|
|
297
|
+
const tree = boundary.current.tree(root.path);
|
|
298
|
+
if (!tree?.root()) throw new Error(`declared user-home root was not registered: ${root.path}`);
|
|
283
299
|
let record = tree.root();
|
|
284
300
|
for (const name of relativePath.split('/')) {
|
|
285
301
|
record = tree.registry.activeAt(record.uuid, name);
|
|
286
|
-
if (!record)
|
|
302
|
+
if (!record) {
|
|
303
|
+
if (!required) return null;
|
|
304
|
+
throw new Error(`declared user-home target was not registered: ${rootKey}/${relativePath}`);
|
|
305
|
+
}
|
|
287
306
|
}
|
|
288
307
|
return record;
|
|
289
308
|
};
|
|
290
309
|
|
|
310
|
+
// An upgraded installation can already watch an older declared root.
|
|
311
|
+
// Re-registering that root must remain only an availability check, so the
|
|
312
|
+
// narrow exception is a root holding an exact node that the layout just
|
|
313
|
+
// materialized or that the registry proves missing. Detect and Record
|
|
314
|
+
// still create/update its identity through the shared boundary.
|
|
315
|
+
const rootsToReconcile = new Set((declaration.materializedExactPaths || [])
|
|
316
|
+
.map((relativePath) => relativePath.split('/')[0]));
|
|
317
|
+
for (const entry of declaration.exactManaged) {
|
|
318
|
+
if (entry.relativePath === entry.rootKey) continue;
|
|
319
|
+
const relativePath = entry.relativePath.slice(entry.rootKey.length + 1);
|
|
320
|
+
if (!recordAt(entry.rootKey, relativePath, { required: false })) {
|
|
321
|
+
rootsToReconcile.add(entry.rootKey);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
for (const rootKey of rootsToReconcile) {
|
|
325
|
+
const root = roots.find((candidate) => candidate.key === rootKey);
|
|
326
|
+
if (!root) throw new Error(`declared user-home root was not registered: ${rootKey}`);
|
|
327
|
+
boundary.current.reconcileDeclared(root.path);
|
|
328
|
+
roots = roots.map((candidate) => candidate.key === rootKey
|
|
329
|
+
? { ...candidate, record: boundary.current.tree(candidate.path).root() }
|
|
330
|
+
: candidate);
|
|
331
|
+
}
|
|
332
|
+
|
|
291
333
|
let exactManaged = declaration.exactManaged.map((entry) => {
|
|
292
334
|
const root = roots.find((candidate) => candidate.key === entry.rootKey);
|
|
293
335
|
const isRoot = entry.relativePath === entry.rootKey;
|
|
294
336
|
const relativePath = isRoot ? '' : entry.relativePath.slice(entry.rootKey.length + 1);
|
|
295
|
-
const record = isRoot ? root?.record : recordAt(relativePath);
|
|
337
|
+
const record = isRoot ? root?.record : recordAt(entry.rootKey, relativePath);
|
|
296
338
|
if (!record) throw new Error(`declared user-home managed target was not registered: ${entry.relativePath}`);
|
|
297
339
|
return {
|
|
298
340
|
key: entry.key,
|
|
@@ -767,6 +809,51 @@ function createRegistrationService({
|
|
|
767
809
|
}
|
|
768
810
|
}
|
|
769
811
|
|
|
812
|
+
/** The status surface composes each layer's own facts without creating a
|
|
813
|
+
* second registry or treating local paths as cloud metadata. A cloud-only
|
|
814
|
+
* catalog record is intentionally visible with `localPath: null`; only
|
|
815
|
+
* `amalgm add` creates a watcher binding for it. */
|
|
816
|
+
function status() {
|
|
817
|
+
const cloud = entityCloud.status();
|
|
818
|
+
const localRecords = boundary.current.registry.syncRecords();
|
|
819
|
+
const localByUuid = new Map(localRecords.map((record) => [record.uuid, record]));
|
|
820
|
+
const cloudByUuid = new Map(cloud.catalog.records.map((record) => [record.uuid, record]));
|
|
821
|
+
const roots = observer.status().map((root) => {
|
|
822
|
+
const tree = boundary.current.tree(root.path);
|
|
823
|
+
const record = tree ? tree.root() : null;
|
|
824
|
+
return {
|
|
825
|
+
uuid: record?.uuid || null,
|
|
826
|
+
path: root.path,
|
|
827
|
+
kind: root.kind,
|
|
828
|
+
watching: root.watching,
|
|
829
|
+
degraded: root.degraded,
|
|
830
|
+
addressWatching: root.addressWatching,
|
|
831
|
+
addressDegraded: root.addressDegraded,
|
|
832
|
+
};
|
|
833
|
+
});
|
|
834
|
+
const rootByUuid = new Map(roots.filter((root) => root.uuid !== null).map((root) => [root.uuid, root]));
|
|
835
|
+
const uuids = new Set([...localByUuid.keys(), ...cloudByUuid.keys()]);
|
|
836
|
+
const entities = [...uuids].sort().map((uuid) => {
|
|
837
|
+
const local = localByUuid.get(uuid) || null;
|
|
838
|
+
const record = cloudByUuid.get(uuid) || local;
|
|
839
|
+
let localPath = null;
|
|
840
|
+
if (local) {
|
|
841
|
+
try { localPath = boundary.current.addressOf(uuid); } catch {}
|
|
842
|
+
}
|
|
843
|
+
return {
|
|
844
|
+
...record,
|
|
845
|
+
// A local journal projects its intent into the catalog before the
|
|
846
|
+
// gateway acknowledges it. Call that fact `catalogued`, not
|
|
847
|
+
// `cloudBacked`: the enclosing cloud summary says whether the
|
|
848
|
+
// catalog is actually pinned to an authoritative current head.
|
|
849
|
+
catalogued: cloudByUuid.has(uuid),
|
|
850
|
+
localPath,
|
|
851
|
+
root: rootByUuid.get(uuid) || null,
|
|
852
|
+
};
|
|
853
|
+
});
|
|
854
|
+
return { cloud, roots, entities };
|
|
855
|
+
}
|
|
856
|
+
|
|
770
857
|
return {
|
|
771
858
|
dir,
|
|
772
859
|
homeDir,
|
|
@@ -820,9 +907,13 @@ function createRegistrationService({
|
|
|
820
907
|
const received = entityCloud.receiveDownload({
|
|
821
908
|
resourceId, contentHash, kind, partIndex, sha256, dataBase64,
|
|
822
909
|
});
|
|
823
|
-
|
|
910
|
+
// A mid-file part cannot complete any waiting tree; only a finished
|
|
911
|
+
// content head can. Advancing on every part re-walks every waiting
|
|
912
|
+
// job once per part, which is quadratic over a large download.
|
|
913
|
+
if (received?.complete) advanceMaterializations();
|
|
824
914
|
return received;
|
|
825
915
|
},
|
|
916
|
+
status,
|
|
826
917
|
advanceMaterializations,
|
|
827
918
|
bootstrapUserHome,
|
|
828
919
|
exactUserHomeCloudRecords,
|
|
@@ -686,7 +686,7 @@ function createRegistry(options) {
|
|
|
686
686
|
|
|
687
687
|
/**
|
|
688
688
|
* First cloud bootstrap may meet a pre-cloud local safe home. Its six
|
|
689
|
-
* roots and
|
|
689
|
+
* roots and ten exact managed nodes are platform truth, so they may
|
|
690
690
|
* adopt the authenticated cloud UUIDs without changing any user-owned
|
|
691
691
|
* descendant identity. This is one atomic graph rekey, never a delete or
|
|
692
692
|
* local-wins merge, and callers must prove that no ordinary entity-cloud
|
|
@@ -316,6 +316,12 @@ async function handleEntityRoutes(ctx) {
|
|
|
316
316
|
return true;
|
|
317
317
|
}
|
|
318
318
|
|
|
319
|
+
if (ctx.pathname === '/entities/status' && ctx.method === 'GET') {
|
|
320
|
+
ctx.res.setHeader('Cache-Control', 'no-store');
|
|
321
|
+
ctx.sendJson(200, registrationService().status());
|
|
322
|
+
return true;
|
|
323
|
+
}
|
|
324
|
+
|
|
319
325
|
const materializationMatch = /^\/entities\/add\/(mat_[A-Za-z0-9_-]+)$/.exec(ctx.pathname);
|
|
320
326
|
if (materializationMatch && ctx.method === 'GET') {
|
|
321
327
|
const service = registrationService();
|