amalgm 0.1.257 → 0.1.259
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/cli.js +207 -3
- package/lib/layout.js +30 -9
- package/lib/shared-realtime-tunnel.js +12 -14
- package/package.json +2 -2
- package/runtime/scripts/amalgm-mcp/lib/layout.js +32 -9
- package/runtime/scripts/amalgm-mcp/observer/index.js +65 -4
- package/runtime/scripts/amalgm-mcp/observer/scan.js +60 -1
- package/runtime/scripts/amalgm-mcp/observer/store.js +43 -8
- package/runtime/scripts/amalgm-mcp/registration/entity-cloud.js +186 -32
- package/runtime/scripts/amalgm-mcp/registration/entity-content.js +33 -6
- package/runtime/scripts/amalgm-mcp/registration/index.js +21 -0
- package/runtime/scripts/amalgm-mcp/registration/service.js +117 -11
- package/runtime/scripts/amalgm-mcp/registry/index.js +1 -1
- package/runtime/scripts/amalgm-mcp/repocard/capture.js +16 -8
- package/runtime/scripts/amalgm-mcp/server/routes/entities.js +6 -0
- package/runtime/scripts/amalgm-mcp/tests/entity-cloud.test.js +67 -8
- package/runtime/scripts/amalgm-mcp/tests/entity-materialization.test.js +3 -3
- package/runtime/scripts/amalgm-mcp/tests/observer.test.js +85 -0
- package/runtime/scripts/amalgm-mcp/tests/registration.service.test.js +95 -3
- package/runtime/scripts/amalgm-mcp/tests/repocard.test.js +21 -0
- package/runtime/scripts/amalgm-mcp/tests/system-catalog.test.js +3 -6
- package/runtime/scripts/amalgm-mcp/tests/toolbox-artifacts.test.js +88 -0
- package/runtime/scripts/amalgm-mcp/tests/toolbox-cloud-registration.test.js +91 -0
- package/runtime/scripts/amalgm-mcp/toolbox/artifacts.js +225 -0
- package/runtime/scripts/amalgm-mcp/toolbox/store.js +59 -10
- package/runtime/scripts/amalgm-mcp/toolbox/system-catalog.js +14 -5
|
@@ -78,6 +78,20 @@ function rowToFile(row) {
|
|
|
78
78
|
};
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
function rowToDir(row) {
|
|
82
|
+
return {
|
|
83
|
+
dirId: row.dir_id, rootId: row.root_id, relPath: row.rel_path,
|
|
84
|
+
device: row.device, inode: row.inode,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function rowToLink(row) {
|
|
89
|
+
return {
|
|
90
|
+
linkId: row.link_id, rootId: row.root_id, relPath: row.rel_path,
|
|
91
|
+
device: row.device, inode: row.inode, target: row.target,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
81
95
|
const TABLES = ['observer_roots', 'observer_files', 'observer_dirs', 'observer_links'];
|
|
82
96
|
|
|
83
97
|
function rowToRoot(row) {
|
|
@@ -127,6 +141,24 @@ function createStore(db) {
|
|
|
127
141
|
db.exec(SCHEMA);
|
|
128
142
|
db.pragma(`user_version = ${GENERATION}`);
|
|
129
143
|
|
|
144
|
+
/**
|
|
145
|
+
* One directory's immediate child rows — the scoped scan's slice of
|
|
146
|
+
* memory, served by the (root_id, rel_path) unique index. '0' is the
|
|
147
|
+
* character after '/', so [`dir/`, `dir0`) is exactly the rel_path
|
|
148
|
+
* range under the directory; depth filters to immediate children
|
|
149
|
+
* because rel_path is flat text.
|
|
150
|
+
*/
|
|
151
|
+
function immediateRows(table, rootId, relDir) {
|
|
152
|
+
if (relDir === '') {
|
|
153
|
+
return db.prepare(`SELECT * FROM ${table} WHERE root_id = ? AND instr(rel_path, '/') = 0 ORDER BY rel_path`)
|
|
154
|
+
.all(rootId);
|
|
155
|
+
}
|
|
156
|
+
const prefix = `${relDir}/`;
|
|
157
|
+
return db.prepare(`SELECT * FROM ${table} WHERE root_id = ? AND rel_path >= ? AND rel_path < ? ORDER BY rel_path`)
|
|
158
|
+
.all(rootId, prefix, `${relDir}0`)
|
|
159
|
+
.filter((row) => !row.rel_path.slice(prefix.length).includes('/'));
|
|
160
|
+
}
|
|
161
|
+
|
|
130
162
|
return {
|
|
131
163
|
addRoot({ rootId, path, kind, device = null, inode = null }) {
|
|
132
164
|
db.prepare('INSERT INTO observer_roots (root_id, path, kind, device, inode) VALUES (?, ?, ?, ?, ?)')
|
|
@@ -177,6 +209,15 @@ function createStore(db) {
|
|
|
177
209
|
return db.prepare('SELECT * FROM observer_files WHERE root_id = ? ORDER BY rel_path').all(rootId)
|
|
178
210
|
.map(rowToFile);
|
|
179
211
|
},
|
|
212
|
+
filesInDir(rootId, relDir) {
|
|
213
|
+
return immediateRows('observer_files', rootId, relDir).map(rowToFile);
|
|
214
|
+
},
|
|
215
|
+
dirsInDir(rootId, relDir) {
|
|
216
|
+
return immediateRows('observer_dirs', rootId, relDir).map(rowToDir);
|
|
217
|
+
},
|
|
218
|
+
linksInDir(rootId, relDir) {
|
|
219
|
+
return immediateRows('observer_links', rootId, relDir).map(rowToLink);
|
|
220
|
+
},
|
|
180
221
|
fileByPath(rootId, relPath) {
|
|
181
222
|
return rowToFile(
|
|
182
223
|
db.prepare('SELECT * FROM observer_files WHERE root_id = ? AND rel_path = ?').get(rootId, relPath),
|
|
@@ -204,10 +245,7 @@ function createStore(db) {
|
|
|
204
245
|
|
|
205
246
|
dirsForRoot(rootId) {
|
|
206
247
|
return db.prepare('SELECT * FROM observer_dirs WHERE root_id = ? ORDER BY rel_path').all(rootId)
|
|
207
|
-
.map(
|
|
208
|
-
dirId: row.dir_id, rootId: row.root_id, relPath: row.rel_path,
|
|
209
|
-
device: row.device, inode: row.inode,
|
|
210
|
-
}));
|
|
248
|
+
.map(rowToDir);
|
|
211
249
|
},
|
|
212
250
|
/**
|
|
213
251
|
* One compare result is ONE truth transition. Movers may exchange
|
|
@@ -240,10 +278,7 @@ function createStore(db) {
|
|
|
240
278
|
// per-row like files — every commit is a valid last-known truth.
|
|
241
279
|
linksForRoot(rootId) {
|
|
242
280
|
return db.prepare('SELECT * FROM observer_links WHERE root_id = ? ORDER BY rel_path').all(rootId)
|
|
243
|
-
.map(
|
|
244
|
-
linkId: row.link_id, rootId: row.root_id, relPath: row.rel_path,
|
|
245
|
-
device: row.device, inode: row.inode, target: row.target,
|
|
246
|
-
}));
|
|
281
|
+
.map(rowToLink);
|
|
247
282
|
},
|
|
248
283
|
insertLink({ linkId, rootId, relPath, device, inode, target }) {
|
|
249
284
|
db.prepare('INSERT INTO observer_links (link_id, root_id, rel_path, device, inode, target) VALUES (?, ?, ?, ?, ?, ?)')
|
|
@@ -11,12 +11,13 @@
|
|
|
11
11
|
|
|
12
12
|
const crypto = require('crypto');
|
|
13
13
|
|
|
14
|
-
// `@1`
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
18
|
-
// home; later user ground enters
|
|
19
|
-
|
|
14
|
+
// `@1` could project arbitrary pre-cloud registrations. `@2` omitted the
|
|
15
|
+
// declared built-in tool entities, so its toolbox root could not prove the
|
|
16
|
+
// snapshot membership it named. A resource contract names immutable authority
|
|
17
|
+
// semantics: both remain quarantined rather than reinterpreted or overwritten.
|
|
18
|
+
// `@3` begins with the complete verified user home; later user ground enters
|
|
19
|
+
// exclusively through `amalgm register`.
|
|
20
|
+
const ENTITY_CLOUD_CONTRACT = 'amalgm-entity-registry@3';
|
|
20
21
|
const ENTITY_CLOUD_SCHEMA_VERSION = 1;
|
|
21
22
|
const BOOTSTRAP_RECONCILIATION_STATE = 'reconciling-bootstrap';
|
|
22
23
|
const SNAPSHOT_REPLACE_OPERATION = 'entity.snapshot.replace';
|
|
@@ -191,6 +192,56 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
191
192
|
};
|
|
192
193
|
}
|
|
193
194
|
|
|
195
|
+
// The catalog mirrors the compacted cloud head as long as every head
|
|
196
|
+
// advance since the last installed snapshot was applied to it in order:
|
|
197
|
+
// journal() upserts this device's own records, ingest() upserts remote
|
|
198
|
+
// ones, and the gateway's fold regenerates only derived container
|
|
199
|
+
// versions, which every catalog consumer already tolerates. A version
|
|
200
|
+
// gap, a remote snapshot replacement, or a restart drops this back to
|
|
201
|
+
// whatever the durable pin proves, and the next snapshot download
|
|
202
|
+
// re-establishes it.
|
|
203
|
+
let catalogLive = null;
|
|
204
|
+
// Mutation rows whose content-upload cursors already exist on disk.
|
|
205
|
+
const contentQueuedSequences = new Set();
|
|
206
|
+
|
|
207
|
+
function catalogPinnedAtHead(current) {
|
|
208
|
+
if (!current) return false;
|
|
209
|
+
const pin = db.prepare(`
|
|
210
|
+
SELECT resource_id, authority_epoch, snapshot_version
|
|
211
|
+
FROM entity_cloud_catalog_state WHERE singleton = 1
|
|
212
|
+
`).get();
|
|
213
|
+
return Boolean(pin && pin.resource_id === current.resourceId
|
|
214
|
+
&& Number(pin.authority_epoch) === current.authorityEpoch
|
|
215
|
+
&& Number(pin.snapshot_version) === current.headVersion);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function unsettledMutationCount(resourceId) {
|
|
219
|
+
return Number(db.prepare(`
|
|
220
|
+
SELECT COUNT(*) AS count FROM entity_cloud_mutations
|
|
221
|
+
WHERE resource_id = ? AND mutation_state <> 'synced'
|
|
222
|
+
`).get(resourceId).count);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function catalogIsLive(current) {
|
|
226
|
+
if (catalogLive === null) {
|
|
227
|
+
catalogLive = catalogPinnedAtHead(current) && unsettledMutationCount(current.resourceId) === 0;
|
|
228
|
+
}
|
|
229
|
+
return catalogLive;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function pinCatalogState({ resourceId, authorityEpoch, headVersion }) {
|
|
233
|
+
db.prepare(`
|
|
234
|
+
INSERT INTO entity_cloud_catalog_state (
|
|
235
|
+
singleton, resource_id, authority_epoch, snapshot_version, updated_at
|
|
236
|
+
) VALUES (1, ?, ?, ?, ?)
|
|
237
|
+
ON CONFLICT(singleton) DO UPDATE SET
|
|
238
|
+
resource_id = excluded.resource_id,
|
|
239
|
+
authority_epoch = excluded.authority_epoch,
|
|
240
|
+
snapshot_version = excluded.snapshot_version,
|
|
241
|
+
updated_at = excluded.updated_at
|
|
242
|
+
`).run(resourceId, authorityEpoch, headVersion, now());
|
|
243
|
+
}
|
|
244
|
+
|
|
194
245
|
function saveReplica(value) {
|
|
195
246
|
db.prepare(`
|
|
196
247
|
INSERT INTO entity_cloud_replica (
|
|
@@ -236,16 +287,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
236
287
|
// A later mutation may update an individual catalog row so live state
|
|
237
288
|
// can be inspected, but that must never make the whole catalog look
|
|
238
289
|
// like a newly compacted snapshot that is safe to materialize.
|
|
239
|
-
|
|
240
|
-
INSERT INTO entity_cloud_catalog_state (
|
|
241
|
-
singleton, resource_id, authority_epoch, snapshot_version, updated_at
|
|
242
|
-
) VALUES (1, ?, ?, ?, ?)
|
|
243
|
-
ON CONFLICT(singleton) DO UPDATE SET
|
|
244
|
-
resource_id = excluded.resource_id,
|
|
245
|
-
authority_epoch = excluded.authority_epoch,
|
|
246
|
-
snapshot_version = excluded.snapshot_version,
|
|
247
|
-
updated_at = excluded.updated_at
|
|
248
|
-
`).run(resourceId, authorityEpoch, headVersion, now());
|
|
290
|
+
pinCatalogState({ resourceId, authorityEpoch, headVersion });
|
|
249
291
|
})();
|
|
250
292
|
return normalized;
|
|
251
293
|
}
|
|
@@ -523,6 +565,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
523
565
|
});
|
|
524
566
|
enqueueBootstrapReconciliation(trustedBootstrap, next);
|
|
525
567
|
})();
|
|
568
|
+
catalogLive = false;
|
|
526
569
|
return {
|
|
527
570
|
resourceId,
|
|
528
571
|
records: snapshot.records.length,
|
|
@@ -557,6 +600,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
557
600
|
state: complete ? 'active' : BOOTSTRAP_RECONCILIATION_STATE,
|
|
558
601
|
snapshotCutoff: null,
|
|
559
602
|
});
|
|
603
|
+
catalogLive = complete;
|
|
560
604
|
return {
|
|
561
605
|
resourceId,
|
|
562
606
|
records: snapshot.records.length,
|
|
@@ -604,6 +648,10 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
604
648
|
state: 'active',
|
|
605
649
|
snapshotCutoff: null,
|
|
606
650
|
});
|
|
651
|
+
// A snapshot installed over unsettled local mutations replaced their
|
|
652
|
+
// catalog upserts; liveness returns with the install that finds an
|
|
653
|
+
// empty outbox.
|
|
654
|
+
catalogLive = unsettledMutationCount(resourceId) === 0;
|
|
607
655
|
return {
|
|
608
656
|
resourceId,
|
|
609
657
|
records: snapshot.records.length,
|
|
@@ -710,6 +758,58 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
710
758
|
return materialization(db.prepare('SELECT * FROM entity_materializations WHERE job_id = ?').get(jobId));
|
|
711
759
|
}
|
|
712
760
|
|
|
761
|
+
/** Read-only cloud-replica truth for `amalgm status`. This deliberately
|
|
762
|
+
* reports a stale catalog as stale instead of presenting its records as a
|
|
763
|
+
* head that is safe to materialize. Status is an observer of authority,
|
|
764
|
+
* never a repair or a hidden snapshot handshake. */
|
|
765
|
+
function status() {
|
|
766
|
+
const current = replica();
|
|
767
|
+
const resourceId = current?.resourceId || null;
|
|
768
|
+
const mutationCounts = new Map();
|
|
769
|
+
if (resourceId) {
|
|
770
|
+
for (const row of db.prepare(`
|
|
771
|
+
SELECT mutation_state, COUNT(*) AS count
|
|
772
|
+
FROM entity_cloud_mutations
|
|
773
|
+
WHERE resource_id = ?
|
|
774
|
+
GROUP BY mutation_state
|
|
775
|
+
`).all(resourceId)) {
|
|
776
|
+
mutationCounts.set(row.mutation_state, Number(row.count));
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
const materializationCounts = new Map();
|
|
780
|
+
if (resourceId) {
|
|
781
|
+
for (const row of db.prepare(`
|
|
782
|
+
SELECT state, COUNT(*) AS count
|
|
783
|
+
FROM entity_materializations
|
|
784
|
+
WHERE resource_id = ?
|
|
785
|
+
GROUP BY state
|
|
786
|
+
`).all(resourceId)) {
|
|
787
|
+
materializationCounts.set(row.state, Number(row.count));
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
const pending = (mutationCounts.get('pending') || 0) + (mutationCounts.get('syncing') || 0);
|
|
791
|
+
const pinned = current ? catalogPinnedAtHead(current) : false;
|
|
792
|
+
return {
|
|
793
|
+
identity: identity(),
|
|
794
|
+
replica: current,
|
|
795
|
+
catalog: {
|
|
796
|
+
pinned,
|
|
797
|
+
current: Boolean(current?.state === 'active' && pinned && pending === 0),
|
|
798
|
+
records: resourceId ? catalogRecords(resourceId) : [],
|
|
799
|
+
},
|
|
800
|
+
outbox: {
|
|
801
|
+
pending: mutationCounts.get('pending') || 0,
|
|
802
|
+
syncing: mutationCounts.get('syncing') || 0,
|
|
803
|
+
rejected: mutationCounts.get('rejected') || 0,
|
|
804
|
+
},
|
|
805
|
+
materializations: {
|
|
806
|
+
waitingForContent: materializationCounts.get('waiting_for_content') || 0,
|
|
807
|
+
materializing: materializationCounts.get('materializing') || 0,
|
|
808
|
+
failed: materializationCounts.get('failed') || 0,
|
|
809
|
+
},
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
|
|
713
813
|
/** A completed add is a live binding, not a one-time download. A remote
|
|
714
814
|
* record beneath that binding moves the durable job back to its content
|
|
715
815
|
* gate. The service owns the later filesystem application; this module
|
|
@@ -810,17 +910,21 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
810
910
|
// Later registrations journal their record transactionally in SQLite.
|
|
811
911
|
// Derive its content-upload cursor only after that commit exists, so a
|
|
812
912
|
// failed transaction cannot publish a private object no entity names.
|
|
913
|
+
// Each mutation's operation is immutable, so it is scanned once per
|
|
914
|
+
// process; a restart simply rescans the still-unsettled rows.
|
|
813
915
|
const rows = db.prepare(`
|
|
814
|
-
SELECT operation_json FROM entity_cloud_mutations
|
|
916
|
+
SELECT sequence, operation_json FROM entity_cloud_mutations
|
|
815
917
|
WHERE resource_id = ? AND mutation_state IN ('pending', 'syncing')
|
|
816
918
|
ORDER BY sequence ASC
|
|
817
919
|
`).all(current.resourceId);
|
|
818
920
|
for (const row of rows) {
|
|
921
|
+
if (contentQueuedSequences.has(row.sequence)) continue;
|
|
819
922
|
const operation = JSON.parse(row.operation_json);
|
|
820
923
|
const records = operation?.kind === SNAPSHOT_REPLACE_OPERATION
|
|
821
924
|
? operation?.snapshot?.records
|
|
822
925
|
: [operation?.record];
|
|
823
926
|
for (const record of records || []) queueContent(record, current.resourceId, { queue: true });
|
|
927
|
+
contentQueuedSequences.add(row.sequence);
|
|
824
928
|
}
|
|
825
929
|
return content.claim({ resourceId: current.resourceId, leaseMs });
|
|
826
930
|
}
|
|
@@ -850,31 +954,59 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
850
954
|
.get(resourceId, mutationId);
|
|
851
955
|
if (!row) throw new Error(`unknown entity cloud mutation ${mutationId}`);
|
|
852
956
|
if (row.mutation_state === 'synced') {
|
|
853
|
-
const unsettled = db.prepare('SELECT COUNT(*) AS count FROM entity_cloud_mutations WHERE resource_id = ? AND mutation_state <> \'synced\'')
|
|
854
|
-
.get(resourceId);
|
|
855
957
|
return {
|
|
856
958
|
duplicate: true,
|
|
857
959
|
version: Number(row.official_version),
|
|
858
|
-
requiresSnapshot: current.state
|
|
960
|
+
requiresSnapshot: current.state === BOOTSTRAP_RECONCILIATION_STATE
|
|
961
|
+
? unsettledMutationCount(resourceId) === 0
|
|
962
|
+
: !catalogIsLive(current),
|
|
859
963
|
};
|
|
860
964
|
}
|
|
861
965
|
if (!['pending', 'syncing'].includes(row.mutation_state)) throw new Error(`entity cloud mutation ${mutationId} cannot sync from ${row.mutation_state}`);
|
|
966
|
+
const live = current.state === BOOTSTRAP_RECONCILIATION_STATE ? false : catalogIsLive(current);
|
|
862
967
|
db.prepare(`
|
|
863
968
|
UPDATE entity_cloud_mutations
|
|
864
969
|
SET mutation_state = 'synced', official_version = ?, cloud_committed_at = ?,
|
|
865
970
|
next_attempt_at = NULL, last_error = NULL, updated_at = ?
|
|
866
971
|
WHERE sequence = ?
|
|
867
972
|
`).run(Number(version), String(committedAt || now()), now(), row.sequence);
|
|
868
|
-
|
|
973
|
+
const previousHead = current.headVersion;
|
|
974
|
+
if (Number(version) > previousHead) {
|
|
869
975
|
saveReplica({ ...current, headVersion: Number(version) });
|
|
870
976
|
}
|
|
871
|
-
const unsettled =
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
977
|
+
const unsettled = unsettledMutationCount(resourceId);
|
|
978
|
+
if (current.state === BOOTSTRAP_RECONCILIATION_STATE) {
|
|
979
|
+
return { duplicate: false, version: Number(version), requiresSnapshot: unsettled === 0 };
|
|
980
|
+
}
|
|
981
|
+
// Local convergence instead of a snapshot re-download per ack. This
|
|
982
|
+
// device journaled these exact records into the catalog when the
|
|
983
|
+
// mutation was created, so an in-order acknowledgement leaves the
|
|
984
|
+
// catalog equal to the compacted head by construction. Anything out
|
|
985
|
+
// of order gives liveness up until the next installed snapshot.
|
|
986
|
+
let operation = null;
|
|
987
|
+
try { operation = JSON.parse(row.operation_json); } catch {}
|
|
988
|
+
if (operation?.kind === SNAPSHOT_REPLACE_OPERATION) {
|
|
989
|
+
if (unsettled === 0 && Array.isArray(operation?.snapshot?.records)) {
|
|
990
|
+
// The replacement operation IS the complete graph at this version.
|
|
991
|
+
replaceCatalog({
|
|
992
|
+
resourceId,
|
|
993
|
+
authorityEpoch: current.authorityEpoch,
|
|
994
|
+
headVersion: Number(version),
|
|
995
|
+
records: operation.snapshot.records,
|
|
996
|
+
});
|
|
997
|
+
catalogLive = true;
|
|
998
|
+
} else {
|
|
999
|
+
catalogLive = false;
|
|
1000
|
+
}
|
|
1001
|
+
} else if (!operation || Number(version) !== previousHead + 1) {
|
|
1002
|
+
catalogLive = false;
|
|
1003
|
+
} else {
|
|
1004
|
+
catalogLive = live;
|
|
1005
|
+
if (catalogLive && unsettled === 0) {
|
|
1006
|
+
pinCatalogState({ resourceId, authorityEpoch: current.authorityEpoch, headVersion: Number(version) });
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
return { duplicate: false, version: Number(version), requiresSnapshot: !catalogLive };
|
|
878
1010
|
})();
|
|
879
1011
|
}
|
|
880
1012
|
|
|
@@ -912,10 +1044,17 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
912
1044
|
if (envelope?.operationKind === SNAPSHOT_REPLACE_OPERATION
|
|
913
1045
|
&& envelope?.operation?.kind === SNAPSHOT_REPLACE_OPERATION) {
|
|
914
1046
|
// The complete replacement is imported only from the active snapshot
|
|
915
|
-
// the gateway
|
|
916
|
-
//
|
|
1047
|
+
// the gateway compacts at lookup. Consuming its tail would reintroduce
|
|
1048
|
+
// the very partial-graph state this operation prevents.
|
|
917
1049
|
if (Number(version) > current.headVersion) saveReplica({ ...current, headVersion: Number(version) });
|
|
918
|
-
|
|
1050
|
+
catalogLive = false;
|
|
1051
|
+
return {
|
|
1052
|
+
duplicate: true,
|
|
1053
|
+
version: Number(version),
|
|
1054
|
+
snapshotRequired: true,
|
|
1055
|
+
requiresSnapshot: true,
|
|
1056
|
+
materializations: [],
|
|
1057
|
+
};
|
|
919
1058
|
}
|
|
920
1059
|
if (envelope?.operationKind !== 'entity.upsert' || envelope?.operation?.kind !== 'entity.upsert') {
|
|
921
1060
|
throw new Error('cloud entity mutation does not belong to this registry');
|
|
@@ -951,10 +1090,24 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
951
1090
|
headVersion: Number(version),
|
|
952
1091
|
record,
|
|
953
1092
|
});
|
|
954
|
-
|
|
1093
|
+
// A remote record applied in official order keeps the catalog at the
|
|
1094
|
+
// compacted head, so no snapshot re-download is needed for it either.
|
|
1095
|
+
const live = catalogIsLive(current);
|
|
1096
|
+
const previousHead = current.headVersion;
|
|
1097
|
+
if (Number(version) > previousHead) saveReplica({ ...current, headVersion: Number(version) });
|
|
1098
|
+
if (Number(version) !== previousHead + 1) {
|
|
1099
|
+
catalogLive = false;
|
|
1100
|
+
} else if (live && unsettledMutationCount(current.resourceId) === 0) {
|
|
1101
|
+
pinCatalogState({
|
|
1102
|
+
resourceId: current.resourceId,
|
|
1103
|
+
authorityEpoch: current.authorityEpoch,
|
|
1104
|
+
headVersion: Number(version),
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
955
1107
|
return {
|
|
956
1108
|
duplicate: true,
|
|
957
1109
|
version: Number(version),
|
|
1110
|
+
requiresSnapshot: !catalogLive,
|
|
958
1111
|
materializations: refreshMaterializations({ resourceId: current.resourceId, record, version }),
|
|
959
1112
|
};
|
|
960
1113
|
}
|
|
@@ -980,6 +1133,7 @@ function createEntityCloud({ db, userId, deviceId, content = null, repoStates =
|
|
|
980
1133
|
completeMaterialization,
|
|
981
1134
|
failMaterialization,
|
|
982
1135
|
materializationStatus,
|
|
1136
|
+
status,
|
|
983
1137
|
hasUnsettledLocalMutation,
|
|
984
1138
|
catalogTree,
|
|
985
1139
|
replica,
|
|
@@ -15,7 +15,10 @@ const fs = require('fs');
|
|
|
15
15
|
const path = require('path');
|
|
16
16
|
|
|
17
17
|
const CONTENT_CONTRACT = 'amalgm-entity-content@1';
|
|
18
|
-
|
|
18
|
+
// One chunk is one WAN round trip. The gateway accepts parts up to this same
|
|
19
|
+
// bound (ENTITY_CONTENT_PART_BYTES); older 256KB chunks remain valid because
|
|
20
|
+
// the manifest rule is `bytes <= CHUNK_BYTES`, never an exact size.
|
|
21
|
+
const CHUNK_BYTES = 4 * 1024 * 1024;
|
|
19
22
|
const HASH = /^[0-9a-f]{64}$/;
|
|
20
23
|
|
|
21
24
|
function digest(bytes) {
|
|
@@ -41,6 +44,10 @@ function createEntityContent({ dir }) {
|
|
|
41
44
|
const chunksDir = path.join(root, 'chunks');
|
|
42
45
|
const manifestsDir = path.join(root, 'manifests');
|
|
43
46
|
const uploadsDir = path.join(root, 'uploads');
|
|
47
|
+
// Unfinished uploads per resource, so claim() does not re-read the whole
|
|
48
|
+
// ledger directory for every part it sends. The on-disk ledger stays the
|
|
49
|
+
// durable truth; this set is rebuilt from it once per process.
|
|
50
|
+
const pendingUploads = new Map();
|
|
44
51
|
|
|
45
52
|
function ensureDirs() {
|
|
46
53
|
for (const target of [chunksDir, manifestsDir, uploadsDir]) {
|
|
@@ -236,14 +243,27 @@ function createEntityContent({ dir }) {
|
|
|
236
243
|
}
|
|
237
244
|
}
|
|
238
245
|
|
|
246
|
+
function pendingSet(resourceId) {
|
|
247
|
+
let pending = pendingUploads.get(resourceId);
|
|
248
|
+
if (!pending) {
|
|
249
|
+
pending = new Set(uploadEntries(resourceId));
|
|
250
|
+
pendingUploads.set(resourceId, pending);
|
|
251
|
+
}
|
|
252
|
+
return pending;
|
|
253
|
+
}
|
|
254
|
+
|
|
239
255
|
function ensureUpload({ resourceId, contentHash }) {
|
|
240
256
|
if (typeof resourceId !== 'string' || !resourceId) throw new Error('entity content upload requires a resource id');
|
|
241
257
|
const content = manifest(contentHash);
|
|
242
258
|
if (!content) throw new Error(`content ${contentHash} was not captured before its entity record committed`);
|
|
243
259
|
const current = readUpload(resourceId, contentHash, content);
|
|
244
|
-
if (current)
|
|
260
|
+
if (current) {
|
|
261
|
+
if (!current.manifestUploaded) pendingSet(resourceId).add(contentHash);
|
|
262
|
+
return current;
|
|
263
|
+
}
|
|
245
264
|
const created = initialUpload(resourceId, contentHash);
|
|
246
265
|
saveUpload(created);
|
|
266
|
+
pendingSet(resourceId).add(contentHash);
|
|
247
267
|
return created;
|
|
248
268
|
}
|
|
249
269
|
|
|
@@ -269,11 +289,15 @@ function createEntityContent({ dir }) {
|
|
|
269
289
|
function claim({ resourceId, leaseMs = 30_000 }) {
|
|
270
290
|
ensureDirs();
|
|
271
291
|
const at = Date.now();
|
|
272
|
-
|
|
292
|
+
const pending = pendingSet(resourceId);
|
|
293
|
+
for (const contentHash of pending) {
|
|
273
294
|
const content = manifest(contentHash);
|
|
274
295
|
if (!content) throw new Error(`entity content ${contentHash} disappeared from the local cache`);
|
|
275
296
|
const upload = readUpload(resourceId, contentHash, content);
|
|
276
|
-
if (!upload || upload.manifestUploaded)
|
|
297
|
+
if (!upload || upload.manifestUploaded) {
|
|
298
|
+
pending.delete(contentHash);
|
|
299
|
+
continue;
|
|
300
|
+
}
|
|
277
301
|
if (upload.leaseUntil && Date.parse(upload.leaseUntil) > at) continue;
|
|
278
302
|
const next = { ...upload, leaseUntil: new Date(at + Math.max(1_000, Number(leaseMs) || 30_000)).toISOString(), updatedAt: now() };
|
|
279
303
|
if (next.nextPart < content.chunks.length) {
|
|
@@ -333,6 +357,7 @@ function createEntityContent({ dir }) {
|
|
|
333
357
|
current.leaseUntil = null;
|
|
334
358
|
current.updatedAt = now();
|
|
335
359
|
saveUpload(current);
|
|
360
|
+
if (current.manifestUploaded) pendingUploads.get(resourceId)?.delete(contentHash);
|
|
336
361
|
return { duplicate: false, uploaded: current.manifestUploaded };
|
|
337
362
|
}
|
|
338
363
|
|
|
@@ -412,9 +437,11 @@ function createEntityContent({ dir }) {
|
|
|
412
437
|
if (!current) return { kind: 'manifest', partIndex: 0, sha256: null };
|
|
413
438
|
for (let partIndex = 0; partIndex < current.chunks.length; partIndex += 1) {
|
|
414
439
|
const chunk = current.chunks[partIndex];
|
|
440
|
+
// Presence + size only: every chunk file was hash-verified before it
|
|
441
|
+
// was written (capture or receive), writes are write-once, and final
|
|
442
|
+
// assembly (`bytes`) re-verifies the whole content head.
|
|
415
443
|
try {
|
|
416
|
-
|
|
417
|
-
if (bytes.length === chunk.bytes && digest(bytes) === chunk.sha256) continue;
|
|
444
|
+
if (fs.statSync(chunkFile(chunk.sha256)).size === chunk.bytes) continue;
|
|
418
445
|
} catch (error) {
|
|
419
446
|
if (error.code !== 'ENOENT') throw error;
|
|
420
447
|
}
|
|
@@ -596,6 +596,26 @@ function createRegistration(options) {
|
|
|
596
596
|
return normalized.map(({ targetPath, identityAt }) => add(targetPath, { identityAt }));
|
|
597
597
|
}
|
|
598
598
|
|
|
599
|
+
/** Reconcile one already-watched tree after the platform has materialized
|
|
600
|
+
* a newly declared exact default inside it. This is deliberately not part
|
|
601
|
+
* of ordinary re-registration: the caller must have already named the
|
|
602
|
+
* exact platform node it needs, and the normal Detect → Record path still
|
|
603
|
+
* decides the resulting record. */
|
|
604
|
+
function reconcileDeclared(targetPath) {
|
|
605
|
+
let ground;
|
|
606
|
+
try {
|
|
607
|
+
ground = fs.lstatSync(targetPath).isSymbolicLink() ? path.resolve(targetPath) : fs.realpathSync(targetPath);
|
|
608
|
+
} catch {
|
|
609
|
+
throw new Error(`declared user-home root is unavailable: ${targetPath}`);
|
|
610
|
+
}
|
|
611
|
+
const covering = treeRootCovering(ground);
|
|
612
|
+
if (!covering) throw new Error(`declared user-home root was not registered: ${targetPath}`);
|
|
613
|
+
observer.scanNow(covering.rootId);
|
|
614
|
+
flushIfDirty();
|
|
615
|
+
requireLiveWatchers(covering.path);
|
|
616
|
+
return { rootPath: covering.path, record: rootRecordOf(covering.rootId) };
|
|
617
|
+
}
|
|
618
|
+
|
|
599
619
|
/** The boundary's whole convergence pass, one act: loss convicted
|
|
600
620
|
* first (loudly — a conviction is never a retry), edges followed,
|
|
601
621
|
* lenses reconciled, renderings projected. Startup, the explicit
|
|
@@ -626,6 +646,7 @@ function createRegistration(options) {
|
|
|
626
646
|
registry,
|
|
627
647
|
add,
|
|
628
648
|
addMany,
|
|
649
|
+
reconcileDeclared,
|
|
629
650
|
|
|
630
651
|
/**
|
|
631
652
|
* `amalgm register <existing-path> --as <address>` — the managed
|