@amalgm/shell 0.1.111 → 0.1.112
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +24 -1
- package/dist/cli.js.map +1 -1
- package/dist/files-trace.d.ts +41 -0
- package/dist/files-trace.js +138 -0
- package/dist/files-trace.js.map +1 -0
- package/dist/gateway.js +7 -1
- package/dist/gateway.js.map +1 -1
- package/dist/machine-host.d.ts +6 -0
- package/dist/machine-host.js +25 -11
- package/dist/machine-host.js.map +1 -1
- package/dist/runtime-command-host.js +7 -1
- package/dist/runtime-command-host.js.map +1 -1
- package/dist/user-ground-host.d.ts +34 -0
- package/dist/user-ground-host.js +292 -71
- package/dist/user-ground-host.js.map +1 -1
- package/package.json +1 -1
package/dist/user-ground-host.js
CHANGED
|
@@ -205,6 +205,11 @@ export class UserGroundHost {
|
|
|
205
205
|
stage(input) {
|
|
206
206
|
emitFilesStage(this.options.onFilesStage, input);
|
|
207
207
|
}
|
|
208
|
+
/** Whether a counter worth real time on the critical path may be computed. */
|
|
209
|
+
measuring() {
|
|
210
|
+
return this.options.filesTraceActive?.()
|
|
211
|
+
?? (this.options.onFilesStage !== undefined || this.options.onDetectScan !== undefined);
|
|
212
|
+
}
|
|
208
213
|
reportDetectScan(evidence) {
|
|
209
214
|
this.options.onDetectScan?.(evidence);
|
|
210
215
|
this.stage({
|
|
@@ -222,7 +227,19 @@ export class UserGroundHost {
|
|
|
222
227
|
gitInspections: evidence.gitInspections,
|
|
223
228
|
repositoryCaptures: evidence.repositoryCaptures,
|
|
224
229
|
records: evidence.records ?? 0,
|
|
225
|
-
recordBytes: evidence.recordBytes
|
|
230
|
+
...(evidence.recordBytes === undefined ? {} : { recordBytes: evidence.recordBytes }),
|
|
231
|
+
// Where the scan actually went. `captureBusyMs / captureMs` is the
|
|
232
|
+
// capture pool's occupancy; at ~1 the pool ran one file at a time.
|
|
233
|
+
walkMs: Math.round(evidence.walkMs ?? 0),
|
|
234
|
+
gitMs: Math.round(evidence.gitMs ?? 0),
|
|
235
|
+
captureMs: Math.round(evidence.captureMs ?? 0),
|
|
236
|
+
captureBusyMs: Math.round(evidence.captureBusyMs ?? 0),
|
|
237
|
+
captureBytes: evidence.captureBytes ?? 0,
|
|
238
|
+
repoIdentityMs: Math.round(evidence.repositoryIdentityMs ?? 0),
|
|
239
|
+
repoCaptureMs: Math.round(evidence.repositoryCaptureMs ?? 0),
|
|
240
|
+
repoSealMs: Math.round(evidence.repositorySealMs ?? 0),
|
|
241
|
+
reconcileMs: Math.round(evidence.reconcileMs ?? 0),
|
|
242
|
+
persistMs: Math.round(evidence.persistMs ?? 0),
|
|
226
243
|
},
|
|
227
244
|
});
|
|
228
245
|
}
|
|
@@ -620,6 +637,7 @@ export class UserGroundHost {
|
|
|
620
637
|
database: this.databasePath(identity),
|
|
621
638
|
cacheDir: this.cacheDir(identity),
|
|
622
639
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
640
|
+
measuring: this.measuring(),
|
|
623
641
|
registrationWorkspaceId: workspaceId,
|
|
624
642
|
suspicions: new Map(observations.map((observation) => [observation.directory, observation.suspicion])),
|
|
625
643
|
});
|
|
@@ -1137,6 +1155,7 @@ export class UserGroundHost {
|
|
|
1137
1155
|
await reconcileGround({
|
|
1138
1156
|
identity, userRoot, database, cacheDir,
|
|
1139
1157
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
1158
|
+
measuring: this.measuring(),
|
|
1140
1159
|
});
|
|
1141
1160
|
return localValue(identity, userRoot, database);
|
|
1142
1161
|
},
|
|
@@ -1851,6 +1870,7 @@ export class UserGroundHost {
|
|
|
1851
1870
|
database: this.databasePath(identity),
|
|
1852
1871
|
cacheDir: this.cacheDir(identity),
|
|
1853
1872
|
onScan: (evidence) => this.reportDetectScan(evidence),
|
|
1873
|
+
measuring: this.measuring(),
|
|
1854
1874
|
suspicions: reconcileSuspicions,
|
|
1855
1875
|
coreScan,
|
|
1856
1876
|
});
|
|
@@ -1999,24 +2019,43 @@ export class UserGroundHost {
|
|
|
1999
2019
|
const state = this.cloudState;
|
|
2000
2020
|
if (!state)
|
|
2001
2021
|
throw new Error("cloud state is unavailable for graph publication");
|
|
2002
|
-
|
|
2003
|
-
|
|
2022
|
+
// Graph preparation is the leg between a committed local ground and the
|
|
2023
|
+
// first wire frame. It is pure CPU and SQLite, so it is measured by its
|
|
2024
|
+
// own parts: whoever reads this ledger must see which part is slow.
|
|
2025
|
+
const preparationStarted = performance.now();
|
|
2026
|
+
const leg = {};
|
|
2027
|
+
const timed = (name, work) => {
|
|
2028
|
+
const started = performance.now();
|
|
2029
|
+
const value = work();
|
|
2030
|
+
leg[`${name}Ms`] = performance.now() - started;
|
|
2031
|
+
return value;
|
|
2032
|
+
};
|
|
2033
|
+
if (trace?.journey === "register") {
|
|
2034
|
+
this.stage({
|
|
2035
|
+
primitive: "register",
|
|
2036
|
+
stage: "graph-preparation",
|
|
2037
|
+
status: "started",
|
|
2038
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2039
|
+
});
|
|
2040
|
+
}
|
|
2041
|
+
const officialRecords = timed("officialRows", () => recordsAtDeliveryPrefix(state.records, readAcceptedResults(this.databasePath(identity), state.resourceId, state.deliveryWatermark, deliveryWatermark)));
|
|
2042
|
+
const localRecords = timed("groundRows", () => recordsRetainingMissingEvidence(readGroundRowsForRoots(this.databasePath(identity), "entities", publicationRootIds)
|
|
2004
2043
|
.map(portableRecord), readGroundRowsForRoots(this.databasePath(identity), "detection_notebook", publicationRootIds)
|
|
2005
|
-
.map(portableRecord));
|
|
2006
|
-
const merged = mergeMaterializedRoots(officialRecords, localRecords);
|
|
2007
|
-
const traveling = travelingRecords(merged, sha256Hex);
|
|
2044
|
+
.map(portableRecord)));
|
|
2045
|
+
const merged = timed("merge", () => mergeMaterializedRoots(officialRecords, localRecords));
|
|
2046
|
+
const traveling = timed("project", () => travelingRecords(merged, sha256Hex));
|
|
2008
2047
|
const impossibleGitLeaf = traveling.find((record) => (record.type === "file.text" || record.type === "file.binary" || record.type === "link")
|
|
2009
2048
|
&& record.payloadVersion?.startsWith("git:"));
|
|
2010
2049
|
if (impossibleGitLeaf) {
|
|
2011
2050
|
const parent = merged.find((record) => record.uuid === impossibleGitLeaf.parentUUID);
|
|
2012
2051
|
throw new Error(`repository-owned leaf escaped its rail: ${impossibleGitLeaf.name} parent=${parent?.type ?? "missing"}`);
|
|
2013
2052
|
}
|
|
2014
|
-
const snapshot = snapshotFromRecords(traveling);
|
|
2053
|
+
const snapshot = timed("normalize", () => snapshotFromRecords(traveling));
|
|
2015
2054
|
// ONE canonical serialization: these exact bytes are the journal blob
|
|
2016
2055
|
// and the checksum's input; they are never re-derived or re-parsed on
|
|
2017
2056
|
// the in-process path.
|
|
2018
|
-
const snapshotBlob = Buffer.from(stableJson(snapshot), "utf8");
|
|
2019
|
-
const checksum = sha256Hex(snapshotBlob);
|
|
2057
|
+
const snapshotBlob = timed("serialize", () => Buffer.from(stableJson(snapshot), "utf8"));
|
|
2058
|
+
const checksum = timed("checksum", () => sha256Hex(snapshotBlob));
|
|
2020
2059
|
const localIds = new Set(localRecords.map((record) => record.uuid));
|
|
2021
2060
|
// Graph membership is never proof of sealed cargo: a record in the cloud
|
|
2022
2061
|
// graph may name content sealed only under a prior content contract,
|
|
@@ -2027,6 +2066,12 @@ export class UserGroundHost {
|
|
|
2027
2066
|
// before returning, even when the graph itself is unchanged.
|
|
2028
2067
|
const cargoRecords = snapshot.records.filter((record) => localIds.has(record.uuid) && artifactForRecord(record) !== null);
|
|
2029
2068
|
if (checksum === state.checksum && deliveryWatermark === state.deliveryWatermark) {
|
|
2069
|
+
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2070
|
+
records: snapshot.records.length,
|
|
2071
|
+
snapshotBytes: snapshotBlob.byteLength,
|
|
2072
|
+
cargoRecords: cargoRecords.length,
|
|
2073
|
+
graphUnchanged: true,
|
|
2074
|
+
});
|
|
2030
2075
|
// The graph is already committed; only cargo can be missing. Repair
|
|
2031
2076
|
// uploads need no journal row — a crash simply re-runs the same
|
|
2032
2077
|
// repair on the next Register.
|
|
@@ -2040,18 +2085,19 @@ export class UserGroundHost {
|
|
|
2040
2085
|
}
|
|
2041
2086
|
return;
|
|
2042
2087
|
}
|
|
2043
|
-
const
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2088
|
+
const replacement = timed("replacement", () => {
|
|
2089
|
+
const rootIds = localRecords
|
|
2090
|
+
.filter((record) => record.parentUUID === null)
|
|
2091
|
+
.map((record) => record.uuid);
|
|
2092
|
+
const roots = new Set(rootIds);
|
|
2093
|
+
const rootOf = recordRootResolver(snapshot.records);
|
|
2094
|
+
return rootReplacementFromRecords(rootIds, snapshot.records.filter((record) => {
|
|
2095
|
+
const root = rootOf(record);
|
|
2096
|
+
return root !== null && roots.has(root);
|
|
2097
|
+
}));
|
|
2051
2098
|
});
|
|
2052
|
-
const replacement = rootReplacementFromRecords(replacementRootIds, replacementRecords);
|
|
2053
2099
|
// The operation's canonical bytes: staged, hashed, and submitted as-is.
|
|
2054
|
-
const operationBlob = Buffer.from(stableJson(replacement), "utf8");
|
|
2100
|
+
const operationBlob = timed("operation", () => Buffer.from(stableJson(replacement), "utf8"));
|
|
2055
2101
|
const entry = {
|
|
2056
2102
|
mutationId: randomUUID(),
|
|
2057
2103
|
resourceId: state.resourceId,
|
|
@@ -2067,7 +2113,15 @@ export class UserGroundHost {
|
|
|
2067
2113
|
status: "pending",
|
|
2068
2114
|
submitted: false,
|
|
2069
2115
|
};
|
|
2070
|
-
supersedeSnapshotJournal(this.databasePath(identity), entry);
|
|
2116
|
+
timed("journal", () => supersedeSnapshotJournal(this.databasePath(identity), entry));
|
|
2117
|
+
this.finishPreparation(trace, preparationStarted, leg, {
|
|
2118
|
+
records: snapshot.records.length,
|
|
2119
|
+
snapshotBytes: snapshotBlob.byteLength,
|
|
2120
|
+
operationBytes: operationBlob.byteLength,
|
|
2121
|
+
operationRoots: replacement.rootIds.length,
|
|
2122
|
+
cargoRecords: cargoRecords.length,
|
|
2123
|
+
graphUnchanged: false,
|
|
2124
|
+
});
|
|
2071
2125
|
// Publish the freshly journaled intent directly: the records are already
|
|
2072
2126
|
// in memory and the operation bytes already exist — nothing is re-read,
|
|
2073
2127
|
// re-parsed, or re-serialized between prepare and the wire.
|
|
@@ -2077,6 +2131,23 @@ export class UserGroundHost {
|
|
|
2077
2131
|
uploadRecords: cargoRecords,
|
|
2078
2132
|
}, trace, manifestHints);
|
|
2079
2133
|
}
|
|
2134
|
+
/** Close the graph-preparation stage with every leg it spent, so a slow
|
|
2135
|
+
* preparation names the slow part instead of one opaque number. */
|
|
2136
|
+
finishPreparation(trace, startedAt, legs, counters) {
|
|
2137
|
+
if (trace?.journey !== "register")
|
|
2138
|
+
return;
|
|
2139
|
+
this.stage({
|
|
2140
|
+
primitive: "register",
|
|
2141
|
+
stage: "graph-preparation",
|
|
2142
|
+
status: "completed",
|
|
2143
|
+
durationMs: performance.now() - startedAt,
|
|
2144
|
+
...(trace.workspaceId ? { workspaceId: trace.workspaceId } : {}),
|
|
2145
|
+
measurements: {
|
|
2146
|
+
...counters,
|
|
2147
|
+
...Object.fromEntries(Object.entries(legs).map(([name, value]) => [name, Math.round(value)])),
|
|
2148
|
+
},
|
|
2149
|
+
});
|
|
2150
|
+
}
|
|
2080
2151
|
/** Which of these heads does the authority hold sealed under the current
|
|
2081
2152
|
* content contract? Batched so a whole workspace costs a handful of
|
|
2082
2153
|
* frames; the answer is the only permission to skip an upload. */
|
|
@@ -2136,10 +2207,16 @@ export class UserGroundHost {
|
|
|
2136
2207
|
...(trace.mutationId ? { mutationId: trace.mutationId } : {}),
|
|
2137
2208
|
});
|
|
2138
2209
|
}
|
|
2210
|
+
const legMs = {};
|
|
2211
|
+
const local = (name, work) => {
|
|
2212
|
+
const at = performance.now();
|
|
2213
|
+
const value = work();
|
|
2214
|
+
legMs[`${name}Ms`] = performance.now() - at;
|
|
2215
|
+
return value;
|
|
2216
|
+
};
|
|
2139
2217
|
if (!publication.submitted) {
|
|
2140
|
-
markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId);
|
|
2218
|
+
local("markSubmitted", () => markSnapshotJournalSubmitted(this.databasePath(identity), publication.mutationId));
|
|
2141
2219
|
}
|
|
2142
|
-
const legMs = {};
|
|
2143
2220
|
try {
|
|
2144
2221
|
frame = await submitMutationOperation(this.wire, {
|
|
2145
2222
|
resourceId: publication.resourceId,
|
|
@@ -2172,7 +2249,27 @@ export class UserGroundHost {
|
|
|
2172
2249
|
}
|
|
2173
2250
|
throw pipelineStageError("cloud graph commit", error);
|
|
2174
2251
|
}
|
|
2252
|
+
this.cloudState = {
|
|
2253
|
+
resourceId: publication.resourceId,
|
|
2254
|
+
authorityEpoch: Number(frame.authority_epoch),
|
|
2255
|
+
headVersion: Number(frame.version),
|
|
2256
|
+
checksum: publication.snapshotChecksum,
|
|
2257
|
+
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2258
|
+
records: publication.snapshotRecords,
|
|
2259
|
+
};
|
|
2260
|
+
local("clearJournal", () => deleteJournalEntry(this.databasePath(identity), publication.mutationId));
|
|
2175
2261
|
if (trace?.journey === "register") {
|
|
2262
|
+
// The authority reports where its side of the commit went; carrying
|
|
2263
|
+
// its legs here gives one ledger line the whole story: local journal
|
|
2264
|
+
// marks, wire time, gateway assemble/validate, and the database RPC.
|
|
2265
|
+
for (const [wire, leg] of [
|
|
2266
|
+
["assemble_ms", "serverAssembleMs"],
|
|
2267
|
+
["validate_ms", "serverValidateMs"],
|
|
2268
|
+
["database_ms", "serverDatabaseMs"],
|
|
2269
|
+
]) {
|
|
2270
|
+
if (typeof frame[wire] === "number")
|
|
2271
|
+
legMs[leg] = frame[wire];
|
|
2272
|
+
}
|
|
2176
2273
|
this.stage({
|
|
2177
2274
|
primitive: "register",
|
|
2178
2275
|
stage: "cloud-graph-commit",
|
|
@@ -2182,19 +2279,11 @@ export class UserGroundHost {
|
|
|
2182
2279
|
measurements: {
|
|
2183
2280
|
version: Number(frame.version),
|
|
2184
2281
|
operationBytes: publication.operationBlob.byteLength,
|
|
2185
|
-
...legMs
|
|
2282
|
+
...Object.fromEntries(Object.entries(legMs)
|
|
2283
|
+
.map(([name, value]) => [name, Math.round(value)])),
|
|
2186
2284
|
},
|
|
2187
2285
|
});
|
|
2188
2286
|
}
|
|
2189
|
-
this.cloudState = {
|
|
2190
|
-
resourceId: publication.resourceId,
|
|
2191
|
-
authorityEpoch: Number(frame.authority_epoch),
|
|
2192
|
-
headVersion: Number(frame.version),
|
|
2193
|
-
checksum: publication.snapshotChecksum,
|
|
2194
|
-
deliveryWatermark: Number(frame.delivery_watermark),
|
|
2195
|
-
records: publication.snapshotRecords,
|
|
2196
|
-
};
|
|
2197
|
-
deleteJournalEntry(this.databasePath(identity), publication.mutationId);
|
|
2198
2287
|
}
|
|
2199
2288
|
}
|
|
2200
2289
|
const sha256Hex = (input) => createHash("sha256").update(input).digest("hex");
|
|
@@ -2286,17 +2375,28 @@ function assertHealthyWatch(evidence, rootId) {
|
|
|
2286
2375
|
* file over a dozen times; the file itself is never deleted by the host, so
|
|
2287
2376
|
* every later open needs only the connection and its pragmas. */
|
|
2288
2377
|
const migratedDatabases = new Set();
|
|
2378
|
+
/** The journal's mutable facts and its immutable payload live in separate
|
|
2379
|
+
* tables on purpose: SQLite rewrites a row's entire record — overflow pages
|
|
2380
|
+
* included — on any column update, so flipping submitted_at on a row that
|
|
2381
|
+
* carried a register-scale snapshot cost a full multi-megabyte rewrite
|
|
2382
|
+
* through the WAL at synchronous=FULL (measured ~7.5s of the production
|
|
2383
|
+
* cloud-commit stage on the 1-GB machine). The blob row is written exactly
|
|
2384
|
+
* once and read only on restart replay; every status transition touches only
|
|
2385
|
+
* the small row. */
|
|
2289
2386
|
const SNAPSHOT_PUBLICATIONS_SCHEMA = `
|
|
2290
2387
|
CREATE TABLE IF NOT EXISTS snapshot_publications (
|
|
2291
2388
|
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
2292
2389
|
mutation_id TEXT NOT NULL UNIQUE,
|
|
2293
2390
|
resource_id TEXT NOT NULL,
|
|
2294
2391
|
meta_json TEXT NOT NULL,
|
|
2295
|
-
snapshot_blob BLOB NOT NULL,
|
|
2296
|
-
operation_blob BLOB NOT NULL,
|
|
2297
2392
|
created_at INTEGER NOT NULL,
|
|
2298
2393
|
status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending', 'rejected')),
|
|
2299
2394
|
submitted_at INTEGER
|
|
2395
|
+
);
|
|
2396
|
+
CREATE TABLE IF NOT EXISTS snapshot_publication_blobs (
|
|
2397
|
+
mutation_id TEXT PRIMARY KEY,
|
|
2398
|
+
snapshot_blob BLOB NOT NULL,
|
|
2399
|
+
operation_blob BLOB NOT NULL
|
|
2300
2400
|
);`;
|
|
2301
2401
|
/** Converts an old-shape publication journal (one nested publication_json
|
|
2302
2402
|
* text per row) into the blob journal, atomically, preserving every row's
|
|
@@ -2328,8 +2428,7 @@ function convertLegacySnapshotJournal(database, columns) {
|
|
|
2328
2428
|
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2329
2429
|
database.exec(`
|
|
2330
2430
|
INSERT INTO snapshot_publications(
|
|
2331
|
-
mutation_id, resource_id, meta_json,
|
|
2332
|
-
created_at, status, submitted_at
|
|
2431
|
+
mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2333
2432
|
)
|
|
2334
2433
|
SELECT
|
|
2335
2434
|
mutation_id,
|
|
@@ -2346,20 +2445,50 @@ function convertLegacySnapshotJournal(database, columns) {
|
|
|
2346
2445
|
FROM json_each(publication_json, '$.uploadRecords') AS uploaded
|
|
2347
2446
|
), json_array())
|
|
2348
2447
|
),
|
|
2448
|
+
created_at,
|
|
2449
|
+
${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
|
|
2450
|
+
${columns.has("submitted_at") ? "submitted_at" : "created_at"}
|
|
2451
|
+
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2452
|
+
`);
|
|
2453
|
+
database.exec(`
|
|
2454
|
+
INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
|
|
2455
|
+
SELECT
|
|
2456
|
+
mutation_id,
|
|
2349
2457
|
CAST(json_extract(publication_json, '$.snapshotJson') AS BLOB),
|
|
2350
2458
|
CAST(json_object(
|
|
2351
2459
|
'kind', 'entity.roots.replace',
|
|
2352
2460
|
'rootIds', json(json_extract(publication_json, '$.replacementRootIds')),
|
|
2353
2461
|
'records', json(json_extract(publication_json, '$.replacementRecords'))
|
|
2354
|
-
) AS BLOB)
|
|
2355
|
-
created_at,
|
|
2356
|
-
${columns.has("status") ? "CASE status WHEN 'rejected' THEN 'rejected' ELSE 'pending' END" : "'pending'"},
|
|
2357
|
-
${columns.has("submitted_at") ? "submitted_at" : "created_at"}
|
|
2462
|
+
) AS BLOB)
|
|
2358
2463
|
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2359
2464
|
`);
|
|
2360
2465
|
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2361
2466
|
})();
|
|
2362
2467
|
}
|
|
2468
|
+
/** Converts the single-table blob journal (blobs stored on the publication
|
|
2469
|
+
* row itself) into the split journal, atomically and entirely inside SQLite —
|
|
2470
|
+
* the same engine-memory law as the nested-text conversion above. Every
|
|
2471
|
+
* field already has its final meaning; the blobs simply move to their
|
|
2472
|
+
* write-once table so status transitions stop rewriting them. */
|
|
2473
|
+
function splitSnapshotJournalBlobs(database) {
|
|
2474
|
+
database.transaction(() => {
|
|
2475
|
+
database.exec("ALTER TABLE snapshot_publications RENAME TO snapshot_publications_legacy");
|
|
2476
|
+
database.exec(SNAPSHOT_PUBLICATIONS_SCHEMA);
|
|
2477
|
+
database.exec(`
|
|
2478
|
+
INSERT INTO snapshot_publications(
|
|
2479
|
+
mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2480
|
+
)
|
|
2481
|
+
SELECT mutation_id, resource_id, meta_json, created_at, status, submitted_at
|
|
2482
|
+
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2483
|
+
`);
|
|
2484
|
+
database.exec(`
|
|
2485
|
+
INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
|
|
2486
|
+
SELECT mutation_id, snapshot_blob, operation_blob
|
|
2487
|
+
FROM snapshot_publications_legacy ORDER BY sequence
|
|
2488
|
+
`);
|
|
2489
|
+
database.exec("DROP TABLE snapshot_publications_legacy");
|
|
2490
|
+
})();
|
|
2491
|
+
}
|
|
2363
2492
|
/** The single opener and schema-migration entry for a user ground database.
|
|
2364
2493
|
* Exported so the upgrade path is provable in isolation — the journal
|
|
2365
2494
|
* conversion law test opens a legacy file through this exact entry inside a
|
|
@@ -2379,6 +2508,9 @@ export function initializeDatabase(file) {
|
|
|
2379
2508
|
if (publicationColumns.has("publication_json")) {
|
|
2380
2509
|
convertLegacySnapshotJournal(database, publicationColumns);
|
|
2381
2510
|
}
|
|
2511
|
+
else if (publicationColumns.has("snapshot_blob")) {
|
|
2512
|
+
splitSnapshotJournalBlobs(database);
|
|
2513
|
+
}
|
|
2382
2514
|
if (migratedDatabases.has(file))
|
|
2383
2515
|
return database;
|
|
2384
2516
|
database.exec(`
|
|
@@ -2955,12 +3087,23 @@ function readSnapshotPublications(file) {
|
|
|
2955
3087
|
const database = initializeDatabase(file);
|
|
2956
3088
|
try {
|
|
2957
3089
|
return database.prepare(`
|
|
2958
|
-
SELECT mutation_id AS mutationId,
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
3090
|
+
SELECT publications.mutation_id AS mutationId,
|
|
3091
|
+
publications.resource_id AS resourceId,
|
|
3092
|
+
publications.meta_json AS metaJson,
|
|
3093
|
+
blobs.snapshot_blob AS snapshotBlob,
|
|
3094
|
+
blobs.operation_blob AS operationBlob,
|
|
3095
|
+
publications.status,
|
|
3096
|
+
publications.submitted_at AS submittedAt
|
|
3097
|
+
FROM snapshot_publications AS publications
|
|
3098
|
+
LEFT JOIN snapshot_publication_blobs AS blobs USING (mutation_id)
|
|
3099
|
+
ORDER BY publications.sequence
|
|
2963
3100
|
`).all().map((row) => {
|
|
3101
|
+
// A publication without its payload is damage, never a skippable row —
|
|
3102
|
+
// silently dropping it would let a new publication supersede a
|
|
3103
|
+
// submitted mutation whose outcome exists only at the authority.
|
|
3104
|
+
if (!row.snapshotBlob || !row.operationBlob) {
|
|
3105
|
+
throw new Error(`snapshot publication ${row.mutationId} has no journaled payload`);
|
|
3106
|
+
}
|
|
2964
3107
|
const meta = JSON.parse(row.metaJson);
|
|
2965
3108
|
// No lawful publication carries zero roots — rootReplacementFromRecords
|
|
2966
3109
|
// refuses them at build time, so a rootless row here is damage.
|
|
@@ -3150,12 +3293,17 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3150
3293
|
if (unresolved) {
|
|
3151
3294
|
throw new Error("a submitted publication with an unknown outcome must resolve before supersession");
|
|
3152
3295
|
}
|
|
3296
|
+
database.prepare(`
|
|
3297
|
+
DELETE FROM snapshot_publication_blobs WHERE mutation_id IN (
|
|
3298
|
+
SELECT mutation_id FROM snapshot_publications WHERE resource_id = ?
|
|
3299
|
+
)
|
|
3300
|
+
`).run(entry.resourceId);
|
|
3153
3301
|
database.prepare("DELETE FROM snapshot_publications WHERE resource_id = ?")
|
|
3154
3302
|
.run(entry.resourceId);
|
|
3155
3303
|
database.prepare(`
|
|
3156
3304
|
INSERT INTO snapshot_publications(
|
|
3157
|
-
mutation_id, resource_id, meta_json,
|
|
3158
|
-
) VALUES (?, ?, ?,
|
|
3305
|
+
mutation_id, resource_id, meta_json, created_at
|
|
3306
|
+
) VALUES (?, ?, ?, ?)
|
|
3159
3307
|
`).run(entry.mutationId, entry.resourceId, stableJson({
|
|
3160
3308
|
authorityEpoch: entry.authorityEpoch,
|
|
3161
3309
|
baseVersion: entry.baseVersion,
|
|
@@ -3164,7 +3312,11 @@ function supersedeSnapshotJournal(file, entry) {
|
|
|
3164
3312
|
operationKind: entry.operationKind,
|
|
3165
3313
|
replacementRootIds: entry.replacementRootIds,
|
|
3166
3314
|
uploadUuids: entry.uploadUuids,
|
|
3167
|
-
}),
|
|
3315
|
+
}), Date.now());
|
|
3316
|
+
database.prepare(`
|
|
3317
|
+
INSERT INTO snapshot_publication_blobs(mutation_id, snapshot_blob, operation_blob)
|
|
3318
|
+
VALUES (?, ?, ?)
|
|
3319
|
+
`).run(entry.mutationId, entry.snapshotBlob, entry.operationBlob);
|
|
3168
3320
|
})();
|
|
3169
3321
|
}
|
|
3170
3322
|
finally {
|
|
@@ -3234,7 +3386,10 @@ function captureChurn(error) {
|
|
|
3234
3386
|
function deleteJournalEntry(file, mutationId) {
|
|
3235
3387
|
const database = initializeDatabase(file);
|
|
3236
3388
|
try {
|
|
3237
|
-
database.
|
|
3389
|
+
database.transaction(() => {
|
|
3390
|
+
database.prepare("DELETE FROM snapshot_publication_blobs WHERE mutation_id = ?").run(mutationId);
|
|
3391
|
+
database.prepare("DELETE FROM snapshot_publications WHERE mutation_id = ?").run(mutationId);
|
|
3392
|
+
})();
|
|
3238
3393
|
}
|
|
3239
3394
|
finally {
|
|
3240
3395
|
database.close();
|
|
@@ -3614,7 +3769,7 @@ function materializedWorkspaceBindings(bindingDir) {
|
|
|
3614
3769
|
return bindings.sort((left, right) => left.workspaceId.localeCompare(right.workspaceId));
|
|
3615
3770
|
}
|
|
3616
3771
|
async function reconcileGround(input) {
|
|
3617
|
-
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, } = input;
|
|
3772
|
+
const { identity, userRoot, database, cacheDir, suspicions, coreScan, registrationWorkspaceId, onScan, measuring = false, } = input;
|
|
3618
3773
|
const resourceId = privateEntityResourceId(identity.userId, sha256Hex);
|
|
3619
3774
|
const bindingDir = workspaceBindingDir(userRoot, identity.deviceId);
|
|
3620
3775
|
const boundRootEntries = materializedWorkspaceBindings(bindingDir);
|
|
@@ -3709,9 +3864,24 @@ async function reconcileGround(input) {
|
|
|
3709
3864
|
contentReads: 0,
|
|
3710
3865
|
gitInspections: 0,
|
|
3711
3866
|
repositoryCaptures: 0,
|
|
3867
|
+
walkMs: 0,
|
|
3868
|
+
captureMs: 0,
|
|
3869
|
+
captureBusyMs: 0,
|
|
3870
|
+
captureBytes: 0,
|
|
3871
|
+
gitMs: 0,
|
|
3872
|
+
repositoryIdentityMs: 0,
|
|
3873
|
+
repositoryCaptureMs: 0,
|
|
3874
|
+
repositorySealMs: 0,
|
|
3875
|
+
reconcileMs: 0,
|
|
3876
|
+
persistMs: 0,
|
|
3712
3877
|
};
|
|
3713
3878
|
const started = performance.now();
|
|
3714
|
-
const result = await reconcileRoot({
|
|
3879
|
+
const result = await reconcileRoot({
|
|
3880
|
+
...parameters,
|
|
3881
|
+
evidence: counters,
|
|
3882
|
+
measureEvidence: measuring,
|
|
3883
|
+
materializedUUIDs,
|
|
3884
|
+
});
|
|
3715
3885
|
if (result.transferFailure !== null) {
|
|
3716
3886
|
transferFailures.push({ rootId: parameters.rootUUID, reason: result.transferFailure });
|
|
3717
3887
|
}
|
|
@@ -3749,7 +3919,14 @@ async function reconcileGround(input) {
|
|
|
3749
3919
|
durationMs: performance.now() - started,
|
|
3750
3920
|
plannedRows: proposals.length,
|
|
3751
3921
|
records: proposals.length,
|
|
3752
|
-
|
|
3922
|
+
// Encoding every proposal to learn its wire size costs one record
|
|
3923
|
+
// construction per entity — real time on the critical path of a cold
|
|
3924
|
+
// Register. It is evidence, so it is paid only when read.
|
|
3925
|
+
...(measuring
|
|
3926
|
+
? {
|
|
3927
|
+
recordBytes: proposals.reduce((bytes, proposal) => bytes + encodedRecordBytes(parameters.resourceId, proposal), 0),
|
|
3928
|
+
}
|
|
3929
|
+
: {}),
|
|
3753
3930
|
});
|
|
3754
3931
|
// Each root commits its local projection before Detect yields. Native
|
|
3755
3932
|
// callbacks can therefore preserve new suspicion between large roots.
|
|
@@ -3848,7 +4025,7 @@ async function reconcileGround(input) {
|
|
|
3848
4025
|
};
|
|
3849
4026
|
}
|
|
3850
4027
|
async function reconcileRoot(input) {
|
|
3851
|
-
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, plansSendRecords, largeManifests, } = input;
|
|
4028
|
+
const { identity, resourceId, rootUUID, rootName, rootPath, rootType, database, cacheDir, policy, bindingDir, suspects = null, evidence, measureEvidence = false, plansSendRecords, largeManifests, } = input;
|
|
3852
4029
|
// Prior rows reach this scan from every binding this root encloses. A row's
|
|
3853
4030
|
// stored relativePath is relative to the root that last committed it, which
|
|
3854
4031
|
// is a different frame when an enclosing boundary was just registered
|
|
@@ -4021,6 +4198,33 @@ async function reconcileRoot(input) {
|
|
|
4021
4198
|
// its own entry in place; every task is awaited before entries are read.
|
|
4022
4199
|
const captureTasks = [];
|
|
4023
4200
|
const captureGate = concurrencyGate(WALK_CAPTURE_CONCURRENCY);
|
|
4201
|
+
// The scan's three costs are metadata traversal, content capture, and Git
|
|
4202
|
+
// inspection. They interleave, so each is accumulated where it is spent
|
|
4203
|
+
// rather than inferred from the total afterwards.
|
|
4204
|
+
let captureOpenedAt = 0;
|
|
4205
|
+
let captureClosedAt = 0;
|
|
4206
|
+
const spent = (work, into) => {
|
|
4207
|
+
if (!measureEvidence || !evidence)
|
|
4208
|
+
return work();
|
|
4209
|
+
const at = performance.now();
|
|
4210
|
+
try {
|
|
4211
|
+
return work();
|
|
4212
|
+
}
|
|
4213
|
+
finally {
|
|
4214
|
+
evidence[into] += performance.now() - at;
|
|
4215
|
+
}
|
|
4216
|
+
};
|
|
4217
|
+
const spentAsync = async (work, into) => {
|
|
4218
|
+
if (!measureEvidence || !evidence)
|
|
4219
|
+
return work();
|
|
4220
|
+
const at = performance.now();
|
|
4221
|
+
try {
|
|
4222
|
+
return await work();
|
|
4223
|
+
}
|
|
4224
|
+
finally {
|
|
4225
|
+
evidence[into] += performance.now() - at;
|
|
4226
|
+
}
|
|
4227
|
+
};
|
|
4024
4228
|
const visit = async (directory, parentUUID, parentType, base = "", activeRepository = parentType === "repo.git"
|
|
4025
4229
|
? {
|
|
4026
4230
|
root: directory,
|
|
@@ -4028,8 +4232,8 @@ async function reconcileRoot(input) {
|
|
|
4028
4232
|
evidence: directory === rootPath ? rootRepositoryEvidence : null,
|
|
4029
4233
|
}
|
|
4030
4234
|
: null, forceObserve = rootRailChanged) => {
|
|
4031
|
-
const children = readdirSync(directory, { withFileTypes: true })
|
|
4032
|
-
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
|
|
4235
|
+
const children = spent(() => readdirSync(directory, { withFileTypes: true })
|
|
4236
|
+
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0), "walkMs");
|
|
4033
4237
|
for (const child of children) {
|
|
4034
4238
|
// Every materialization boundary reserves its root .amalgm/ directory
|
|
4035
4239
|
// for machine intents, staging, and recovery. This is structural host
|
|
@@ -4045,7 +4249,7 @@ async function reconcileRoot(input) {
|
|
|
4045
4249
|
if (!policy(relativePath))
|
|
4046
4250
|
continue;
|
|
4047
4251
|
const absolutePath = join(directory, child.name);
|
|
4048
|
-
const stats = lstatSync(absolutePath);
|
|
4252
|
+
const stats = spent(() => lstatSync(absolutePath), "walkMs");
|
|
4049
4253
|
evidence && (evidence.entries += 1);
|
|
4050
4254
|
// The walk is synchronous metadata work. Yield periodically so the
|
|
4051
4255
|
// wire, Receive, and Apply keep running while a large root is proved.
|
|
@@ -4076,7 +4280,7 @@ async function reconcileRoot(input) {
|
|
|
4076
4280
|
if (observe && repositoryPath && activeRepository && !activeRepository.evidence) {
|
|
4077
4281
|
evidence && (evidence.gitInspections += 1);
|
|
4078
4282
|
try {
|
|
4079
|
-
activeRepository.evidence = inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths);
|
|
4283
|
+
activeRepository.evidence = spent(() => inspectGitRegistration(activeRepository.root, activeRepository.evidencePaths), "gitMs");
|
|
4080
4284
|
}
|
|
4081
4285
|
catch {
|
|
4082
4286
|
// Classification already settled from the marker. If Git cannot
|
|
@@ -4169,11 +4373,19 @@ async function reconcileRoot(input) {
|
|
|
4169
4373
|
});
|
|
4170
4374
|
if (deferredCapture) {
|
|
4171
4375
|
const entry = entries[entries.length - 1];
|
|
4376
|
+
if (measureEvidence && evidence && captureOpenedAt === 0)
|
|
4377
|
+
captureOpenedAt = performance.now();
|
|
4172
4378
|
captureTasks.push(captureGate(async () => {
|
|
4379
|
+
const taskStarted = measureEvidence ? performance.now() : 0;
|
|
4173
4380
|
let current = stats;
|
|
4174
4381
|
for (let attempt = 1;; attempt++) {
|
|
4175
4382
|
try {
|
|
4176
4383
|
const captured = await captureContentFile(absolutePath, current, activeRepository || !plansSendRecords ? null : cacheDir);
|
|
4384
|
+
if (measureEvidence && evidence) {
|
|
4385
|
+
evidence.captureBusyMs += performance.now() - taskStarted;
|
|
4386
|
+
evidence.captureBytes += captured.bytes;
|
|
4387
|
+
captureClosedAt = performance.now();
|
|
4388
|
+
}
|
|
4177
4389
|
entry.type = classifyFile({ binary: captured.binary });
|
|
4178
4390
|
entry.payloadVersion = captured.contentHash;
|
|
4179
4391
|
entry.contentVerifiedAtMs = Date.now();
|
|
@@ -4219,6 +4431,10 @@ async function reconcileRoot(input) {
|
|
|
4219
4431
|
};
|
|
4220
4432
|
await visit(rootPath, rootUUID, rootType);
|
|
4221
4433
|
await Promise.all(captureTasks);
|
|
4434
|
+
if (measureEvidence && evidence && captureOpenedAt !== 0) {
|
|
4435
|
+
evidence.captureMs += captureClosedAt - captureOpenedAt;
|
|
4436
|
+
}
|
|
4437
|
+
const reconcileStarted = measureEvidence ? performance.now() : 0;
|
|
4222
4438
|
// When every entry sits at its stored address with its stored type and
|
|
4223
4439
|
// physical fingerprint (honoring any fixed UUID), path-first claiming makes
|
|
4224
4440
|
// the multi-pass reconciliation the identity map — resolve directly. Any
|
|
@@ -4252,6 +4468,8 @@ async function reconcileRoot(input) {
|
|
|
4252
4468
|
inode: entry.inode,
|
|
4253
4469
|
...(entry.fixedUUID ? { fixedUUID: entry.fixedUUID } : {}),
|
|
4254
4470
|
})));
|
|
4471
|
+
if (measureEvidence && evidence)
|
|
4472
|
+
evidence.reconcileMs += performance.now() - reconcileStarted;
|
|
4255
4473
|
const resolvedByProvisional = new Map();
|
|
4256
4474
|
for (const entry of entries) {
|
|
4257
4475
|
const resolved = resolvedIdentity.get(entry.relativePath) ?? entry.uuid;
|
|
@@ -4318,13 +4536,13 @@ async function reconcileRoot(input) {
|
|
|
4318
4536
|
contentVerifiedAtMs: entry.contentVerifiedAtMs,
|
|
4319
4537
|
};
|
|
4320
4538
|
});
|
|
4321
|
-
const persistCurrentRows = () => {
|
|
4539
|
+
const persistCurrentRows = () => spent(() => {
|
|
4322
4540
|
const rows = rowsFromEntries();
|
|
4323
4541
|
if (input.suspicion === undefined) {
|
|
4324
4542
|
persistRows(database, identity, resourceId, rootUUID, rows, existingRows);
|
|
4325
4543
|
}
|
|
4326
4544
|
return rows;
|
|
4327
|
-
};
|
|
4545
|
+
}, "persistMs");
|
|
4328
4546
|
// Seal nested repositories first so their independent rows are complete
|
|
4329
4547
|
// before the enclosing graph is persisted. Their state is not an input to
|
|
4330
4548
|
// the enclosing identity map; only their address boundary is.
|
|
@@ -4452,7 +4670,7 @@ async function reconcileRoot(input) {
|
|
|
4452
4670
|
}
|
|
4453
4671
|
}
|
|
4454
4672
|
const fingerprintBefore = gitEvidenceFingerprint(repository.absolutePath);
|
|
4455
|
-
let repoIdentity = await refreshRepositoryIdentity
|
|
4673
|
+
let repoIdentity = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
|
|
4456
4674
|
/* The rows and the transport identity are derived from the same fresh
|
|
4457
4675
|
* worktree evidence. A later Watch ring for one of these dirty paths is
|
|
4458
4676
|
* therefore ordinary equality, not a transport-only echo. */
|
|
@@ -4490,8 +4708,8 @@ async function reconcileRoot(input) {
|
|
|
4490
4708
|
let captured = null;
|
|
4491
4709
|
let coherent = false;
|
|
4492
4710
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
4493
|
-
captured = captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity);
|
|
4494
|
-
const after = await refreshRepositoryIdentity
|
|
4711
|
+
captured = spent(() => captureRepository(repository.absolutePath, repoIdentity, prior, previousIdentity), "repositoryCaptureMs");
|
|
4712
|
+
const after = await spentAsync(refreshRepositoryIdentity, "repositoryIdentityMs");
|
|
4495
4713
|
if (sameGitIdentity(after, repoIdentity)) {
|
|
4496
4714
|
repoIdentity = [...captured.identity];
|
|
4497
4715
|
coherent = true;
|
|
@@ -4513,15 +4731,18 @@ async function reconcileRoot(input) {
|
|
|
4513
4731
|
}
|
|
4514
4732
|
repository.payloadVersion = captured.stateId;
|
|
4515
4733
|
repository.transportVersion = captured.transportVersion;
|
|
4516
|
-
|
|
4517
|
-
|
|
4518
|
-
|
|
4519
|
-
|
|
4520
|
-
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4524
|
-
|
|
4734
|
+
const settled = captured;
|
|
4735
|
+
const currentLayout = spent(() => {
|
|
4736
|
+
if (settled.bytes) {
|
|
4737
|
+
sealCapturedArtifact(cacheDir, {
|
|
4738
|
+
entityId: repository.uuid,
|
|
4739
|
+
entityType: "repo.git",
|
|
4740
|
+
kind: "git",
|
|
4741
|
+
contentHash: settled.transportVersion,
|
|
4742
|
+
}, settled.bytes);
|
|
4743
|
+
}
|
|
4744
|
+
return inspectRepositoryTransportFile(join(cacheDir, `${settled.transportVersion}.bin`));
|
|
4745
|
+
}, "repositorySealMs");
|
|
4525
4746
|
replayByUuid.set(repository.uuid, {
|
|
4526
4747
|
kind: "repo.git",
|
|
4527
4748
|
stateId: currentLayout.stateId,
|