@aiwg/cli 2026.8.26 → 2026.8.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/artifacts/backends/graphology-backend.js +11 -2
- package/dist/src/artifacts/backends/json-backend.js +12 -2
- package/dist/src/artifacts/backends/sqlite-backend.js +20 -7
- package/dist/src/artifacts/graph-backend.js +16 -0
- package/dist/src/storage/backends/postgres.js +6 -1
- package/dist/src/storage/index.js +1 -1
- package/dist/src/storage/migration-protocol.js +228 -50
- package/dist/src/storage/qualification.js +39 -5
- package/package.json +1 -1
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* @source @src/artifacts/graph-backend.ts
|
|
12
12
|
* @tests @test/unit/artifacts/graphology-backend.test.ts
|
|
13
13
|
*/
|
|
14
|
+
import { compareGraphIds, pageGraphIds } from '../graph-backend.js';
|
|
14
15
|
import { normalizeEdges } from '../types.js';
|
|
15
16
|
import { loadFeaturePackage } from '../../features/runtime.js';
|
|
16
17
|
/**
|
|
@@ -73,7 +74,15 @@ export class GraphologyBackend {
|
|
|
73
74
|
return this.graph.getNodeAttributes(id);
|
|
74
75
|
}
|
|
75
76
|
nodes() {
|
|
76
|
-
return this.graph.nodes();
|
|
77
|
+
return this.graph.nodes().sort(compareGraphIds);
|
|
78
|
+
}
|
|
79
|
+
queryNodes(filters) {
|
|
80
|
+
return this.graph.nodes()
|
|
81
|
+
.filter((id) => Object.entries(filters).every(([key, value]) => this.graph.getNodeAttribute(id, key) === value))
|
|
82
|
+
.sort(compareGraphIds);
|
|
83
|
+
}
|
|
84
|
+
pageNodes(limit, after) {
|
|
85
|
+
return pageGraphIds(this.graph.nodes(), limit, after);
|
|
77
86
|
}
|
|
78
87
|
// --- Traversal ---
|
|
79
88
|
neighbors(nodeId, direction, edgeType) {
|
|
@@ -98,7 +107,7 @@ export class GraphologyBackend {
|
|
|
98
107
|
// Add the "other" node
|
|
99
108
|
results.add(src === nodeId ? tgt : src);
|
|
100
109
|
}
|
|
101
|
-
return [...results];
|
|
110
|
+
return [...results].sort(compareGraphIds);
|
|
102
111
|
}
|
|
103
112
|
// --- Set operations ---
|
|
104
113
|
intersection(setA, setB) {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* @source @src/artifacts/graph-backend.ts
|
|
10
10
|
* @tests @test/unit/artifacts/graph-backend.test.ts
|
|
11
11
|
*/
|
|
12
|
+
import { compareGraphIds, pageGraphIds } from '../graph-backend.js';
|
|
12
13
|
import { normalizeEdges } from '../types.js';
|
|
13
14
|
/**
|
|
14
15
|
* JSON-backed graph using plain objects and JS Set operations.
|
|
@@ -52,7 +53,16 @@ export class JsonGraphBackend {
|
|
|
52
53
|
return this.graph.get(id)?.attrs;
|
|
53
54
|
}
|
|
54
55
|
nodes() {
|
|
55
|
-
return [...this.graph.keys()];
|
|
56
|
+
return [...this.graph.keys()].sort(compareGraphIds);
|
|
57
|
+
}
|
|
58
|
+
queryNodes(filters) {
|
|
59
|
+
return [...this.graph]
|
|
60
|
+
.filter(([, node]) => Object.entries(filters).every(([key, value]) => node.attrs[key] === value))
|
|
61
|
+
.map(([id]) => id)
|
|
62
|
+
.sort(compareGraphIds);
|
|
63
|
+
}
|
|
64
|
+
pageNodes(limit, after) {
|
|
65
|
+
return pageGraphIds([...this.graph.keys()], limit, after);
|
|
56
66
|
}
|
|
57
67
|
// --- Traversal ---
|
|
58
68
|
neighbors(nodeId, direction, edgeType) {
|
|
@@ -74,7 +84,7 @@ export class JsonGraphBackend {
|
|
|
74
84
|
}
|
|
75
85
|
}
|
|
76
86
|
}
|
|
77
|
-
return [...results];
|
|
87
|
+
return [...results].sort(compareGraphIds);
|
|
78
88
|
}
|
|
79
89
|
// --- Set operations ---
|
|
80
90
|
intersection(setA, setB) {
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* @source @src/artifacts/graph-backend.ts
|
|
14
14
|
* @tests @test/unit/artifacts/sqlite-backend.test.ts
|
|
15
15
|
*/
|
|
16
|
+
import { compareGraphIds, pageGraphIds } from '../graph-backend.js';
|
|
16
17
|
import { normalizeEdges } from '../types.js';
|
|
17
18
|
import { requireFeaturePackage } from '../../features/runtime.js';
|
|
18
19
|
const SCHEMA_VERSION = 1;
|
|
@@ -131,23 +132,35 @@ export class SqliteGraphBackend {
|
|
|
131
132
|
return JSON.parse(row.attrs);
|
|
132
133
|
}
|
|
133
134
|
nodes() {
|
|
134
|
-
return this.db.prepare('SELECT id FROM nodes').all()
|
|
135
|
+
return this.db.prepare('SELECT id FROM nodes ORDER BY id COLLATE BINARY').all()
|
|
136
|
+
.map((r) => r.id);
|
|
135
137
|
}
|
|
136
138
|
queryNodes(filters) {
|
|
137
139
|
const clauses = [];
|
|
138
140
|
const values = [];
|
|
139
141
|
if (filters.type !== undefined) {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
+
if (filters.type === null)
|
|
143
|
+
clauses.push('type IS NULL');
|
|
144
|
+
else {
|
|
145
|
+
clauses.push('type = ?');
|
|
146
|
+
values.push(filters.type);
|
|
147
|
+
}
|
|
142
148
|
}
|
|
143
149
|
if (filters.phase !== undefined) {
|
|
144
|
-
|
|
145
|
-
|
|
150
|
+
if (filters.phase === null)
|
|
151
|
+
clauses.push('phase IS NULL');
|
|
152
|
+
else {
|
|
153
|
+
clauses.push('phase = ?');
|
|
154
|
+
values.push(filters.phase);
|
|
155
|
+
}
|
|
146
156
|
}
|
|
147
157
|
const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : '';
|
|
148
|
-
return this.db.prepare(`SELECT id FROM nodes ${where} ORDER BY id`).all(...values)
|
|
158
|
+
return this.db.prepare(`SELECT id FROM nodes ${where} ORDER BY id COLLATE BINARY`).all(...values)
|
|
149
159
|
.map((row) => row.id);
|
|
150
160
|
}
|
|
161
|
+
pageNodes(limit, after) {
|
|
162
|
+
return pageGraphIds(this.nodes(), limit, after);
|
|
163
|
+
}
|
|
151
164
|
// --- Traversal ---
|
|
152
165
|
neighbors(nodeId, direction, edgeType) {
|
|
153
166
|
const results = new Set();
|
|
@@ -171,7 +184,7 @@ export class SqliteGraphBackend {
|
|
|
171
184
|
for (const row of rows)
|
|
172
185
|
results.add(row.target);
|
|
173
186
|
}
|
|
174
|
-
return [...results];
|
|
187
|
+
return [...results].sort(compareGraphIds);
|
|
175
188
|
}
|
|
176
189
|
// --- Set operations (native SQL) ---
|
|
177
190
|
intersection(setA, setB) {
|
|
@@ -10,6 +10,22 @@
|
|
|
10
10
|
* @source @src/artifacts/types.ts
|
|
11
11
|
* @tests @test/unit/artifacts/graph-backend.test.ts
|
|
12
12
|
*/
|
|
13
|
+
/** SQLite BINARY collation and local backends share this UTF-8 byte order. */
|
|
14
|
+
export function compareGraphIds(left, right) {
|
|
15
|
+
return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8'));
|
|
16
|
+
}
|
|
17
|
+
export function pageGraphIds(ids, limit, after) {
|
|
18
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 10_000) {
|
|
19
|
+
throw new Error('graph page limit must be an integer from 1 through 10000');
|
|
20
|
+
}
|
|
21
|
+
const ordered = [...ids].sort(compareGraphIds);
|
|
22
|
+
const eligible = after === undefined ? ordered : ordered.filter(id => compareGraphIds(id, after) > 0);
|
|
23
|
+
const nodes = eligible.slice(0, limit);
|
|
24
|
+
return {
|
|
25
|
+
nodes,
|
|
26
|
+
...(eligible.length > limit && nodes.length ? { nextCursor: nodes[nodes.length - 1] } : {}),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
13
29
|
/**
|
|
14
30
|
* Create a graph backend instance.
|
|
15
31
|
*
|
|
@@ -387,7 +387,7 @@ async function createPool(options) {
|
|
|
387
387
|
const Pool = (pg.Pool ?? defaultExport?.Pool);
|
|
388
388
|
if (!Pool)
|
|
389
389
|
throw new PostgresBackendError('AIWG_POSTGRES_DRIVER_INVALID', 'optional pg package does not export Pool');
|
|
390
|
-
|
|
390
|
+
const pool = new Pool({
|
|
391
391
|
connectionString,
|
|
392
392
|
max: bounded(options.maxConnections, DEFAULT_POOL_MAX, 1, 100, 'maxConnections'),
|
|
393
393
|
connectionTimeoutMillis: bounded(options.connectionTimeoutMs, DEFAULT_TIMEOUT_MS, 1, 600_000, 'connectionTimeoutMs'),
|
|
@@ -395,6 +395,11 @@ async function createPool(options) {
|
|
|
395
395
|
application_name: options.applicationName ?? 'aiwg',
|
|
396
396
|
ssl: options.ssl === 'disable' ? false : { rejectUnauthorized: options.ssl === 'verify-full' },
|
|
397
397
|
});
|
|
398
|
+
// pg emits idle-client failures at pool scope. Registering a listener keeps
|
|
399
|
+
// server restarts from becoming uncaught process exceptions; failed clients
|
|
400
|
+
// are evicted by pg and the next backend operation reconnects normally.
|
|
401
|
+
pool.on?.('error', () => undefined);
|
|
402
|
+
return pool;
|
|
398
403
|
}
|
|
399
404
|
function isLoopbackPostgresUrl(connectionString) {
|
|
400
405
|
try {
|
|
@@ -22,7 +22,7 @@ export { ObsidianAdapter } from './backends/obsidian.js';
|
|
|
22
22
|
export { LogseqAdapter } from './backends/logseq.js';
|
|
23
23
|
export { FortemiAdapter } from './backends/fortemi.js';
|
|
24
24
|
export { STORAGE_BACKEND_CONTRACT, STORAGE_BACKEND_MATRIX, StorageCapabilityError, negotiateStorageCapabilities, } from './backend-contract.js';
|
|
25
|
-
export { STORAGE_MIGRATION_PROTOCOL, MigrationProtocolError, StorageMigrationCoordinator, approvalDigest, digestRecords, validateManifest, } from './migration-protocol.js';
|
|
25
|
+
export { STORAGE_MIGRATION_PROTOCOL, MigrationProtocolError, StorageMigrationCoordinator, approvalDigest, digestRecordChunks, digestRecords, validateManifest, } from './migration-protocol.js';
|
|
26
26
|
export { PostgresStorageBackend, PostgresBackendError, } from './backends/postgres.js';
|
|
27
27
|
export { POSTGRES_SCHEMA_VERSION, POSTGRES_SCHEMA_V1_SQL, inspectPostgresSchema, upgradePostgresSchemaV1, rollbackPostgresSchemaV1, postgresLeastPrivilegeSql, } from './backends/postgres-schema.js';
|
|
28
28
|
export { PostgrestStorageBackend, PostgrestBackendError, } from './backends/postgrest.js';
|
|
@@ -37,10 +37,28 @@ export class StorageMigrationCoordinator {
|
|
|
37
37
|
this.rollbackWindowMs = boundedInteger(options.rollbackWindowMs, 86_400_000, 1, 31_536_000_000, 'rollbackWindowMs');
|
|
38
38
|
this.now = options.now ?? (() => new Date());
|
|
39
39
|
this.random = options.random ?? Math.random;
|
|
40
|
+
if (options.mode === 'online' && !options.compareSourceRevisions) {
|
|
41
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_REVISION_ORDER_REQUIRED', 'online migration requires a source-revision comparator');
|
|
42
|
+
}
|
|
40
43
|
}
|
|
41
44
|
async preview() {
|
|
42
45
|
this.throwIfAborted();
|
|
43
|
-
const
|
|
46
|
+
const boundary = await this.options.safety.prepare(this.options.mode, this.source.identity, this.options.signal);
|
|
47
|
+
try {
|
|
48
|
+
assertBoundaryState(boundary, this.options.mode === 'offline' ? 'quiesced' : 'tracking', 'initial');
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
await this.options.safety.release(boundary, 'failed');
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
54
|
+
let snapshot;
|
|
55
|
+
try {
|
|
56
|
+
snapshot = await this.source.snapshot(this.options.signal);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
await this.options.safety.release(boundary, 'failed');
|
|
60
|
+
throw error;
|
|
61
|
+
}
|
|
44
62
|
const createdAt = this.now().toISOString();
|
|
45
63
|
const sourceDigest = digestRecords(snapshot.records);
|
|
46
64
|
const manifest = {
|
|
@@ -67,57 +85,69 @@ export class StorageMigrationCoordinator {
|
|
|
67
85
|
},
|
|
68
86
|
digests: { source: sourceDigest },
|
|
69
87
|
records: [],
|
|
88
|
+
boundaries: { initial: boundary },
|
|
70
89
|
};
|
|
71
|
-
|
|
90
|
+
try {
|
|
91
|
+
await this.store.save(manifest);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
await this.options.safety.release(boundary, 'failed');
|
|
95
|
+
throw error;
|
|
96
|
+
}
|
|
72
97
|
return manifest;
|
|
73
98
|
}
|
|
74
99
|
async apply(manifest) {
|
|
75
100
|
validateManifest(manifest, this.source.identity, this.destination.identity);
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
manifest.state = 'copying';
|
|
81
|
-
await this.persist(manifest);
|
|
82
|
-
await this.copyRecords(snapshot.records, manifest);
|
|
83
|
-
if (manifest.mode === 'online') {
|
|
84
|
-
if (!this.source.changes) {
|
|
85
|
-
throw new MigrationProtocolError('AIWG_MIGRATION_CURSOR_UNAVAILABLE', 'online migration requires a source change cursor');
|
|
101
|
+
try {
|
|
102
|
+
const snapshot = await this.source.snapshot(this.options.signal);
|
|
103
|
+
if (digestRecords(snapshot.records) !== manifest.digests.source) {
|
|
104
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SNAPSHOT_CHANGED', 'source snapshot no longer matches the preview; create a new preview');
|
|
86
105
|
}
|
|
87
|
-
manifest.state = '
|
|
106
|
+
manifest.state = 'copying';
|
|
88
107
|
await this.persist(manifest);
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
this.
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
manifest.checkpoint = {
|
|
96
|
-
...manifest.checkpoint,
|
|
97
|
-
...(cursor === undefined ? {} : { cursor }),
|
|
98
|
-
highWaterMark: page.highWaterMark,
|
|
99
|
-
drained: page.records.length === 0 || cursor === undefined,
|
|
100
|
-
};
|
|
108
|
+
await this.copyRecords(snapshot.records, manifest);
|
|
109
|
+
if (manifest.mode === 'online') {
|
|
110
|
+
if (!this.source.changes) {
|
|
111
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_CURSOR_UNAVAILABLE', 'online migration requires a source change cursor');
|
|
112
|
+
}
|
|
113
|
+
manifest.state = 'replaying';
|
|
101
114
|
await this.persist(manifest);
|
|
102
|
-
|
|
103
|
-
break;
|
|
115
|
+
await this.replayChanges(manifest);
|
|
104
116
|
}
|
|
117
|
+
const finalBoundary = await this.options.safety.freeze(manifest, this.options.signal);
|
|
118
|
+
manifest.boundaries.final = finalBoundary;
|
|
119
|
+
assertBoundaryState(finalBoundary, 'quiesced', 'final');
|
|
120
|
+
await this.persist(manifest);
|
|
121
|
+
if (manifest.mode === 'online')
|
|
122
|
+
await this.replayChanges(manifest);
|
|
123
|
+
manifest.state = 'verifying';
|
|
124
|
+
await this.persist(manifest);
|
|
125
|
+
const verification = await this.verify(manifest);
|
|
126
|
+
if (!verification.valid) {
|
|
127
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_PARITY_FAILED', summarizeVerificationFailure(verification));
|
|
128
|
+
}
|
|
129
|
+
manifest.digests.destination = verification.destinationDigest;
|
|
130
|
+
manifest.verification = verification;
|
|
131
|
+
manifest.digests.verification = sha256(stableStringify(verification));
|
|
132
|
+
manifest.state = 'awaiting-approval';
|
|
133
|
+
manifest.updatedAt = this.now().toISOString();
|
|
134
|
+
manifest.digests.approval = approvalDigest(manifest);
|
|
135
|
+
await this.save(manifest);
|
|
136
|
+
return manifest;
|
|
105
137
|
}
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
const verification = await this.verify(manifest);
|
|
109
|
-
if (!verification.valid) {
|
|
138
|
+
catch (error) {
|
|
139
|
+
const stage = manifest.state;
|
|
110
140
|
manifest.state = 'failed';
|
|
111
|
-
manifest.failure = { stage
|
|
112
|
-
|
|
113
|
-
|
|
141
|
+
manifest.failure = { stage, message: errorMessage(error) };
|
|
142
|
+
try {
|
|
143
|
+
await this.persist(manifest);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
// Preserve the triggering failure; the manifest store error is already observable at its source.
|
|
147
|
+
}
|
|
148
|
+
await this.options.safety.release(manifest.boundaries.final ?? manifest.boundaries.initial, 'failed');
|
|
149
|
+
throw error;
|
|
114
150
|
}
|
|
115
|
-
manifest.digests.destination = verification.destinationDigest;
|
|
116
|
-
manifest.state = 'awaiting-approval';
|
|
117
|
-
manifest.updatedAt = this.now().toISOString();
|
|
118
|
-
manifest.digests.approval = approvalDigest(manifest);
|
|
119
|
-
await this.save(manifest);
|
|
120
|
-
return manifest;
|
|
121
151
|
}
|
|
122
152
|
async verify(manifest) {
|
|
123
153
|
const [sourceRecords, destinationRecords] = await Promise.all([
|
|
@@ -131,16 +161,35 @@ export class StorageMigrationCoordinator {
|
|
|
131
161
|
const corrupt = [...sourceMap.entries()]
|
|
132
162
|
.filter(([key, record]) => {
|
|
133
163
|
const destination = destinationMap.get(key);
|
|
134
|
-
return destination !== undefined && (destination.
|
|
164
|
+
return destination !== undefined && (destination.sourceRevision !== record.sourceRevision ||
|
|
165
|
+
destination.digest !== record.digest ||
|
|
135
166
|
Boolean(destination.tombstone) !== Boolean(record.tombstone));
|
|
136
167
|
})
|
|
137
168
|
.map(([key]) => key)
|
|
138
169
|
.sort();
|
|
139
|
-
const
|
|
170
|
+
const boundaryReady = manifest.boundaries.final?.state === 'quiesced';
|
|
171
|
+
const lag = Number(!boundaryReady || (manifest.mode === 'online' && !manifest.checkpoint.drained));
|
|
140
172
|
const sourceDigest = digestRecords(sourceRecords);
|
|
141
173
|
const destinationDigest = digestRecords(destinationRecords);
|
|
174
|
+
const sourceChunks = digestRecordChunks(sourceRecords, this.batchSize);
|
|
175
|
+
const destinationChunks = digestRecordChunks(destinationRecords, this.batchSize);
|
|
176
|
+
const chunks = {
|
|
177
|
+
valid: stableStringify(sourceChunks) === stableStringify(destinationChunks),
|
|
178
|
+
source: sourceChunks,
|
|
179
|
+
destination: destinationChunks,
|
|
180
|
+
};
|
|
181
|
+
const semantic = await this.options.verifier.verify({
|
|
182
|
+
manifest,
|
|
183
|
+
source: this.source,
|
|
184
|
+
destination: this.destination,
|
|
185
|
+
sourceRecords,
|
|
186
|
+
destinationRecords,
|
|
187
|
+
signal: this.options.signal,
|
|
188
|
+
});
|
|
189
|
+
validateSemanticVerification(semantic);
|
|
142
190
|
return {
|
|
143
|
-
valid: missing.length === 0 && unexpected.length === 0 && corrupt.length === 0 && lag === 0
|
|
191
|
+
valid: missing.length === 0 && unexpected.length === 0 && corrupt.length === 0 && lag === 0 &&
|
|
192
|
+
sourceDigest === destinationDigest && chunks.valid && semanticIsValid(semantic),
|
|
144
193
|
sourceCount: sourceRecords.length,
|
|
145
194
|
destinationCount: destinationRecords.length,
|
|
146
195
|
sourceDigest,
|
|
@@ -149,6 +198,8 @@ export class StorageMigrationCoordinator {
|
|
|
149
198
|
unexpected,
|
|
150
199
|
corrupt,
|
|
151
200
|
lag,
|
|
201
|
+
chunks,
|
|
202
|
+
semantic,
|
|
152
203
|
};
|
|
153
204
|
}
|
|
154
205
|
async cutover(manifest, approval) {
|
|
@@ -164,14 +215,22 @@ export class StorageMigrationCoordinator {
|
|
|
164
215
|
if (!verification.valid) {
|
|
165
216
|
throw new MigrationProtocolError('AIWG_MIGRATION_PARITY_FAILED', summarizeVerificationFailure(verification));
|
|
166
217
|
}
|
|
218
|
+
if (!manifest.digests.verification || sha256(stableStringify(verification)) !== manifest.digests.verification) {
|
|
219
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_VERIFICATION_CHANGED', 'migration verification changed after approval; verify and approve a new manifest');
|
|
220
|
+
}
|
|
221
|
+
const finalBoundary = manifest.boundaries.final;
|
|
222
|
+
if (!finalBoundary) {
|
|
223
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_BOUNDARY_MISSING', 'migration has no final quiesced source boundary');
|
|
224
|
+
}
|
|
167
225
|
manifest.state = 'cutover';
|
|
168
226
|
await this.persist(manifest);
|
|
169
227
|
try {
|
|
170
|
-
await this.routing.
|
|
171
|
-
await this.
|
|
228
|
+
manifest.routingSwitch = await this.routing.switchAtomically(manifest);
|
|
229
|
+
await this.options.safety.release(finalBoundary, 'cutover');
|
|
172
230
|
}
|
|
173
231
|
catch (error) {
|
|
174
232
|
await this.routing.restoreSource(manifest);
|
|
233
|
+
await this.options.safety.release(finalBoundary, 'failed');
|
|
175
234
|
manifest.state = 'failed';
|
|
176
235
|
manifest.failure = { stage: 'cutover', message: errorMessage(error) };
|
|
177
236
|
await this.persist(manifest);
|
|
@@ -195,13 +254,47 @@ export class StorageMigrationCoordinator {
|
|
|
195
254
|
throw new MigrationProtocolError('AIWG_MIGRATION_NOT_ROLLBACKABLE', `cannot roll back state ${manifest.state}`);
|
|
196
255
|
}
|
|
197
256
|
await this.routing.restoreSource(manifest);
|
|
257
|
+
await this.options.safety.release(manifest.boundaries.final ?? manifest.boundaries.initial, 'rollback');
|
|
198
258
|
manifest.state = 'rolled-back';
|
|
199
259
|
await this.persist(manifest);
|
|
200
260
|
return manifest;
|
|
201
261
|
}
|
|
202
262
|
async copyRecords(records, manifest) {
|
|
203
263
|
const completed = new Set(manifest.records.map(receiptKey));
|
|
204
|
-
const
|
|
264
|
+
const latest = new Map();
|
|
265
|
+
for (const receipt of manifest.records)
|
|
266
|
+
latest.set(identityKey(receipt.identity), receipt);
|
|
267
|
+
const selected = new Map();
|
|
268
|
+
for (const record of records) {
|
|
269
|
+
if (completed.has(recordKey(record)))
|
|
270
|
+
continue;
|
|
271
|
+
const key = identityKey(record.identity);
|
|
272
|
+
const receipt = latest.get(key);
|
|
273
|
+
if (receipt && this.options.compareSourceRevisions) {
|
|
274
|
+
const order = compareRevisions(this.options.compareSourceRevisions, record.sourceRevision, receipt.sourceRevision);
|
|
275
|
+
if (order < 0)
|
|
276
|
+
continue;
|
|
277
|
+
if (order === 0) {
|
|
278
|
+
if (record.digest !== receipt.contentDigest)
|
|
279
|
+
throw revisionConflict(key, record.sourceRevision);
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
const prior = selected.get(key);
|
|
284
|
+
if (!prior) {
|
|
285
|
+
selected.set(key, record);
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
if (!this.options.compareSourceRevisions) {
|
|
289
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_DUPLICATE_IDENTITY', `snapshot contains multiple records for ${key}`);
|
|
290
|
+
}
|
|
291
|
+
const order = compareRevisions(this.options.compareSourceRevisions, record.sourceRevision, prior.sourceRevision);
|
|
292
|
+
if (order > 0)
|
|
293
|
+
selected.set(key, record);
|
|
294
|
+
else if (order === 0 && record.digest !== prior.digest)
|
|
295
|
+
throw revisionConflict(key, record.sourceRevision);
|
|
296
|
+
}
|
|
297
|
+
const pending = [...selected.values()].sort((left, right) => identityKey(left.identity).localeCompare(identityKey(right.identity)));
|
|
205
298
|
const batches = chunk(pending, this.batchSize);
|
|
206
299
|
let next = 0;
|
|
207
300
|
const workers = Array.from({ length: Math.min(this.concurrency, batches.length) }, async () => {
|
|
@@ -210,7 +303,7 @@ export class StorageMigrationCoordinator {
|
|
|
210
303
|
if (index >= batches.length)
|
|
211
304
|
return;
|
|
212
305
|
const batch = batches[index];
|
|
213
|
-
const mutations = batch.map(record => mutationFor(manifest.migrationId, record));
|
|
306
|
+
const mutations = batch.map(record => mutationFor(manifest.migrationId, record, latest.get(identityKey(record.identity))?.destinationRevision));
|
|
214
307
|
const receipt = await this.retry(() => this.destination.commitBatch(mutations, this.options.signal), `batch ${index}`);
|
|
215
308
|
if (!receipt.committed || receipt.recordReceipts.length !== batch.length) {
|
|
216
309
|
throw new MigrationProtocolError('AIWG_MIGRATION_INVALID_RECEIPT', `destination returned an invalid receipt for batch ${index}`);
|
|
@@ -242,6 +335,27 @@ export class StorageMigrationCoordinator {
|
|
|
242
335
|
});
|
|
243
336
|
await Promise.all(workers);
|
|
244
337
|
}
|
|
338
|
+
async replayChanges(manifest) {
|
|
339
|
+
if (!this.source.changes) {
|
|
340
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_CURSOR_UNAVAILABLE', 'online migration requires a source change cursor');
|
|
341
|
+
}
|
|
342
|
+
let cursor = manifest.checkpoint.cursor ?? manifest.snapshot.cursor;
|
|
343
|
+
for (;;) {
|
|
344
|
+
this.throwIfAborted();
|
|
345
|
+
const page = await this.source.changes(cursor, this.options.signal);
|
|
346
|
+
await this.copyRecords(page.records, manifest);
|
|
347
|
+
cursor = page.nextCursor ?? page.highWaterMark;
|
|
348
|
+
manifest.checkpoint = {
|
|
349
|
+
...manifest.checkpoint,
|
|
350
|
+
cursor,
|
|
351
|
+
highWaterMark: page.highWaterMark,
|
|
352
|
+
drained: page.records.length === 0 || page.nextCursor === undefined,
|
|
353
|
+
};
|
|
354
|
+
await this.persist(manifest);
|
|
355
|
+
if (manifest.checkpoint.drained)
|
|
356
|
+
break;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
245
359
|
async retry(operation, label) {
|
|
246
360
|
let attempt = 0;
|
|
247
361
|
for (;;) {
|
|
@@ -250,7 +364,7 @@ export class StorageMigrationCoordinator {
|
|
|
250
364
|
return await operation();
|
|
251
365
|
}
|
|
252
366
|
catch (error) {
|
|
253
|
-
const retryable = error
|
|
367
|
+
const retryable = isRetryableError(error);
|
|
254
368
|
if (!retryable)
|
|
255
369
|
throw error;
|
|
256
370
|
if (attempt >= this.maxRetries) {
|
|
@@ -290,6 +404,10 @@ export function digestRecords(records) {
|
|
|
290
404
|
.sort((left, right) => identityKey(left.identity).localeCompare(identityKey(right.identity)));
|
|
291
405
|
return sha256(stableStringify(normalized));
|
|
292
406
|
}
|
|
407
|
+
export function digestRecordChunks(records, chunkSize) {
|
|
408
|
+
const sorted = [...records].sort((left, right) => identityKey(left.identity).localeCompare(identityKey(right.identity)));
|
|
409
|
+
return chunk(sorted, chunkSize).map(recordsInChunk => digestRecords(recordsInChunk));
|
|
410
|
+
}
|
|
293
411
|
export function approvalDigest(manifest) {
|
|
294
412
|
const copy = structuredClone(manifest);
|
|
295
413
|
delete copy.digests.approval;
|
|
@@ -302,6 +420,10 @@ export function validateManifest(manifest, source, destination) {
|
|
|
302
420
|
if (!manifest.migrationId || !manifest.snapshot?.id || !manifest.digests?.source) {
|
|
303
421
|
throw new MigrationProtocolError('AIWG_MIGRATION_MANIFEST_CORRUPT', 'migration manifest is missing required identity or digest fields');
|
|
304
422
|
}
|
|
423
|
+
if (!manifest.boundaries?.initial) {
|
|
424
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_BOUNDARY_MISSING', 'migration manifest has no durable source boundary receipt');
|
|
425
|
+
}
|
|
426
|
+
assertBoundaryState(manifest.boundaries.initial, manifest.mode === 'offline' ? 'quiesced' : 'tracking', 'initial');
|
|
305
427
|
if (source && stableStringify(manifest.source) !== stableStringify(source)) {
|
|
306
428
|
throw new MigrationProtocolError('AIWG_MIGRATION_SOURCE_MISMATCH', 'manifest source does not match the active source');
|
|
307
429
|
}
|
|
@@ -309,13 +431,29 @@ export function validateManifest(manifest, source, destination) {
|
|
|
309
431
|
throw new MigrationProtocolError('AIWG_MIGRATION_DESTINATION_MISMATCH', 'manifest destination does not match the active destination');
|
|
310
432
|
}
|
|
311
433
|
}
|
|
312
|
-
function
|
|
434
|
+
function assertBoundaryState(boundary, expected, stage) {
|
|
435
|
+
if (!boundary.boundaryId || !boundary.establishedAt || boundary.state !== expected) {
|
|
436
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_BOUNDARY_INVALID', `${stage} source boundary must provide a durable ${expected} receipt`);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
function mutationFor(migrationId, record, expectedRevision) {
|
|
313
440
|
return {
|
|
314
441
|
operation: record.tombstone ? 'delete' : 'upsert',
|
|
315
442
|
record,
|
|
316
443
|
idempotencyKey: sha256(`${migrationId}\0${recordKey(record)}`),
|
|
444
|
+
...(expectedRevision === undefined ? {} : { expectedRevision }),
|
|
317
445
|
};
|
|
318
446
|
}
|
|
447
|
+
function revisionConflict(identity, revision) {
|
|
448
|
+
return new MigrationProtocolError('AIWG_MIGRATION_REVISION_CONFLICT', `source revision ${revision} has conflicting content for ${identity}`);
|
|
449
|
+
}
|
|
450
|
+
function compareRevisions(comparator, left, right) {
|
|
451
|
+
const result = comparator(left, right);
|
|
452
|
+
if (!Number.isFinite(result)) {
|
|
453
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_REVISION_ORDER_INVALID', 'source-revision comparator returned a non-finite result');
|
|
454
|
+
}
|
|
455
|
+
return Math.sign(result);
|
|
456
|
+
}
|
|
319
457
|
function recordMap(records) {
|
|
320
458
|
return new Map(records.map(record => [identityKey(record.identity), record]));
|
|
321
459
|
}
|
|
@@ -357,11 +495,51 @@ function boundedInteger(raw, fallback, min, max, label) {
|
|
|
357
495
|
return value;
|
|
358
496
|
}
|
|
359
497
|
function summarizeVerificationFailure(verification) {
|
|
360
|
-
|
|
498
|
+
const semanticFailures = Object.values(verification.semantic).filter(check => !check.valid).length;
|
|
499
|
+
return `migration parity failed: missing=${verification.missing.length} unexpected=${verification.unexpected.length} corrupt=${verification.corrupt.length} chunks=${verification.chunks.valid ? 'valid' : 'invalid'} semantic=${semanticFailures} lag=${verification.lag}`;
|
|
500
|
+
}
|
|
501
|
+
function validateSemanticVerification(verification) {
|
|
502
|
+
for (const dimension of [
|
|
503
|
+
'schema', 'constraints', 'countsByType', 'edgeIntegrity', 'queryParity', 'traversalParity',
|
|
504
|
+
]) {
|
|
505
|
+
const check = verification?.[dimension];
|
|
506
|
+
if (!check || typeof check.valid !== 'boolean' || !Number.isInteger(check.declared) || check.declared < 0 ||
|
|
507
|
+
!Number.isInteger(check.checked) || check.checked < 0 || check.checked !== check.declared || !Array.isArray(check.failures)) {
|
|
508
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', `semantic verification is incomplete for ${dimension}`);
|
|
509
|
+
}
|
|
510
|
+
if (check.valid !== (check.failures.length === 0)) {
|
|
511
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', `semantic verification validity disagrees with failures for ${dimension}`);
|
|
512
|
+
}
|
|
513
|
+
if (check.evidenceDigest !== undefined && !/^[0-9A-Za-z._:-]+$/.test(check.evidenceDigest)) {
|
|
514
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', `semantic verification evidence digest is invalid for ${dimension}`);
|
|
515
|
+
}
|
|
516
|
+
if (['schema', 'constraints', 'queryParity'].includes(dimension) && check.declared < 1) {
|
|
517
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', `semantic verification must exercise at least one ${dimension} check`);
|
|
518
|
+
}
|
|
519
|
+
if (dimension === 'countsByType') {
|
|
520
|
+
if (!isCountMap(check.sourceCounts) || !isCountMap(check.destinationCounts)) {
|
|
521
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', 'countsByType must include non-negative integer source and destination maps');
|
|
522
|
+
}
|
|
523
|
+
const declaredTypes = new Set([...Object.keys(check.sourceCounts), ...Object.keys(check.destinationCounts)]).size;
|
|
524
|
+
if (check.declared !== declaredTypes || (check.valid && stableStringify(check.sourceCounts) !== stableStringify(check.destinationCounts))) {
|
|
525
|
+
throw new MigrationProtocolError('AIWG_MIGRATION_SEMANTIC_VERIFICATION_INVALID', 'countsByType scope or validity does not match its declared maps');
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
function semanticIsValid(verification) {
|
|
531
|
+
return Object.values(verification).every(check => check.valid);
|
|
532
|
+
}
|
|
533
|
+
function isCountMap(value) {
|
|
534
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value) &&
|
|
535
|
+
Object.values(value).every(count => Number.isInteger(count) && count >= 0);
|
|
361
536
|
}
|
|
362
537
|
function errorMessage(error) {
|
|
363
538
|
return error instanceof Error ? error.message : String(error);
|
|
364
539
|
}
|
|
540
|
+
function isRetryableError(error) {
|
|
541
|
+
return typeof error === 'object' && error !== null && 'retryable' in error && error.retryable === true;
|
|
542
|
+
}
|
|
365
543
|
async function abortableDelay(ms, signal) {
|
|
366
544
|
if (!signal) {
|
|
367
545
|
await new Promise(resolve => setTimeout(resolve, ms));
|
|
@@ -9,6 +9,10 @@ export async function qualifyStorageBackend(endpoint, options) {
|
|
|
9
9
|
if (options.records.length !== options.scope.declaredRecords) {
|
|
10
10
|
throw new Error('declared record scope does not match the qualification corpus');
|
|
11
11
|
}
|
|
12
|
+
const readerCount = bounded(options.scope.readers, 1, 1, 32, 'readers');
|
|
13
|
+
if (options.scope.operations !== options.records.length + readerCount) {
|
|
14
|
+
throw new Error('declared operation scope must equal record writes plus concurrent readers');
|
|
15
|
+
}
|
|
12
16
|
const now = options.now ?? (() => new Date());
|
|
13
17
|
const startedAt = now().toISOString();
|
|
14
18
|
const cpuBefore = process.cpuUsage();
|
|
@@ -21,7 +25,7 @@ export async function qualifyStorageBackend(endpoint, options) {
|
|
|
21
25
|
const maxRetries = bounded(options.maxRetries, 5, 0, 20, 'maxRetries');
|
|
22
26
|
const baseBackoffMs = bounded(options.baseBackoffMs, 2, 0, 60_000, 'baseBackoffMs');
|
|
23
27
|
const batches = distribute(options.records, options.scope.writers);
|
|
24
|
-
|
|
28
|
+
const writes = Promise.all(batches.map(async (records, writer) => {
|
|
25
29
|
for (const record of records) {
|
|
26
30
|
const mutation = mutationFor(record, `qualification:${writer}:${record.identity.path}:${record.sourceRevision}`);
|
|
27
31
|
const operationStart = performance.now();
|
|
@@ -48,19 +52,33 @@ export async function qualifyStorageBackend(endpoint, options) {
|
|
|
48
52
|
sideEffects.push({ operation: `write:${record.identity.path}`, outcome: replayed ? 'replayed' : 'committed', batchId: receipt.batchId });
|
|
49
53
|
}
|
|
50
54
|
}));
|
|
55
|
+
const reads = Promise.all(Array.from({ length: readerCount }, async (_, reader) => {
|
|
56
|
+
const operationStart = performance.now();
|
|
57
|
+
try {
|
|
58
|
+
await endpoint.readAll();
|
|
59
|
+
latencies.push(performance.now() - operationStart);
|
|
60
|
+
sideEffects.push({ operation: `read:${reader}`, outcome: 'committed' });
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
errors += 1;
|
|
64
|
+
sideEffects.push({ operation: `read:${reader}`, outcome: 'failed' });
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
}));
|
|
68
|
+
await Promise.all([writes, reads]);
|
|
51
69
|
const expectedDigest = digestRecords(options.records);
|
|
52
70
|
const observed = [...await endpoint.readAll()];
|
|
53
71
|
const verification = verifyExactRecords(options.records, observed);
|
|
54
72
|
const durationMs = Math.max(performance.now() - start, 0.001);
|
|
55
73
|
const cpu = process.cpuUsage(cpuBefore);
|
|
56
|
-
const supplied = options.resourceObservation?.() ?? {};
|
|
57
|
-
const operations = options.records.length +
|
|
74
|
+
const supplied = await options.resourceObservation?.() ?? {};
|
|
75
|
+
const operations = options.records.length + readerCount;
|
|
58
76
|
const report = {
|
|
59
77
|
schemaVersion: STORAGE_QUALIFICATION_REPORT,
|
|
60
78
|
runId: createHash('sha256').update(`${options.scope.commit}\0${options.scope.backend}\0${startedAt}`).digest('hex'),
|
|
61
79
|
startedAt,
|
|
62
80
|
completedAt: now().toISOString(),
|
|
63
|
-
scope: { ...options.scope, observedRecords: observed.length },
|
|
81
|
+
scope: { ...options.scope, observedRecords: observed.length, observedOperations: sideEffects.length },
|
|
64
82
|
verification: { ...verification, expectedDigest, observedDigest: digestRecords(observed) },
|
|
65
83
|
latencyMs: percentiles(latencies),
|
|
66
84
|
throughputPerSecond: operations / (durationMs / 1000),
|
|
@@ -71,7 +89,14 @@ export async function qualifyStorageBackend(endpoint, options) {
|
|
|
71
89
|
resources: {
|
|
72
90
|
cpuUserMicros: cpu.user, cpuSystemMicros: cpu.system,
|
|
73
91
|
rssBytes: Math.max(process.memoryUsage().rss, rssBefore),
|
|
74
|
-
|
|
92
|
+
databaseBytes: resourceMetric(supplied.databaseBytes, 'databaseBytes'),
|
|
93
|
+
writeAmplification: resourceMetric(supplied.writeAmplification, 'writeAmplification'),
|
|
94
|
+
walBytes: resourceMetric(supplied.walBytes, 'walBytes'),
|
|
95
|
+
lockWaits: resourceMetric(supplied.lockWaits, 'lockWaits'),
|
|
96
|
+
poolSaturation: resourceMetric(supplied.poolSaturation, 'poolSaturation'),
|
|
97
|
+
migrationMs: resourceMetric(supplied.migrationMs, 'migrationMs'),
|
|
98
|
+
recoveryMs: resourceMetric(supplied.recoveryMs, 'recoveryMs'),
|
|
99
|
+
transportOverheadMs: resourceMetric(supplied.transportOverheadMs, 'transportOverheadMs'),
|
|
75
100
|
},
|
|
76
101
|
sideEffects: sideEffects.sort((a, b) => a.operation.localeCompare(b.operation)),
|
|
77
102
|
};
|
|
@@ -109,6 +134,8 @@ export function assertCurrentStorageEvidence(report, commit) {
|
|
|
109
134
|
throw new Error('storage qualification record is stale for the requested commit');
|
|
110
135
|
if (report.scope.declaredRecords !== report.scope.observedRecords)
|
|
111
136
|
throw new Error('storage qualification scope is incomplete');
|
|
137
|
+
if (report.scope.operations !== report.scope.observedOperations)
|
|
138
|
+
throw new Error('storage qualification operation scope is incomplete');
|
|
112
139
|
}
|
|
113
140
|
function mutationFor(record, idempotencyKey) {
|
|
114
141
|
return { operation: record.tombstone ? 'delete' : 'upsert', record, idempotencyKey };
|
|
@@ -134,4 +161,11 @@ function bounded(value, fallback, min, max, label) {
|
|
|
134
161
|
throw new Error(`${label} must be an integer from ${min} through ${max}`);
|
|
135
162
|
return resolved;
|
|
136
163
|
}
|
|
164
|
+
function resourceMetric(value, label) {
|
|
165
|
+
if (value === undefined || value === null)
|
|
166
|
+
return null;
|
|
167
|
+
if (!Number.isFinite(value) || value < 0)
|
|
168
|
+
throw new Error(`${label} must be a finite non-negative number`);
|
|
169
|
+
return value;
|
|
170
|
+
}
|
|
137
171
|
//# sourceMappingURL=qualification.js.map
|