@git.zone/cli 7.4.1 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/mod_services/classes.dockercontainer.d.ts +13 -2
- package/dist_ts/mod_services/classes.dockercontainer.js +40 -40
- package/dist_ts/mod_services/classes.globalregistry.d.ts +12 -0
- package/dist_ts/mod_services/classes.globalregistry.js +96 -13
- package/dist_ts/mod_services/classes.globalregistrystore.d.ts +7 -1
- package/dist_ts/mod_services/classes.globalregistrystore.js +6 -6
- package/dist_ts/mod_services/classes.legacymongomigration.d.ts +239 -0
- package/dist_ts/mod_services/classes.legacymongomigration.js +970 -0
- package/dist_ts/mod_services/classes.serviceconfiguration.d.ts +18 -8
- package/dist_ts/mod_services/classes.serviceconfiguration.js +47 -35
- package/dist_ts/mod_services/classes.servicedatamarker.d.ts +42 -1
- package/dist_ts/mod_services/classes.servicedatamarker.js +101 -8
- package/dist_ts/mod_services/classes.servicemanager.d.ts +99 -33
- package/dist_ts/mod_services/classes.servicemanager.js +652 -384
- package/dist_ts/mod_services/classes.servicepruner.js +27 -20
- package/dist_ts/mod_services/index.js +51 -7
- package/dist_ts/mod_services/mod.plugins.d.ts +2 -1
- package/dist_ts/mod_services/mod.plugins.js +4 -2
- package/dist_ts_migration/002.runtime-config.d.ts +1 -1
- package/dist_ts_migration/002.runtime-config.js +8 -2
- package/dist_ts_migration/003.service-data-marker.js +125 -30
- package/dist_ts_migration/004.global-registry.js +30 -8
- package/package.json +1 -1
- package/readme.md +127 -28
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/mod_services/classes.dockercontainer.ts +58 -50
- package/ts/mod_services/classes.globalregistry.ts +115 -14
- package/ts/mod_services/classes.globalregistrystore.ts +16 -6
- package/ts/mod_services/classes.legacymongomigration.ts +1308 -0
- package/ts/mod_services/classes.serviceconfiguration.ts +64 -40
- package/ts/mod_services/classes.servicedatamarker.ts +149 -8
- package/ts/mod_services/classes.servicemanager.ts +877 -454
- package/ts/mod_services/classes.servicepruner.ts +27 -19
- package/ts/mod_services/index.ts +67 -6
- package/ts/mod_services/mod.plugins.ts +4 -1
- package/ts_migration/002.runtime-config.ts +8 -1
- package/ts_migration/003.service-data-marker.ts +168 -29
- package/ts_migration/004.global-registry.ts +38 -7
|
@@ -0,0 +1,1308 @@
|
|
|
1
|
+
import * as plugins from './mod.plugins.js';
|
|
2
|
+
import type { IServiceConfig } from './classes.serviceconfiguration.js';
|
|
3
|
+
import type {
|
|
4
|
+
IDockerCreateArgvOptions,
|
|
5
|
+
IDockerExecArgvOptions,
|
|
6
|
+
IDockerExecArgvResult,
|
|
7
|
+
IFencedBindOperationOptions,
|
|
8
|
+
IRemoveDataDirectoryOptions,
|
|
9
|
+
} from './classes.dockercontainer.js';
|
|
10
|
+
import type { IRegisteredProject } from './classes.globalregistrystore.js';
|
|
11
|
+
import {
|
|
12
|
+
getLegacyMongoContainerLabels,
|
|
13
|
+
getLegacyMongoDataDirectory,
|
|
14
|
+
getServiceContainerLabels,
|
|
15
|
+
getServiceDataDirectory,
|
|
16
|
+
hasExactServiceContainerLabels,
|
|
17
|
+
legacyMongoImage,
|
|
18
|
+
markerRecordsLegacyDataDirectory,
|
|
19
|
+
noSqlDbImage,
|
|
20
|
+
readServiceDataMarker,
|
|
21
|
+
recordLegacyServiceDataDirectory,
|
|
22
|
+
serviceBindOperation,
|
|
23
|
+
serviceDataOwner,
|
|
24
|
+
serviceOperationLabel,
|
|
25
|
+
serviceToolLabel,
|
|
26
|
+
} from './classes.servicedatamarker.js';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Journal of the one-way mongod → NoSQLDB data migration of a project.
|
|
30
|
+
*
|
|
31
|
+
* Invariant: while the journal is `running`, no application has ever been able
|
|
32
|
+
* to reach the NoSQLDB data. The restore-phase NoSQLDB container has no network
|
|
33
|
+
* and no published port, and its restart policy is `no`; the published service
|
|
34
|
+
* container is created only after the journal has moved to `committing`. Any
|
|
35
|
+
* NoSQLDB data that exists under a `running` journal was therefore written by
|
|
36
|
+
* the migration itself and is discarded by the rollback.
|
|
37
|
+
*/
|
|
38
|
+
export interface ILegacyMongoMigrationJournal {
|
|
39
|
+
owner: typeof serviceDataOwner;
|
|
40
|
+
kind: 'mongodb-nosqldb-migration';
|
|
41
|
+
projectPath: string;
|
|
42
|
+
token: string;
|
|
43
|
+
createdAt: string;
|
|
44
|
+
/**
|
|
45
|
+
* `running`: rollback required before anything else may happen.
|
|
46
|
+
* `committing`: verification passed; publishing the NoSQLDB service and
|
|
47
|
+
* removing the legacy container remain.
|
|
48
|
+
* `completed`: permanent record; the legacy data is preserved evidence.
|
|
49
|
+
*/
|
|
50
|
+
state: 'running' | 'committing' | 'completed';
|
|
51
|
+
evidenceDirectory: string;
|
|
52
|
+
/** The proven legacy container; the rollback restores its running state. */
|
|
53
|
+
legacyContainer?: {
|
|
54
|
+
id: string;
|
|
55
|
+
name: string;
|
|
56
|
+
wasRunning: boolean;
|
|
57
|
+
};
|
|
58
|
+
/** The canonical NoSQLDB data directory was absent and is owned by this migration. */
|
|
59
|
+
targetCreated: boolean;
|
|
60
|
+
completedAt?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface ILegacyMongoMigrationDocker {
|
|
64
|
+
execArgv(
|
|
65
|
+
argvArg: readonly string[],
|
|
66
|
+
optionsArg: IDockerExecArgvOptions,
|
|
67
|
+
): Promise<IDockerExecArgvResult>;
|
|
68
|
+
createArgv(
|
|
69
|
+
optionsArg: IDockerCreateArgvOptions,
|
|
70
|
+
executionOptionsArg: IDockerExecArgvOptions,
|
|
71
|
+
): Promise<IDockerExecArgvResult>;
|
|
72
|
+
runFencedBindOperation(optionsArg: IFencedBindOperationOptions): Promise<void>;
|
|
73
|
+
removeDataDirectoryHoldingLock(
|
|
74
|
+
hostPathArg: string,
|
|
75
|
+
imageArg: string,
|
|
76
|
+
optionsArg: IRemoveDataDirectoryOptions,
|
|
77
|
+
): Promise<boolean>;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** Everything the migration needs from its owning service manager. */
|
|
81
|
+
export interface ILegacyMongoMigrationHost {
|
|
82
|
+
projectPath: string;
|
|
83
|
+
docker: ILegacyMongoMigrationDocker;
|
|
84
|
+
config: IServiceConfig;
|
|
85
|
+
containerNames: { nosqldb: string; legacyMongo: string };
|
|
86
|
+
/** `uid:gid` the tool helpers run as, so migration evidence stays user-owned. */
|
|
87
|
+
helperUser: string;
|
|
88
|
+
discoverContainerIdByExactName(
|
|
89
|
+
containerNameArg: string,
|
|
90
|
+
dockerTimeoutArg: () => IDockerExecArgvOptions,
|
|
91
|
+
): Promise<string | undefined>;
|
|
92
|
+
getRegistryProjects(): Promise<Record<string, IRegisteredProject>>;
|
|
93
|
+
clearRegistryLegacyMongoContainer(projectPathArg: string, containerNameArg: string): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* Create and start the canonical NoSQLDB container for the restore phase:
|
|
96
|
+
* no network, no published port, restart policy `no`. Readiness is proven
|
|
97
|
+
* by the migration through its own namespace.
|
|
98
|
+
*/
|
|
99
|
+
startMigrationNoSqlDb(): Promise<{ id: string }>;
|
|
100
|
+
/** Gracefully stop the restore-phase NoSQLDB container when it is running. */
|
|
101
|
+
stopMigrationNoSqlDb(): Promise<void>;
|
|
102
|
+
/**
|
|
103
|
+
* Replace the stopped restore-phase container with the published service
|
|
104
|
+
* container on the same data and wait until it is ready.
|
|
105
|
+
*/
|
|
106
|
+
publishMigratedNoSqlDb(): Promise<{ id: string }>;
|
|
107
|
+
log(levelArg: 'info' | 'note' | 'ok' | 'error', messageArg: string): void;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Registry or marker bookkeeping a `none` assessment leaves to the locked `run()`. */
|
|
111
|
+
export type TLegacyMongoFollowUp =
|
|
112
|
+
| { kind: 'clear-registry'; containerName: string }
|
|
113
|
+
| { kind: 'record-empty-legacy-data' };
|
|
114
|
+
|
|
115
|
+
export type TLegacyMongoAssessment =
|
|
116
|
+
| { kind: 'none'; followUp?: TLegacyMongoFollowUp }
|
|
117
|
+
| { kind: 'migratable'; source: ILegacyMongoSource }
|
|
118
|
+
| { kind: 'blocked'; evidence: string[]; reason: string };
|
|
119
|
+
|
|
120
|
+
export interface ILegacyMongoSource {
|
|
121
|
+
container?: ILegacyMongoContainer;
|
|
122
|
+
dataPath: string;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface ILegacyMongoContainer {
|
|
126
|
+
id: string;
|
|
127
|
+
name: string;
|
|
128
|
+
running: boolean;
|
|
129
|
+
/** Immutable image id the legacy mongod runs; the reader opens the data with it. */
|
|
130
|
+
imageId: string;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export const legacyMongoMigrationJournalFile = '.gitzone-mongodb-migration.json';
|
|
134
|
+
export const legacyMongoMigrationEvidenceDirectoryName = 'mongodb-migration';
|
|
135
|
+
const queryTimeoutMs = 60_000;
|
|
136
|
+
const toolTimeoutMs = 30 * 60_000;
|
|
137
|
+
const toolContainerPath = '/migration';
|
|
138
|
+
/** Seconds mongod gets for a clean shutdown before Docker kills it. */
|
|
139
|
+
const mongodStopGraceSeconds = 60;
|
|
140
|
+
const mongodStopTimeoutMs = (mongodStopGraceSeconds + 60) * 1_000;
|
|
141
|
+
const readerReadinessTimeoutMs = 10 * 60_000;
|
|
142
|
+
const readerPollIntervalMs = 1_000;
|
|
143
|
+
const targetReadinessTimeoutMs = 2 * 60_000;
|
|
144
|
+
/** mongod fatal assertion: `recoverFromOplogAsStandalone` needs a stable checkpoint. */
|
|
145
|
+
const noStableCheckpointLogId = 31229;
|
|
146
|
+
const readerWirePort = '27017';
|
|
147
|
+
|
|
148
|
+
export const getLegacyMongoMigrationJournalPath = (projectPathArg: string): string =>
|
|
149
|
+
plugins.path.join(plugins.path.resolve(projectPathArg), '.nogit', legacyMongoMigrationJournalFile);
|
|
150
|
+
|
|
151
|
+
export const getLegacyMongoMigrationEvidenceRoot = (projectPathArg: string): string =>
|
|
152
|
+
plugins.path.join(
|
|
153
|
+
plugins.path.resolve(projectPathArg),
|
|
154
|
+
'.nogit',
|
|
155
|
+
legacyMongoMigrationEvidenceDirectoryName,
|
|
156
|
+
);
|
|
157
|
+
|
|
158
|
+
/** Name of the temporary mongod that reads the frozen legacy data for one migration run. */
|
|
159
|
+
export const getLegacyMongoReaderName = (legacyContainerNameArg: string, tokenArg: string): string =>
|
|
160
|
+
`${legacyContainerNameArg}-migration-${tokenArg}`;
|
|
161
|
+
|
|
162
|
+
/** Exact label envelope of that reader: an operation helper owned by one migration run. */
|
|
163
|
+
export const getLegacyMongoReaderLabels = (
|
|
164
|
+
projectPathArg: string,
|
|
165
|
+
tokenArg: string,
|
|
166
|
+
): Record<string, string> => ({
|
|
167
|
+
'git.zone.tool': serviceToolLabel,
|
|
168
|
+
'git.zone.service': 'mongodb',
|
|
169
|
+
'git.zone.project-path': plugins.path.resolve(projectPathArg),
|
|
170
|
+
'git.zone.data-path': getLegacyMongoDataDirectory(projectPathArg),
|
|
171
|
+
'git.zone.safe-to-prune': 'false',
|
|
172
|
+
[serviceOperationLabel]: serviceBindOperation,
|
|
173
|
+
'git.zone.migration-token': tokenArg,
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* mongod arguments of the reader. It runs standalone, so the replica-set
|
|
178
|
+
* configuration stored in the data (which names the legacy port) is never
|
|
179
|
+
* consulted, and without access control, because it has no network: only
|
|
180
|
+
* helpers that join its namespace can reach it. `recoverFromOplogAsStandalone`
|
|
181
|
+
* replays the oplog up to its last entry, exactly as the replica-set member
|
|
182
|
+
* would on startup, and then keeps the node read-only. A plain standalone start
|
|
183
|
+
* would expose only the last stable checkpoint and lose every write after it
|
|
184
|
+
* when the legacy mongod had not shut down cleanly.
|
|
185
|
+
*/
|
|
186
|
+
export const getLegacyMongoReaderCommand = (oplogRecoveryArg: boolean): string[] => [
|
|
187
|
+
'--port',
|
|
188
|
+
readerWirePort,
|
|
189
|
+
'--bind_ip',
|
|
190
|
+
'127.0.0.1',
|
|
191
|
+
...(oplogRecoveryArg ? ['--setParameter', 'recoverFromOplogAsStandalone=true'] : []),
|
|
192
|
+
];
|
|
193
|
+
|
|
194
|
+
const isPlainObject = (valueArg: unknown): valueArg is Record<string, unknown> =>
|
|
195
|
+
typeof valueArg === 'object' && valueArg !== null && !Array.isArray(valueArg);
|
|
196
|
+
|
|
197
|
+
const isContainerId = (valueArg: unknown): valueArg is string =>
|
|
198
|
+
typeof valueArg === 'string' && /^[a-f0-9]{64}$/u.test(valueArg);
|
|
199
|
+
|
|
200
|
+
const isImageId = (valueArg: unknown): valueArg is string =>
|
|
201
|
+
typeof valueArg === 'string' && /^sha256:[a-f0-9]{64}$/u.test(valueArg);
|
|
202
|
+
|
|
203
|
+
/** Validate a persisted journal exactly; anything else is refused, never guessed. */
|
|
204
|
+
export const parseLegacyMongoMigrationJournal = (
|
|
205
|
+
valueArg: unknown,
|
|
206
|
+
projectPathArg: string,
|
|
207
|
+
): ILegacyMongoMigrationJournal => {
|
|
208
|
+
const projectPath = plugins.path.resolve(projectPathArg);
|
|
209
|
+
const invalid = new Error(
|
|
210
|
+
`Invalid or foreign MongoDB migration journal: ${getLegacyMongoMigrationJournalPath(projectPath)}`,
|
|
211
|
+
);
|
|
212
|
+
if (
|
|
213
|
+
!isPlainObject(valueArg) ||
|
|
214
|
+
Object.keys(valueArg).some(
|
|
215
|
+
(keyArg) =>
|
|
216
|
+
![
|
|
217
|
+
'owner',
|
|
218
|
+
'kind',
|
|
219
|
+
'projectPath',
|
|
220
|
+
'token',
|
|
221
|
+
'createdAt',
|
|
222
|
+
'state',
|
|
223
|
+
'evidenceDirectory',
|
|
224
|
+
'legacyContainer',
|
|
225
|
+
'targetCreated',
|
|
226
|
+
'completedAt',
|
|
227
|
+
].includes(keyArg),
|
|
228
|
+
) ||
|
|
229
|
+
valueArg.owner !== serviceDataOwner ||
|
|
230
|
+
valueArg.kind !== 'mongodb-nosqldb-migration' ||
|
|
231
|
+
valueArg.projectPath !== projectPath ||
|
|
232
|
+
typeof valueArg.token !== 'string' ||
|
|
233
|
+
!/^[a-f0-9]{16}$/u.test(valueArg.token) ||
|
|
234
|
+
typeof valueArg.createdAt !== 'string' ||
|
|
235
|
+
!['running', 'committing', 'completed'].includes(valueArg.state as string) ||
|
|
236
|
+
typeof valueArg.evidenceDirectory !== 'string' ||
|
|
237
|
+
plugins.path.dirname(valueArg.evidenceDirectory) !==
|
|
238
|
+
getLegacyMongoMigrationEvidenceRoot(projectPath) ||
|
|
239
|
+
typeof valueArg.targetCreated !== 'boolean' ||
|
|
240
|
+
(valueArg.completedAt !== undefined && typeof valueArg.completedAt !== 'string')
|
|
241
|
+
) {
|
|
242
|
+
throw invalid;
|
|
243
|
+
}
|
|
244
|
+
const legacyContainer = valueArg.legacyContainer;
|
|
245
|
+
if (legacyContainer !== undefined) {
|
|
246
|
+
if (
|
|
247
|
+
!isPlainObject(legacyContainer) ||
|
|
248
|
+
!isContainerId(legacyContainer.id) ||
|
|
249
|
+
typeof legacyContainer.name !== 'string' ||
|
|
250
|
+
!legacyContainer.name ||
|
|
251
|
+
typeof legacyContainer.wasRunning !== 'boolean' ||
|
|
252
|
+
Object.keys(legacyContainer).length !== 3
|
|
253
|
+
) {
|
|
254
|
+
throw invalid;
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
return valueArg as unknown as ILegacyMongoMigrationJournal;
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
export interface ILegacyMongoInventoryEntry {
|
|
261
|
+
count: number;
|
|
262
|
+
indexes: string[];
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export type TLegacyMongoInventory = Record<string, ILegacyMongoInventoryEntry>;
|
|
266
|
+
|
|
267
|
+
/** What the inventory helper reports for one side of the migration. */
|
|
268
|
+
export interface ILegacyMongoInventoryReport {
|
|
269
|
+
collections: TLegacyMongoInventory;
|
|
270
|
+
/** View namespaces; NoSQLDB has no general views, so any view blocks the migration. */
|
|
271
|
+
views: string[];
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/** Validate an inventory report written by the tool helper. */
|
|
275
|
+
export const parseLegacyMongoInventory = (valueArg: unknown): ILegacyMongoInventoryReport => {
|
|
276
|
+
if (
|
|
277
|
+
!isPlainObject(valueArg) ||
|
|
278
|
+
Object.keys(valueArg).length !== 2 ||
|
|
279
|
+
!isPlainObject(valueArg.collections) ||
|
|
280
|
+
!Array.isArray(valueArg.views) ||
|
|
281
|
+
!valueArg.views.every((viewArg) => typeof viewArg === 'string')
|
|
282
|
+
) {
|
|
283
|
+
throw new Error('MongoDB migration inventory is not a {collections, views} report.');
|
|
284
|
+
}
|
|
285
|
+
const collections: TLegacyMongoInventory = {};
|
|
286
|
+
for (const [namespace, entry] of Object.entries(valueArg.collections)) {
|
|
287
|
+
if (
|
|
288
|
+
!isPlainObject(entry) ||
|
|
289
|
+
typeof entry.count !== 'number' ||
|
|
290
|
+
!Number.isSafeInteger(entry.count) ||
|
|
291
|
+
entry.count < 0 ||
|
|
292
|
+
!Array.isArray(entry.indexes) ||
|
|
293
|
+
!entry.indexes.every((indexArg) => typeof indexArg === 'string')
|
|
294
|
+
) {
|
|
295
|
+
throw new Error(`MongoDB migration inventory entry ${namespace} is invalid.`);
|
|
296
|
+
}
|
|
297
|
+
collections[namespace] = {
|
|
298
|
+
count: entry.count,
|
|
299
|
+
indexes: [...(entry.indexes as string[])].sort(),
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
return { collections, views: [...(valueArg.views as string[])].sort() };
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
/** Every namespace, document count, and index name must match exactly. */
|
|
306
|
+
export const compareLegacyMongoInventories = (
|
|
307
|
+
sourceArg: TLegacyMongoInventory,
|
|
308
|
+
targetArg: TLegacyMongoInventory,
|
|
309
|
+
): string[] => {
|
|
310
|
+
const differences: string[] = [];
|
|
311
|
+
const namespaces = [...new Set([...Object.keys(sourceArg), ...Object.keys(targetArg)])].sort();
|
|
312
|
+
for (const namespace of namespaces) {
|
|
313
|
+
const source = sourceArg[namespace];
|
|
314
|
+
const target = targetArg[namespace];
|
|
315
|
+
if (!source) {
|
|
316
|
+
differences.push(`${namespace}: present only in NoSQLDB`);
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
if (!target) {
|
|
320
|
+
differences.push(`${namespace}: missing in NoSQLDB`);
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
if (source.count !== target.count) {
|
|
324
|
+
differences.push(
|
|
325
|
+
`${namespace}: ${source.count} document(s) in MongoDB, ${target.count} in NoSQLDB`,
|
|
326
|
+
);
|
|
327
|
+
}
|
|
328
|
+
if (source.indexes.join('\u0000') !== target.indexes.join('\u0000')) {
|
|
329
|
+
differences.push(
|
|
330
|
+
`${namespace}: indexes [${source.indexes.join(', ')}] in MongoDB, [${target.indexes.join(', ')}] in NoSQLDB`,
|
|
331
|
+
);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
return differences;
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
/** Shell step shared by the tools: write the URI into a private config file, never argv. */
|
|
338
|
+
const writeToolConfig =
|
|
339
|
+
'umask 077; printf "uri: \'%s\'\\n" "$GITZONE_MONGODB_URI" > /tmp/gitzone-mongodb-tools.yaml';
|
|
340
|
+
|
|
341
|
+
export const legacyMongoDumpCommand = [
|
|
342
|
+
'-c',
|
|
343
|
+
`set -eu; ${writeToolConfig}; ` +
|
|
344
|
+
`mongodump --config /tmp/gitzone-mongodb-tools.yaml --archive=${toolContainerPath}/dump.archive ` +
|
|
345
|
+
`> ${toolContainerPath}/dump.log 2>&1`,
|
|
346
|
+
];
|
|
347
|
+
|
|
348
|
+
export const legacyMongoRestoreCommand = [
|
|
349
|
+
'-c',
|
|
350
|
+
`set -eu; ${writeToolConfig}; ` +
|
|
351
|
+
`mongorestore --config /tmp/gitzone-mongodb-tools.yaml --archive=${toolContainerPath}/dump.archive ` +
|
|
352
|
+
`--nsExclude 'admin.*' --nsExclude 'config.*' --nsExclude 'local.*' --stopOnError ` +
|
|
353
|
+
`> ${toolContainerPath}/restore.log 2>&1`,
|
|
354
|
+
];
|
|
355
|
+
|
|
356
|
+
export const getLegacyMongoInventoryCommand = (sideArg: 'source' | 'target'): string[] => [
|
|
357
|
+
'-c',
|
|
358
|
+
`set -eu; mongosh --nodb --quiet --norc ${toolContainerPath}/inventory.js ` +
|
|
359
|
+
`> ${toolContainerPath}/${sideArg}-inventory.log 2>&1`,
|
|
360
|
+
];
|
|
361
|
+
|
|
362
|
+
export const legacyMongoTargetReadinessCommand = [
|
|
363
|
+
'-c',
|
|
364
|
+
`set -eu; mongosh --nodb --quiet --norc ${toolContainerPath}/readiness.js ` +
|
|
365
|
+
`> ${toolContainerPath}/target-readiness.log 2>&1`,
|
|
366
|
+
];
|
|
367
|
+
|
|
368
|
+
/** Connect loop shared by the helper scripts; the URI arrives by environment only. */
|
|
369
|
+
const connectScript = (timeoutMsArg: number): string => `
|
|
370
|
+
const deadline = Date.now() + ${timeoutMsArg};
|
|
371
|
+
let client;
|
|
372
|
+
for (;;) {
|
|
373
|
+
try {
|
|
374
|
+
client = new Mongo(process.env.GITZONE_MONGODB_URI);
|
|
375
|
+
const reply = client.getDB('admin').runCommand({ ping: 1 });
|
|
376
|
+
if (!reply || Number(reply.ok.toString()) !== 1) { throw new Error('ping failed: ' + JSON.stringify(reply)); }
|
|
377
|
+
break;
|
|
378
|
+
} catch (error) {
|
|
379
|
+
if (Date.now() > deadline) { throw error; }
|
|
380
|
+
sleep(1000);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
`;
|
|
384
|
+
|
|
385
|
+
/** mongosh script that waits until the NoSQLDB container answers an authenticated ping. */
|
|
386
|
+
export const legacyMongoReadinessScript = `${connectScript(targetReadinessTimeoutMs)}
|
|
387
|
+
print('ready');
|
|
388
|
+
`;
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* mongosh script listing every non-system namespace with its document count and
|
|
392
|
+
* index names, plus every view. Numbers are normalised: MongoDB answers `count`
|
|
393
|
+
* as int32 while NoSQLDB answers int64. Views are listed, not counted:
|
|
394
|
+
* `listIndexes` fails on a view, and NoSQLDB cannot hold one.
|
|
395
|
+
*/
|
|
396
|
+
export const legacyMongoInventoryScript = `${connectScript(60_000)}
|
|
397
|
+
const toNumber = (value) => (typeof value === 'number' ? value : Number(value.toString()));
|
|
398
|
+
const assertOk = (reply, what) => {
|
|
399
|
+
if (!reply || toNumber(reply.ok) !== 1) { throw new Error(what + ' failed: ' + JSON.stringify(reply)); }
|
|
400
|
+
return reply;
|
|
401
|
+
};
|
|
402
|
+
const readCursor = (db, reply, collection) => {
|
|
403
|
+
const documents = [...reply.cursor.firstBatch];
|
|
404
|
+
let cursorId = reply.cursor.id;
|
|
405
|
+
while (cursorId && cursorId.toString() !== '0') {
|
|
406
|
+
const next = assertOk(db.runCommand({ getMore: cursorId, collection }), 'getMore');
|
|
407
|
+
documents.push(...next.cursor.nextBatch);
|
|
408
|
+
cursorId = next.cursor.id;
|
|
409
|
+
}
|
|
410
|
+
return documents;
|
|
411
|
+
};
|
|
412
|
+
const collections = {};
|
|
413
|
+
const views = [];
|
|
414
|
+
const databases = assertOk(client.getDB('admin').runCommand({ listDatabases: 1, nameOnly: true }), 'listDatabases').databases;
|
|
415
|
+
for (const database of databases) {
|
|
416
|
+
if (['admin', 'config', 'local'].includes(database.name)) { continue; }
|
|
417
|
+
const db = client.getDB(database.name);
|
|
418
|
+
const entries = readCursor(db, assertOk(db.runCommand({ listCollections: 1, nameOnly: true }), 'listCollections'), '$cmd.listCollections');
|
|
419
|
+
for (const entry of entries) {
|
|
420
|
+
if (entry.name.startsWith('system.')) { continue; }
|
|
421
|
+
const namespace = database.name + '.' + entry.name;
|
|
422
|
+
if (entry.type === 'view') { views.push(namespace); continue; }
|
|
423
|
+
const count = toNumber(assertOk(db.runCommand({ count: entry.name }), 'count').n);
|
|
424
|
+
const indexes = readCursor(db, assertOk(db.runCommand({ listIndexes: entry.name }), 'listIndexes'), entry.name)
|
|
425
|
+
.map((index) => index.name)
|
|
426
|
+
.sort();
|
|
427
|
+
collections[namespace] = { count, indexes };
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
require('fs').writeFileSync(process.env.GITZONE_INVENTORY_FILE, JSON.stringify({ collections, views }));
|
|
431
|
+
`;
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Migrates a project's legacy mongod data into its NoSQLDB service.
|
|
435
|
+
*
|
|
436
|
+
* Freeze first: the legacy container is stopped before anything reads its
|
|
437
|
+
* data, so no application can write after the snapshot is taken. Inventory
|
|
438
|
+
* and dump come from a temporary, network-less mongod started from the legacy
|
|
439
|
+
* container's own image on the same bind mount; only that mongod opens the
|
|
440
|
+
* legacy data, and GitZone never deletes it. The restore goes into a NoSQLDB
|
|
441
|
+
* container no application can reach, which is published only after the
|
|
442
|
+
* verification committed. Every step is recorded in a journal before it
|
|
443
|
+
* mutates anything, and any failure before verification rolls back to the
|
|
444
|
+
* exact pre-migration runtime state.
|
|
445
|
+
*/
|
|
446
|
+
export class LegacyMongoMigration {
|
|
447
|
+
private readonly projectPath: string;
|
|
448
|
+
|
|
449
|
+
constructor(private readonly host: ILegacyMongoMigrationHost) {
|
|
450
|
+
this.projectPath = plugins.path.resolve(host.projectPath);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
private get journalPath(): string {
|
|
454
|
+
return getLegacyMongoMigrationJournalPath(this.projectPath);
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
private get legacyDataPath(): string {
|
|
458
|
+
return getLegacyMongoDataDirectory(this.projectPath);
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
private get targetDataPath(): string {
|
|
462
|
+
return getServiceDataDirectory(this.projectPath, 'mongodb');
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
private timeout(timeoutMsArg = queryTimeoutMs): () => IDockerExecArgvOptions {
|
|
466
|
+
return () => ({ timeoutMs: timeoutMsArg });
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
public async readJournal(): Promise<ILegacyMongoMigrationJournal | undefined> {
|
|
470
|
+
let content: string;
|
|
471
|
+
try {
|
|
472
|
+
content = await plugins.fs.readFile(this.journalPath, 'utf8');
|
|
473
|
+
} catch (error) {
|
|
474
|
+
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
|
475
|
+
return undefined;
|
|
476
|
+
}
|
|
477
|
+
throw error;
|
|
478
|
+
}
|
|
479
|
+
let parsed: unknown;
|
|
480
|
+
try {
|
|
481
|
+
parsed = JSON.parse(content);
|
|
482
|
+
} catch {
|
|
483
|
+
throw new Error(`Invalid MongoDB migration journal: ${this.journalPath}`);
|
|
484
|
+
}
|
|
485
|
+
return parseLegacyMongoMigrationJournal(parsed, this.projectPath);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private async writeJournal(journalArg: ILegacyMongoMigrationJournal): Promise<void> {
|
|
489
|
+
const tempPath = `${this.journalPath}.tmp-${process.pid}-${plugins.crypto
|
|
490
|
+
.randomBytes(12)
|
|
491
|
+
.toString('hex')}`;
|
|
492
|
+
try {
|
|
493
|
+
await plugins.fs.writeFile(tempPath, `${JSON.stringify(journalArg, null, 2)}\n`, {
|
|
494
|
+
encoding: 'utf8',
|
|
495
|
+
flag: 'wx',
|
|
496
|
+
mode: 0o600,
|
|
497
|
+
});
|
|
498
|
+
await plugins.fs.rename(tempPath, this.journalPath);
|
|
499
|
+
} catch (error) {
|
|
500
|
+
await plugins.fs.rm(tempPath, { force: true }).catch(() => {});
|
|
501
|
+
throw new Error(`Could not persist MongoDB migration journal: ${this.journalPath}`, {
|
|
502
|
+
cause: error,
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
/**
|
|
508
|
+
* Resolve interrupted work, then migrate legacy state when it is proven owned.
|
|
509
|
+
* Throws before any Docker mutation when legacy state exists but ownership
|
|
510
|
+
* cannot be proven. Callers hold the project's `mongodb` data lock.
|
|
511
|
+
*/
|
|
512
|
+
public async run(): Promise<'none' | 'migrated'> {
|
|
513
|
+
const journal = await this.readJournal();
|
|
514
|
+
if (journal?.state === 'running') {
|
|
515
|
+
this.host.log('note', ' Rolling back an interrupted MongoDB migration...');
|
|
516
|
+
await this.rollback(journal);
|
|
517
|
+
} else if (journal?.state === 'committing') {
|
|
518
|
+
this.host.log('note', ' Completing a verified MongoDB migration...');
|
|
519
|
+
await this.commit(journal);
|
|
520
|
+
return 'migrated';
|
|
521
|
+
} else if (journal?.state === 'completed') {
|
|
522
|
+
return 'none';
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
const assessment = await this.assess();
|
|
526
|
+
if (assessment.kind === 'none') {
|
|
527
|
+
await this.applyFollowUp(assessment.followUp);
|
|
528
|
+
return 'none';
|
|
529
|
+
}
|
|
530
|
+
if (assessment.kind === 'blocked') {
|
|
531
|
+
throw new Error(
|
|
532
|
+
`Legacy MongoDB state blocks NoSQLDB startup: ${assessment.evidence.join(', ')}. ` +
|
|
533
|
+
`${assessment.reason} GitZone did not start, stop, remove, or modify anything. ` +
|
|
534
|
+
'Recovery: (1) stop the legacy container and export every required database with ' +
|
|
535
|
+
'`mongodump` from the legacy mongod; (2) move `.nogit/mongodata` and the legacy container ' +
|
|
536
|
+
'out of the way (rename them) and keep them as a backup; (3) run `gitzone services start mongo`; ' +
|
|
537
|
+
'(4) import the dump with `mongorestore` and verify every collection before disposing of the backup.',
|
|
538
|
+
);
|
|
539
|
+
}
|
|
540
|
+
await this.migrate(assessment.source);
|
|
541
|
+
return 'migrated';
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
private async applyFollowUp(followUpArg: TLegacyMongoFollowUp | undefined): Promise<void> {
|
|
545
|
+
if (followUpArg?.kind === 'clear-registry') {
|
|
546
|
+
// The recorded legacy container is proven absent and left no data.
|
|
547
|
+
await this.host.clearRegistryLegacyMongoContainer(this.projectPath, followUpArg.containerName);
|
|
548
|
+
} else if (followUpArg?.kind === 'record-empty-legacy-data') {
|
|
549
|
+
// Nothing was ever written; the empty directory stays as evidence.
|
|
550
|
+
await recordLegacyServiceDataDirectory(this.projectPath, 'mongo');
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
/**
|
|
555
|
+
* Legacy state that `run()` would act on. A journal means the migration
|
|
556
|
+
* already owns the state (resume, rollback, or a completed record), so only
|
|
557
|
+
* a project without one is classified. Read-only, like `assess()`.
|
|
558
|
+
*/
|
|
559
|
+
public async assessPending(): Promise<TLegacyMongoAssessment> {
|
|
560
|
+
if (await this.readJournal()) {
|
|
561
|
+
return { kind: 'none' };
|
|
562
|
+
}
|
|
563
|
+
return this.assess();
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/**
|
|
567
|
+
* Classify legacy state without changing anything: no Docker mutation, no
|
|
568
|
+
* registry or marker write. Bookkeeping a `none` result implies is returned
|
|
569
|
+
* as `followUp` and applied only by `run()`, under the data lock.
|
|
570
|
+
*/
|
|
571
|
+
public async assess(): Promise<TLegacyMongoAssessment> {
|
|
572
|
+
const names = this.host.containerNames;
|
|
573
|
+
const projects = await this.host.getRegistryProjects();
|
|
574
|
+
const registryProject = Object.entries(projects).find(
|
|
575
|
+
([registeredPath]) => plugins.path.resolve(registeredPath) === this.projectPath,
|
|
576
|
+
)?.[1];
|
|
577
|
+
const registryName = registryProject?.legacy?.mongoContainer;
|
|
578
|
+
const candidateNames = [...new Set([names.legacyMongo, ...(registryName ? [registryName] : [])])];
|
|
579
|
+
const existing: Array<{ name: string; id: string }> = [];
|
|
580
|
+
for (const name of candidateNames) {
|
|
581
|
+
const id = await this.host.discoverContainerIdByExactName(name, this.timeout());
|
|
582
|
+
if (id) {
|
|
583
|
+
existing.push({ name, id });
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
const dataExists = await plugins.smartfs.directory(this.legacyDataPath).exists();
|
|
587
|
+
|
|
588
|
+
if (existing.length === 0 && !dataExists) {
|
|
589
|
+
return registryName
|
|
590
|
+
? { kind: 'none', followUp: { kind: 'clear-registry', containerName: registryName } }
|
|
591
|
+
: { kind: 'none' };
|
|
592
|
+
}
|
|
593
|
+
|
|
594
|
+
const evidence = [
|
|
595
|
+
...existing.map((entry) => `container ${entry.name}`),
|
|
596
|
+
...(dataExists ? [`data directory ${this.legacyDataPath}`] : []),
|
|
597
|
+
];
|
|
598
|
+
const blocked = (reasonArg: string): TLegacyMongoAssessment => ({
|
|
599
|
+
kind: 'blocked',
|
|
600
|
+
evidence,
|
|
601
|
+
reason: reasonArg,
|
|
602
|
+
});
|
|
603
|
+
if (existing.length > 1) {
|
|
604
|
+
return blocked('More than one legacy MongoDB container exists; ownership is ambiguous.');
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
const targetContainerId = await this.host.discoverContainerIdByExactName(
|
|
608
|
+
names.nosqldb,
|
|
609
|
+
this.timeout(),
|
|
610
|
+
);
|
|
611
|
+
if (
|
|
612
|
+
targetContainerId ||
|
|
613
|
+
(await plugins.smartfs.directory(this.targetDataPath).exists())
|
|
614
|
+
) {
|
|
615
|
+
return blocked(
|
|
616
|
+
`NoSQLDB state already exists (${names.nosqldb} or ${this.targetDataPath}); ` +
|
|
617
|
+
'GitZone will not merge two databases.',
|
|
618
|
+
);
|
|
619
|
+
}
|
|
620
|
+
if (!dataExists) {
|
|
621
|
+
return blocked(`The legacy container exists but its data directory ${this.legacyDataPath} is missing.`);
|
|
622
|
+
}
|
|
623
|
+
const dataEntries = await plugins.fs.readdir(this.legacyDataPath);
|
|
624
|
+
|
|
625
|
+
if (existing.length === 0) {
|
|
626
|
+
const marker = await readServiceDataMarker(this.projectPath);
|
|
627
|
+
const dataProven =
|
|
628
|
+
markerRecordsLegacyDataDirectory(marker, this.projectPath, 'mongo') || !!registryName;
|
|
629
|
+
if (!dataProven) {
|
|
630
|
+
return blocked(
|
|
631
|
+
'Neither the service marker nor a legacy MongoDB registry claim proves GitZone created this data.',
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
if (dataEntries.length === 0) {
|
|
635
|
+
return { kind: 'none', followUp: { kind: 'record-empty-legacy-data' } };
|
|
636
|
+
}
|
|
637
|
+
return { kind: 'migratable', source: { dataPath: this.legacyDataPath } };
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
const container = await this.inspectLegacyContainer(existing[0].id, existing[0].name);
|
|
641
|
+
const labelOwned = hasExactServiceContainerLabels(
|
|
642
|
+
container.labels,
|
|
643
|
+
getLegacyMongoContainerLabels(this.projectPath),
|
|
644
|
+
);
|
|
645
|
+
const unlabeled = !Object.keys(container.labels).some((keyArg) => keyArg.startsWith('git.zone.'));
|
|
646
|
+
const registryClaims = Object.entries(projects).filter(
|
|
647
|
+
([, projectArg]) =>
|
|
648
|
+
projectArg.legacy?.mongoContainer === container.name ||
|
|
649
|
+
Object.values(projectArg.containers).includes(container.name),
|
|
650
|
+
);
|
|
651
|
+
const registryOwned =
|
|
652
|
+
unlabeled &&
|
|
653
|
+
registryClaims.length === 1 &&
|
|
654
|
+
plugins.path.resolve(registryClaims[0][0]) === this.projectPath &&
|
|
655
|
+
registryClaims[0][1].legacy?.mongoContainer === container.name;
|
|
656
|
+
if (!labelOwned && !registryOwned) {
|
|
657
|
+
return blocked(`Ownership of container ${container.name} is not proven for this project.`);
|
|
658
|
+
}
|
|
659
|
+
if (container.image !== legacyMongoImage) {
|
|
660
|
+
return blocked(`Container ${container.name} runs ${container.image}, not ${legacyMongoImage}.`);
|
|
661
|
+
}
|
|
662
|
+
if (!container.mountsProven) {
|
|
663
|
+
return blocked(
|
|
664
|
+
`Container ${container.name} does not mount exactly ${this.legacyDataPath} at /data/db.`,
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
return {
|
|
668
|
+
kind: 'migratable',
|
|
669
|
+
source: {
|
|
670
|
+
container: {
|
|
671
|
+
id: container.id,
|
|
672
|
+
name: container.name,
|
|
673
|
+
running: container.running,
|
|
674
|
+
imageId: container.imageId,
|
|
675
|
+
},
|
|
676
|
+
dataPath: this.legacyDataPath,
|
|
677
|
+
},
|
|
678
|
+
};
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
private async inspectLegacyContainer(
|
|
682
|
+
idArg: string,
|
|
683
|
+
expectedNameArg: string,
|
|
684
|
+
): Promise<{
|
|
685
|
+
id: string;
|
|
686
|
+
name: string;
|
|
687
|
+
running: boolean;
|
|
688
|
+
image: string;
|
|
689
|
+
imageId: string;
|
|
690
|
+
labels: Record<string, string>;
|
|
691
|
+
mountsProven: boolean;
|
|
692
|
+
}> {
|
|
693
|
+
const result = await this.host.docker.execArgv(['inspect', idArg], { timeoutMs: queryTimeoutMs });
|
|
694
|
+
const invalid = new Error(`Docker returned an invalid legacy MongoDB inspection for ${expectedNameArg}.`);
|
|
695
|
+
let parsed: unknown;
|
|
696
|
+
try {
|
|
697
|
+
parsed = JSON.parse(result.stdout);
|
|
698
|
+
} catch {
|
|
699
|
+
throw invalid;
|
|
700
|
+
}
|
|
701
|
+
if (!Array.isArray(parsed) || parsed.length !== 1 || !isPlainObject(parsed[0])) {
|
|
702
|
+
throw invalid;
|
|
703
|
+
}
|
|
704
|
+
const inspected = parsed[0];
|
|
705
|
+
const config = inspected.Config;
|
|
706
|
+
const state = inspected.State;
|
|
707
|
+
const name = typeof inspected.Name === 'string' ? inspected.Name.replace(/^\//u, '') : '';
|
|
708
|
+
if (
|
|
709
|
+
inspected.Id !== idArg ||
|
|
710
|
+
name !== expectedNameArg ||
|
|
711
|
+
!isImageId(inspected.Image) ||
|
|
712
|
+
!isPlainObject(config) ||
|
|
713
|
+
!isPlainObject(state) ||
|
|
714
|
+
typeof state.Running !== 'boolean' ||
|
|
715
|
+
typeof config.Image !== 'string' ||
|
|
716
|
+
(config.Labels !== null && config.Labels !== undefined && !isPlainObject(config.Labels)) ||
|
|
717
|
+
!Array.isArray(inspected.Mounts)
|
|
718
|
+
) {
|
|
719
|
+
throw invalid;
|
|
720
|
+
}
|
|
721
|
+
const labels: Record<string, string> = {};
|
|
722
|
+
for (const [key, value] of Object.entries(config.Labels || {})) {
|
|
723
|
+
if (typeof value !== 'string') {
|
|
724
|
+
throw invalid;
|
|
725
|
+
}
|
|
726
|
+
labels[key] = value;
|
|
727
|
+
}
|
|
728
|
+
const mounts = inspected.Mounts as unknown[];
|
|
729
|
+
const bindMounts = mounts.filter((mountArg) => isPlainObject(mountArg) && mountArg.Type === 'bind');
|
|
730
|
+
const otherMountsAreAnonymousConfigVolumes = mounts.every(
|
|
731
|
+
(mountArg) =>
|
|
732
|
+
isPlainObject(mountArg) &&
|
|
733
|
+
(mountArg.Type === 'bind' ||
|
|
734
|
+
(mountArg.Type === 'volume' && mountArg.Destination === '/data/configdb')),
|
|
735
|
+
);
|
|
736
|
+
const mountsProven =
|
|
737
|
+
otherMountsAreAnonymousConfigVolumes &&
|
|
738
|
+
bindMounts.length === 1 &&
|
|
739
|
+
isPlainObject(bindMounts[0]) &&
|
|
740
|
+
typeof bindMounts[0].Source === 'string' &&
|
|
741
|
+
plugins.path.resolve(bindMounts[0].Source) === this.legacyDataPath &&
|
|
742
|
+
bindMounts[0].Destination === '/data/db';
|
|
743
|
+
return {
|
|
744
|
+
id: idArg,
|
|
745
|
+
name,
|
|
746
|
+
running: state.Running,
|
|
747
|
+
image: config.Image,
|
|
748
|
+
imageId: inspected.Image,
|
|
749
|
+
labels,
|
|
750
|
+
mountsProven,
|
|
751
|
+
};
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
private buildToolUri(portArg: string, authEnabledArg: boolean): string {
|
|
755
|
+
const config = this.host.config;
|
|
756
|
+
// Single quotes are the YAML delimiter of the tools' config file.
|
|
757
|
+
const encode = (valueArg: string): string =>
|
|
758
|
+
encodeURIComponent(valueArg).replace(/'/gu, '%27');
|
|
759
|
+
const credentials = authEnabledArg
|
|
760
|
+
? `${encode(config.MONGODB_USER)}:${encode(config.MONGODB_PASS)}@`
|
|
761
|
+
: '';
|
|
762
|
+
const query = authEnabledArg ? 'authSource=admin&directConnection=true' : 'directConnection=true';
|
|
763
|
+
return `mongodb://${credentials}127.0.0.1:${portArg}/?${query}`;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
private helperLabels(evidenceDirectoryArg: string): Record<string, string> {
|
|
767
|
+
return {
|
|
768
|
+
'git.zone.tool': serviceToolLabel,
|
|
769
|
+
'git.zone.service': 'mongodb',
|
|
770
|
+
'git.zone.project-path': this.projectPath,
|
|
771
|
+
'git.zone.data-path': plugins.path.resolve(evidenceDirectoryArg),
|
|
772
|
+
'git.zone.safe-to-prune': 'false',
|
|
773
|
+
[serviceOperationLabel]: serviceBindOperation,
|
|
774
|
+
};
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
private async runTool(optionsArg: {
|
|
778
|
+
step: string;
|
|
779
|
+
logFile: string;
|
|
780
|
+
evidenceDirectory: string;
|
|
781
|
+
networkContainerId: string;
|
|
782
|
+
command: string[];
|
|
783
|
+
environment: Record<string, string>;
|
|
784
|
+
}): Promise<void> {
|
|
785
|
+
try {
|
|
786
|
+
await this.host.docker.runFencedBindOperation({
|
|
787
|
+
image: legacyMongoImage,
|
|
788
|
+
hostPath: optionsArg.evidenceDirectory,
|
|
789
|
+
containerPath: toolContainerPath,
|
|
790
|
+
entrypoint: '/bin/sh',
|
|
791
|
+
command: optionsArg.command,
|
|
792
|
+
labels: this.helperLabels(optionsArg.evidenceDirectory),
|
|
793
|
+
user: this.host.helperUser,
|
|
794
|
+
network: `container:${optionsArg.networkContainerId}`,
|
|
795
|
+
environment: { HOME: '/tmp', ...optionsArg.environment },
|
|
796
|
+
executionOptions: this.timeout(toolTimeoutMs),
|
|
797
|
+
rollbackTimeoutMs: 10_000,
|
|
798
|
+
});
|
|
799
|
+
} catch (error) {
|
|
800
|
+
const logTail = await this.readRedactedLogTail(
|
|
801
|
+
plugins.path.join(optionsArg.evidenceDirectory, optionsArg.logFile),
|
|
802
|
+
);
|
|
803
|
+
throw new Error(
|
|
804
|
+
`MongoDB migration ${optionsArg.step} failed.` +
|
|
805
|
+
(logTail ? ` Tool output (redacted):\n${logTail}` : ''),
|
|
806
|
+
{ cause: error },
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
private redact(contentArg: string): string {
|
|
812
|
+
const secrets = [this.host.config.MONGODB_PASS, this.host.config.MONGODB_ADMIN_PASSWORD]
|
|
813
|
+
.filter((secretArg) => typeof secretArg === 'string' && secretArg.length > 0)
|
|
814
|
+
.flatMap((secretArg) => [secretArg, encodeURIComponent(secretArg)])
|
|
815
|
+
.sort((secretA, secretB) => secretB.length - secretA.length);
|
|
816
|
+
let redacted = contentArg;
|
|
817
|
+
for (const secret of secrets) {
|
|
818
|
+
redacted = redacted.split(secret).join('***');
|
|
819
|
+
}
|
|
820
|
+
return redacted.trim().split(/\r?\n/u).slice(-20).join('\n');
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
private async readRedactedLogTail(logPathArg: string): Promise<string> {
|
|
824
|
+
try {
|
|
825
|
+
return this.redact(await plugins.fs.readFile(logPathArg, 'utf8'));
|
|
826
|
+
} catch {
|
|
827
|
+
return '';
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
private async readInventory(pathArg: string): Promise<ILegacyMongoInventoryReport> {
|
|
832
|
+
let content: string;
|
|
833
|
+
try {
|
|
834
|
+
content = await plugins.fs.readFile(pathArg, 'utf8');
|
|
835
|
+
} catch (error) {
|
|
836
|
+
throw new Error(`MongoDB migration inventory is missing: ${pathArg}`, { cause: error });
|
|
837
|
+
}
|
|
838
|
+
try {
|
|
839
|
+
return parseLegacyMongoInventory(JSON.parse(content));
|
|
840
|
+
} catch (error) {
|
|
841
|
+
throw new Error(`MongoDB migration inventory is invalid: ${pathArg}`, { cause: error });
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
private async ensureImagePresent(imageArg: string): Promise<void> {
|
|
846
|
+
try {
|
|
847
|
+
await this.host.docker.execArgv(['image', 'inspect', imageArg], { timeoutMs: queryTimeoutMs });
|
|
848
|
+
} catch {
|
|
849
|
+
this.host.log('note', ` Pulling ${imageArg}...`);
|
|
850
|
+
await this.host.docker.execArgv(['pull', imageArg], { timeoutMs: toolTimeoutMs });
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
|
|
854
|
+
private async resolveImageId(imageArg: string): Promise<string> {
|
|
855
|
+
const result = await this.host.docker.execArgv(
|
|
856
|
+
['image', 'inspect', '--format', '{{.Id}}', imageArg],
|
|
857
|
+
{ timeoutMs: queryTimeoutMs },
|
|
858
|
+
);
|
|
859
|
+
const imageId = result.stdout.trim();
|
|
860
|
+
if (!isImageId(imageId)) {
|
|
861
|
+
throw new Error(`Docker returned an invalid image id for ${imageArg}.`);
|
|
862
|
+
}
|
|
863
|
+
return imageId;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
private async migrate(sourceArg: ILegacyMongoSource): Promise<void> {
|
|
867
|
+
const config = this.host.config;
|
|
868
|
+
this.host.log('note', ' Legacy MongoDB data found; migrating it to NoSQLDB...');
|
|
869
|
+
// Every image must be present before the legacy database is touched.
|
|
870
|
+
await this.ensureImagePresent(noSqlDbImage);
|
|
871
|
+
await this.ensureImagePresent(legacyMongoImage);
|
|
872
|
+
// The reader opens the data with the exact image the legacy mongod ran.
|
|
873
|
+
const readerImageId = sourceArg.container?.imageId ?? (await this.resolveImageId(legacyMongoImage));
|
|
874
|
+
await recordLegacyServiceDataDirectory(this.projectPath, 'mongo');
|
|
875
|
+
|
|
876
|
+
const token = plugins.crypto.randomBytes(8).toString('hex');
|
|
877
|
+
const timestamp = new Date().toISOString().replace(/[-:]/gu, '').replace(/\.\d+Z$/u, 'Z');
|
|
878
|
+
const evidenceDirectory = plugins.path.join(
|
|
879
|
+
getLegacyMongoMigrationEvidenceRoot(this.projectPath),
|
|
880
|
+
`${timestamp}-${token}`,
|
|
881
|
+
);
|
|
882
|
+
await plugins.fs.mkdir(evidenceDirectory, { recursive: true, mode: 0o700 });
|
|
883
|
+
for (const [fileName, script] of [
|
|
884
|
+
['inventory.js', legacyMongoInventoryScript],
|
|
885
|
+
['readiness.js', legacyMongoReadinessScript],
|
|
886
|
+
]) {
|
|
887
|
+
await plugins.fs.writeFile(plugins.path.join(evidenceDirectory, fileName), script, {
|
|
888
|
+
encoding: 'utf8',
|
|
889
|
+
mode: 0o600,
|
|
890
|
+
});
|
|
891
|
+
}
|
|
892
|
+
const journal: ILegacyMongoMigrationJournal = {
|
|
893
|
+
owner: serviceDataOwner,
|
|
894
|
+
kind: 'mongodb-nosqldb-migration',
|
|
895
|
+
projectPath: this.projectPath,
|
|
896
|
+
token,
|
|
897
|
+
createdAt: new Date().toISOString(),
|
|
898
|
+
state: 'running',
|
|
899
|
+
evidenceDirectory,
|
|
900
|
+
...(sourceArg.container
|
|
901
|
+
? {
|
|
902
|
+
legacyContainer: {
|
|
903
|
+
id: sourceArg.container.id,
|
|
904
|
+
name: sourceArg.container.name,
|
|
905
|
+
wasRunning: sourceArg.container.running,
|
|
906
|
+
},
|
|
907
|
+
}
|
|
908
|
+
: {}),
|
|
909
|
+
targetCreated: false,
|
|
910
|
+
};
|
|
911
|
+
await this.writeJournal(journal);
|
|
912
|
+
|
|
913
|
+
try {
|
|
914
|
+
// 1. Freeze: stop the legacy mongod before anything reads its data, so
|
|
915
|
+
// no application write can land after the snapshot.
|
|
916
|
+
if (sourceArg.container?.running) {
|
|
917
|
+
this.host.log('note', ` Stopping ${sourceArg.container.name} so nothing writes during the migration...`);
|
|
918
|
+
await this.stopMongodContainer(sourceArg.container.id);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// 2. Inventory and dump from a temporary network-less, read-only reader.
|
|
922
|
+
const readerId = await this.startReader(readerImageId, token);
|
|
923
|
+
const sourceUri = this.buildToolUri(readerWirePort, false);
|
|
924
|
+
this.host.log('note', ' Dumping legacy MongoDB data...');
|
|
925
|
+
await this.runTool({
|
|
926
|
+
step: 'source inventory',
|
|
927
|
+
logFile: 'source-inventory.log',
|
|
928
|
+
evidenceDirectory,
|
|
929
|
+
networkContainerId: readerId,
|
|
930
|
+
command: getLegacyMongoInventoryCommand('source'),
|
|
931
|
+
environment: {
|
|
932
|
+
GITZONE_MONGODB_URI: sourceUri,
|
|
933
|
+
GITZONE_INVENTORY_FILE: `${toolContainerPath}/source-inventory.json`,
|
|
934
|
+
},
|
|
935
|
+
});
|
|
936
|
+
const sourceInventory = await this.readInventory(
|
|
937
|
+
plugins.path.join(evidenceDirectory, 'source-inventory.json'),
|
|
938
|
+
);
|
|
939
|
+
if (sourceInventory.views.length > 0) {
|
|
940
|
+
throw new Error(
|
|
941
|
+
`MongoDB views cannot be migrated: NoSQLDB does not support general views. ` +
|
|
942
|
+
`Views found: ${sourceInventory.views.join(', ')}. Drop them in the legacy database ` +
|
|
943
|
+
'(and recreate what they computed in application code), then start again.',
|
|
944
|
+
);
|
|
945
|
+
}
|
|
946
|
+
await this.runTool({
|
|
947
|
+
step: 'dump',
|
|
948
|
+
logFile: 'dump.log',
|
|
949
|
+
evidenceDirectory,
|
|
950
|
+
networkContainerId: readerId,
|
|
951
|
+
command: legacyMongoDumpCommand,
|
|
952
|
+
environment: { GITZONE_MONGODB_URI: sourceUri },
|
|
953
|
+
});
|
|
954
|
+
await this.removeReader(token);
|
|
955
|
+
|
|
956
|
+
// 3. Restore into a NoSQLDB container no application can reach.
|
|
957
|
+
journal.targetCreated = true;
|
|
958
|
+
await this.writeJournal(journal);
|
|
959
|
+
const target = await this.host.startMigrationNoSqlDb();
|
|
960
|
+
const targetUri = this.buildToolUri('27017', config.MONGODB_AUTH_ENABLED !== false);
|
|
961
|
+
await this.runTool({
|
|
962
|
+
step: 'NoSQLDB readiness',
|
|
963
|
+
logFile: 'target-readiness.log',
|
|
964
|
+
evidenceDirectory,
|
|
965
|
+
networkContainerId: target.id,
|
|
966
|
+
command: legacyMongoTargetReadinessCommand,
|
|
967
|
+
environment: { GITZONE_MONGODB_URI: targetUri },
|
|
968
|
+
});
|
|
969
|
+
this.host.log('note', ' Restoring into NoSQLDB...');
|
|
970
|
+
await this.runTool({
|
|
971
|
+
step: 'restore',
|
|
972
|
+
logFile: 'restore.log',
|
|
973
|
+
evidenceDirectory,
|
|
974
|
+
networkContainerId: target.id,
|
|
975
|
+
command: legacyMongoRestoreCommand,
|
|
976
|
+
environment: { GITZONE_MONGODB_URI: targetUri },
|
|
977
|
+
});
|
|
978
|
+
await this.runTool({
|
|
979
|
+
step: 'target inventory',
|
|
980
|
+
logFile: 'target-inventory.log',
|
|
981
|
+
evidenceDirectory,
|
|
982
|
+
networkContainerId: target.id,
|
|
983
|
+
command: getLegacyMongoInventoryCommand('target'),
|
|
984
|
+
environment: {
|
|
985
|
+
GITZONE_MONGODB_URI: targetUri,
|
|
986
|
+
GITZONE_INVENTORY_FILE: `${toolContainerPath}/target-inventory.json`,
|
|
987
|
+
},
|
|
988
|
+
});
|
|
989
|
+
|
|
990
|
+
// 4. Verify every namespace before anything becomes irreversible.
|
|
991
|
+
const targetInventory = await this.readInventory(
|
|
992
|
+
plugins.path.join(evidenceDirectory, 'target-inventory.json'),
|
|
993
|
+
);
|
|
994
|
+
const differences = compareLegacyMongoInventories(
|
|
995
|
+
sourceInventory.collections,
|
|
996
|
+
targetInventory.collections,
|
|
997
|
+
);
|
|
998
|
+
if (differences.length > 0) {
|
|
999
|
+
throw new Error(
|
|
1000
|
+
`MongoDB migration verification failed:\n ${differences.slice(0, 20).join('\n ')}` +
|
|
1001
|
+
(differences.length > 20 ? `\n ... and ${differences.length - 20} more` : ''),
|
|
1002
|
+
);
|
|
1003
|
+
}
|
|
1004
|
+
this.host.log(
|
|
1005
|
+
'ok',
|
|
1006
|
+
` Verified ${Object.keys(sourceInventory.collections).length} collection(s): document counts and indexes match ✓`,
|
|
1007
|
+
);
|
|
1008
|
+
// The restore-phase container stops before the journal leaves `running`:
|
|
1009
|
+
// past that point the published service may take application writes.
|
|
1010
|
+
await this.host.stopMigrationNoSqlDb();
|
|
1011
|
+
journal.state = 'committing';
|
|
1012
|
+
await this.writeJournal(journal);
|
|
1013
|
+
} catch (error) {
|
|
1014
|
+
const primaryError =
|
|
1015
|
+
error instanceof Error ? error : new Error('MongoDB migration failed with a non-Error rejection.');
|
|
1016
|
+
try {
|
|
1017
|
+
await this.rollback(journal);
|
|
1018
|
+
} catch (rollbackError) {
|
|
1019
|
+
throw new AggregateError(
|
|
1020
|
+
[primaryError, rollbackError],
|
|
1021
|
+
'MongoDB migration failed and its rollback could not be completed; ' +
|
|
1022
|
+
`the journal ${this.journalPath} is kept so the next start retries the rollback.`,
|
|
1023
|
+
);
|
|
1024
|
+
}
|
|
1025
|
+
throw new Error(
|
|
1026
|
+
`${primaryError.message}\nThe migration was rolled back: the legacy MongoDB container and ` +
|
|
1027
|
+
`${this.legacyDataPath} are unchanged. Evidence: ${evidenceDirectory}`,
|
|
1028
|
+
{ cause: primaryError },
|
|
1029
|
+
);
|
|
1030
|
+
}
|
|
1031
|
+
await this.commit(journal);
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
/** Stop a mongod container by id, giving it time for a clean shutdown. */
|
|
1035
|
+
private async stopMongodContainer(idArg: string): Promise<void> {
|
|
1036
|
+
await this.host.docker.execArgv(['stop', '-t', String(mongodStopGraceSeconds), idArg], {
|
|
1037
|
+
timeoutMs: mongodStopTimeoutMs,
|
|
1038
|
+
});
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
private async createReader(imageIdArg: string, tokenArg: string, oplogRecoveryArg: boolean): Promise<string> {
|
|
1042
|
+
const name = getLegacyMongoReaderName(this.host.containerNames.legacyMongo, tokenArg);
|
|
1043
|
+
const creation = await this.host.docker.createArgv(
|
|
1044
|
+
{
|
|
1045
|
+
name,
|
|
1046
|
+
image: imageIdArg,
|
|
1047
|
+
volumes: { [this.legacyDataPath]: '/data/db' },
|
|
1048
|
+
labels: getLegacyMongoReaderLabels(this.projectPath, tokenArg),
|
|
1049
|
+
network: 'none',
|
|
1050
|
+
restart: 'no',
|
|
1051
|
+
command: getLegacyMongoReaderCommand(oplogRecoveryArg),
|
|
1052
|
+
},
|
|
1053
|
+
{ timeoutMs: queryTimeoutMs },
|
|
1054
|
+
);
|
|
1055
|
+
const id = creation.stdout.trim();
|
|
1056
|
+
if (!isContainerId(id)) {
|
|
1057
|
+
throw new Error(`Docker returned an invalid id while creating ${name}.`);
|
|
1058
|
+
}
|
|
1059
|
+
await this.host.docker.execArgv(['start', id], { timeoutMs: queryTimeoutMs });
|
|
1060
|
+
return id;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
/**
|
|
1064
|
+
* Open the frozen legacy data with a temporary mongod and wait until it
|
|
1065
|
+
* answers. A data directory that never ran as a replica-set member has no
|
|
1066
|
+
* stable checkpoint and no oplog to replay; mongod refuses oplog recovery
|
|
1067
|
+
* for it (fatal assertion 31229), and it is opened as the plain standalone
|
|
1068
|
+
* it always was.
|
|
1069
|
+
*/
|
|
1070
|
+
private async startReader(imageIdArg: string, tokenArg: string): Promise<string> {
|
|
1071
|
+
const readerId = await this.createReader(imageIdArg, tokenArg, true);
|
|
1072
|
+
if ((await this.waitForReader(readerId, tokenArg)) === 'ready') {
|
|
1073
|
+
return readerId;
|
|
1074
|
+
}
|
|
1075
|
+
this.host.log('note', ' The legacy data never ran as a replica set; opening it as a standalone...');
|
|
1076
|
+
await this.removeReader(tokenArg);
|
|
1077
|
+
const standaloneId = await this.createReader(imageIdArg, tokenArg, false);
|
|
1078
|
+
if ((await this.waitForReader(standaloneId, tokenArg)) !== 'ready') {
|
|
1079
|
+
throw new Error('The temporary MongoDB reader refused the legacy data without oplog recovery.');
|
|
1080
|
+
}
|
|
1081
|
+
return standaloneId;
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
private async waitForReader(
|
|
1085
|
+
idArg: string,
|
|
1086
|
+
tokenArg: string,
|
|
1087
|
+
): Promise<'ready' | 'no-stable-checkpoint'> {
|
|
1088
|
+
const name = getLegacyMongoReaderName(this.host.containerNames.legacyMongo, tokenArg);
|
|
1089
|
+
const deadline = Date.now() + readerReadinessTimeoutMs;
|
|
1090
|
+
for (;;) {
|
|
1091
|
+
try {
|
|
1092
|
+
const ping = await this.host.docker.execArgv(
|
|
1093
|
+
['exec', idArg, 'mongosh', '--quiet', '--norc', '--eval', 'db.adminCommand({ ping: 1 }).ok'],
|
|
1094
|
+
{ timeoutMs: queryTimeoutMs },
|
|
1095
|
+
);
|
|
1096
|
+
if (ping.stdout.trim() === '1') {
|
|
1097
|
+
return 'ready';
|
|
1098
|
+
}
|
|
1099
|
+
} catch {
|
|
1100
|
+
// Not accepting connections yet, or no longer running.
|
|
1101
|
+
}
|
|
1102
|
+
const reader = await this.inspectReader(idArg, name, tokenArg);
|
|
1103
|
+
if (!reader.running) {
|
|
1104
|
+
const logs = await this.host.docker.execArgv(['logs', '--tail', '100', idArg], {
|
|
1105
|
+
timeoutMs: queryTimeoutMs,
|
|
1106
|
+
});
|
|
1107
|
+
const output = `${logs.stdout}\n${logs.stderr}`;
|
|
1108
|
+
if (output.includes(`"id":${noStableCheckpointLogId},`)) {
|
|
1109
|
+
return 'no-stable-checkpoint';
|
|
1110
|
+
}
|
|
1111
|
+
throw new Error(
|
|
1112
|
+
`The temporary MongoDB reader ${name} exited while opening the legacy data:\n${this.redact(output)}`,
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
if (Date.now() > deadline) {
|
|
1116
|
+
throw new Error(`The temporary MongoDB reader ${name} did not become ready in time.`);
|
|
1117
|
+
}
|
|
1118
|
+
await plugins.smartdelay.delayFor(readerPollIntervalMs);
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
|
|
1122
|
+
/** Inspect a reader and prove it is the one this migration run created. */
|
|
1123
|
+
private async inspectReader(
|
|
1124
|
+
idArg: string,
|
|
1125
|
+
nameArg: string,
|
|
1126
|
+
tokenArg: string,
|
|
1127
|
+
): Promise<{ running: boolean }> {
|
|
1128
|
+
const result = await this.host.docker.execArgv(['inspect', idArg], { timeoutMs: queryTimeoutMs });
|
|
1129
|
+
let parsed: unknown;
|
|
1130
|
+
try {
|
|
1131
|
+
parsed = JSON.parse(result.stdout);
|
|
1132
|
+
} catch {
|
|
1133
|
+
parsed = undefined;
|
|
1134
|
+
}
|
|
1135
|
+
const inspected = Array.isArray(parsed) && parsed.length === 1 ? parsed[0] : undefined;
|
|
1136
|
+
if (
|
|
1137
|
+
!isPlainObject(inspected) ||
|
|
1138
|
+
inspected.Id !== idArg ||
|
|
1139
|
+
inspected.Name !== `/${nameArg}` ||
|
|
1140
|
+
!isPlainObject(inspected.State) ||
|
|
1141
|
+
typeof inspected.State.Running !== 'boolean' ||
|
|
1142
|
+
!isPlainObject(inspected.Config) ||
|
|
1143
|
+
!isPlainObject(inspected.Config.Labels) ||
|
|
1144
|
+
!hasExactServiceContainerLabels(
|
|
1145
|
+
inspected.Config.Labels as Record<string, string>,
|
|
1146
|
+
getLegacyMongoReaderLabels(this.projectPath, tokenArg),
|
|
1147
|
+
)
|
|
1148
|
+
) {
|
|
1149
|
+
throw new Error(`Refusing to touch ${nameArg}: it is not this migration's MongoDB reader.`);
|
|
1150
|
+
}
|
|
1151
|
+
return { running: inspected.State.Running };
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/** Stop and remove this run's reader, if it exists; the legacy data stays in place. */
|
|
1155
|
+
private async removeReader(tokenArg: string): Promise<void> {
|
|
1156
|
+
const name = getLegacyMongoReaderName(this.host.containerNames.legacyMongo, tokenArg);
|
|
1157
|
+
const id = await this.host.discoverContainerIdByExactName(name, this.timeout());
|
|
1158
|
+
if (!id) {
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
const reader = await this.inspectReader(id, name, tokenArg);
|
|
1162
|
+
if (reader.running) {
|
|
1163
|
+
await this.stopMongodContainer(id);
|
|
1164
|
+
}
|
|
1165
|
+
// `-v` drops only the image's anonymous `/data/configdb` volume this reader created.
|
|
1166
|
+
await this.host.docker.execArgv(['rm', '-v', id], { timeoutMs: queryTimeoutMs });
|
|
1167
|
+
}
|
|
1168
|
+
|
|
1169
|
+
/** Inspect a container by id and prove it still is the journaled legacy container. */
|
|
1170
|
+
private async revalidateJournaledLegacyContainer(
|
|
1171
|
+
legacyArg: NonNullable<ILegacyMongoMigrationJournal['legacyContainer']>,
|
|
1172
|
+
): Promise<{ running: boolean } | undefined> {
|
|
1173
|
+
const id = await this.host.discoverContainerIdByExactName(legacyArg.name, this.timeout());
|
|
1174
|
+
if (!id) {
|
|
1175
|
+
return undefined;
|
|
1176
|
+
}
|
|
1177
|
+
if (id !== legacyArg.id) {
|
|
1178
|
+
throw new Error(
|
|
1179
|
+
`Refusing to touch ${legacyArg.name}: its immutable id changed since the migration started.`,
|
|
1180
|
+
);
|
|
1181
|
+
}
|
|
1182
|
+
const inspection = await this.inspectLegacyContainer(id, legacyArg.name);
|
|
1183
|
+
return { running: inspection.running };
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
private async removeMigrationTarget(): Promise<void> {
|
|
1187
|
+
const name = this.host.containerNames.nosqldb;
|
|
1188
|
+
const id = await this.host.discoverContainerIdByExactName(name, this.timeout());
|
|
1189
|
+
if (id) {
|
|
1190
|
+
const result = await this.host.docker.execArgv(['inspect', id], { timeoutMs: queryTimeoutMs });
|
|
1191
|
+
const parsed: unknown = JSON.parse(result.stdout);
|
|
1192
|
+
const labels =
|
|
1193
|
+
Array.isArray(parsed) &&
|
|
1194
|
+
isPlainObject(parsed[0]) &&
|
|
1195
|
+
isPlainObject(parsed[0].Config) &&
|
|
1196
|
+
isPlainObject(parsed[0].Config.Labels)
|
|
1197
|
+
? (parsed[0].Config.Labels as Record<string, string>)
|
|
1198
|
+
: {};
|
|
1199
|
+
const expected = getServiceContainerLabels({
|
|
1200
|
+
projectPath: this.projectPath,
|
|
1201
|
+
service: 'mongodb',
|
|
1202
|
+
dataPath: this.targetDataPath,
|
|
1203
|
+
});
|
|
1204
|
+
if (!hasExactServiceContainerLabels(labels, expected)) {
|
|
1205
|
+
throw new Error(`Refusing to remove ${name} during rollback: ownership is not proven.`);
|
|
1206
|
+
}
|
|
1207
|
+
await this.host.docker.execArgv(['rm', '-f', id], { timeoutMs: queryTimeoutMs });
|
|
1208
|
+
}
|
|
1209
|
+
await this.host.docker.removeDataDirectoryHoldingLock(this.targetDataPath, noSqlDbImage, {
|
|
1210
|
+
projectPath: this.projectPath,
|
|
1211
|
+
service: 'mongodb',
|
|
1212
|
+
});
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
/**
|
|
1216
|
+
* Restore the pre-migration runtime state. The legacy data is never touched.
|
|
1217
|
+
* NoSQLDB data under a `running` journal was written only by this migration
|
|
1218
|
+
* (no application could reach it), so discarding it loses nothing.
|
|
1219
|
+
*/
|
|
1220
|
+
public async rollback(journalArg: ILegacyMongoMigrationJournal): Promise<void> {
|
|
1221
|
+
const errors: Error[] = [];
|
|
1222
|
+
const attempt = async (operationArg: () => Promise<void>): Promise<void> => {
|
|
1223
|
+
try {
|
|
1224
|
+
await operationArg();
|
|
1225
|
+
} catch (error) {
|
|
1226
|
+
errors.push(error instanceof Error ? error : new Error('Rollback step failed.'));
|
|
1227
|
+
}
|
|
1228
|
+
};
|
|
1229
|
+
// The reader must release the legacy data before the legacy mongod opens it again.
|
|
1230
|
+
let readerReleased = false;
|
|
1231
|
+
await attempt(async () => {
|
|
1232
|
+
await this.removeReader(journalArg.token);
|
|
1233
|
+
readerReleased = true;
|
|
1234
|
+
});
|
|
1235
|
+
if (journalArg.targetCreated) {
|
|
1236
|
+
await attempt(() => this.removeMigrationTarget());
|
|
1237
|
+
}
|
|
1238
|
+
const legacy = journalArg.legacyContainer;
|
|
1239
|
+
if (legacy && readerReleased) {
|
|
1240
|
+
await attempt(async () => {
|
|
1241
|
+
const current = await this.revalidateJournaledLegacyContainer(legacy);
|
|
1242
|
+
if (!current) {
|
|
1243
|
+
return;
|
|
1244
|
+
}
|
|
1245
|
+
if (legacy.wasRunning && !current.running) {
|
|
1246
|
+
await this.host.docker.execArgv(['start', legacy.id], { timeoutMs: queryTimeoutMs });
|
|
1247
|
+
} else if (!legacy.wasRunning && current.running) {
|
|
1248
|
+
await this.stopMongodContainer(legacy.id);
|
|
1249
|
+
}
|
|
1250
|
+
});
|
|
1251
|
+
}
|
|
1252
|
+
// The dump is a derived copy of data that stays in place; logs remain as evidence.
|
|
1253
|
+
await attempt(() =>
|
|
1254
|
+
plugins.fs.rm(plugins.path.join(journalArg.evidenceDirectory, 'dump.archive'), {
|
|
1255
|
+
force: true,
|
|
1256
|
+
}),
|
|
1257
|
+
);
|
|
1258
|
+
if (errors.length > 0) {
|
|
1259
|
+
throw new AggregateError(errors, 'MongoDB migration rollback did not complete.');
|
|
1260
|
+
}
|
|
1261
|
+
await plugins.fs.rm(this.journalPath, { force: true });
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
/**
|
|
1265
|
+
* Publish the verified NoSQLDB service, remove the legacy container, and keep
|
|
1266
|
+
* a permanent completion record. Every step is idempotent, so an interrupted
|
|
1267
|
+
* commit resumes on the next start.
|
|
1268
|
+
*/
|
|
1269
|
+
private async commit(journalArg: ILegacyMongoMigrationJournal): Promise<void> {
|
|
1270
|
+
this.host.log('note', ' Publishing the verified NoSQLDB service...');
|
|
1271
|
+
await this.host.publishMigratedNoSqlDb();
|
|
1272
|
+
const legacy = journalArg.legacyContainer;
|
|
1273
|
+
if (legacy) {
|
|
1274
|
+
const current = await this.revalidateJournaledLegacyContainer(legacy);
|
|
1275
|
+
if (current) {
|
|
1276
|
+
if (current.running) {
|
|
1277
|
+
throw new Error(
|
|
1278
|
+
`Legacy MongoDB container ${legacy.name} is running again; refusing to remove it after migration.`,
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
// `-v` drops only anonymous volumes: the proven shape allows just the
|
|
1282
|
+
// image's `/data/configdb` one, which would otherwise be orphaned. The
|
|
1283
|
+
// bind-mounted legacy data directory is never touched.
|
|
1284
|
+
await this.host.docker.execArgv(['rm', '-v', legacy.id], { timeoutMs: queryTimeoutMs });
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
// A legacy registry reference now names a container that no longer exists.
|
|
1288
|
+
const projects = await this.host.getRegistryProjects();
|
|
1289
|
+
const registryName = Object.entries(projects).find(
|
|
1290
|
+
([registeredPath]) => plugins.path.resolve(registeredPath) === this.projectPath,
|
|
1291
|
+
)?.[1]?.legacy?.mongoContainer;
|
|
1292
|
+
if (
|
|
1293
|
+
registryName &&
|
|
1294
|
+
!(await this.host.discoverContainerIdByExactName(registryName, this.timeout()))
|
|
1295
|
+
) {
|
|
1296
|
+
await this.host.clearRegistryLegacyMongoContainer(this.projectPath, registryName);
|
|
1297
|
+
}
|
|
1298
|
+
journalArg.state = 'completed';
|
|
1299
|
+
journalArg.completedAt = new Date().toISOString();
|
|
1300
|
+
await this.writeJournal(journalArg);
|
|
1301
|
+
this.host.log('ok', ' Legacy MongoDB data migrated to NoSQLDB ✓');
|
|
1302
|
+
this.host.log(
|
|
1303
|
+
'note',
|
|
1304
|
+
` Preserved, never deleted by GitZone: ${this.legacyDataPath} and ${journalArg.evidenceDirectory}. ` +
|
|
1305
|
+
'Remove them manually once the migrated data is confirmed.',
|
|
1306
|
+
);
|
|
1307
|
+
}
|
|
1308
|
+
}
|