@iwonz/ywtc 3.1.0 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +25 -7
- package/dist/adapters/confluence.d.ts +6 -0
- package/dist/adapters/confluence.js +28 -3
- package/dist/adapters/confluence.js.map +1 -1
- package/dist/adapters/yandex.d.ts +6 -0
- package/dist/adapters/yandex.js +32 -5
- package/dist/adapters/yandex.js.map +1 -1
- package/dist/cli-arguments.d.ts +11 -2
- package/dist/cli-arguments.js +20 -1
- package/dist/cli-arguments.js.map +1 -1
- package/dist/cli.js +58 -1
- package/dist/cli.js.map +1 -1
- package/dist/consistency.d.ts +11 -0
- package/dist/consistency.js +38 -0
- package/dist/consistency.js.map +1 -0
- package/dist/content-model.d.ts +16 -1
- package/dist/content-model.js +518 -30
- package/dist/content-model.js.map +1 -1
- package/dist/conversion.d.ts +1 -0
- package/dist/conversion.js +17 -0
- package/dist/conversion.js.map +1 -1
- package/dist/domain.d.ts +39 -5
- package/dist/domain.js.map +1 -1
- package/dist/filesystem-export.js +6 -5
- package/dist/filesystem-export.js.map +1 -1
- package/dist/interactive-init.js +12 -8
- package/dist/interactive-init.js.map +1 -1
- package/dist/issues.d.ts +23 -0
- package/dist/issues.js +31 -0
- package/dist/issues.js.map +1 -0
- package/dist/migrator.d.ts +8 -1
- package/dist/migrator.js +557 -166
- package/dist/migrator.js.map +1 -1
- package/dist/report-catalog.d.ts +67 -0
- package/dist/report-catalog.js +119 -0
- package/dist/report-catalog.js.map +1 -0
- package/dist/report.d.ts +18 -2
- package/dist/report.js +36 -7
- package/dist/report.js.map +1 -1
- package/dist/resource-reconciliation.d.ts +21 -0
- package/dist/resource-reconciliation.js +72 -0
- package/dist/resource-reconciliation.js.map +1 -0
- package/dist/state.d.ts +3 -0
- package/dist/state.js +3 -0
- package/dist/state.js.map +1 -1
- package/package.json +2 -1
package/dist/migrator.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import PQueue from 'p-queue';
|
|
5
5
|
import { createProviderAdapter } from './adapters/factory.js';
|
|
6
|
-
import { appendGridSnapshots, normalizePageContent, normalizedPreviewHtml, renderedPreviewHtml, renderConfluenceStorage, renderYfm, } from './content-model.js';
|
|
6
|
+
import { appendGridSnapshots, assertDestinationContent, isLegacyTransferPlaceholder, normalizePageContent, normalizedPreviewHtml, rawContentFallback, renderedPreviewHtml, renderConfluenceStorage, renderYfm, setSourceResourceBinding, sourceResourceBindings, verifyDestinationContent, } from './content-model.js';
|
|
7
|
+
import { ConfirmedDestinationMissingError, verifyConfirmedDestinationPage } from './consistency.js';
|
|
7
8
|
import { assertTransferCapabilities } from './domain.js';
|
|
8
9
|
import { FilesystemExporter } from './filesystem-export.js';
|
|
9
10
|
import { discoverPageGraph } from './graph.js';
|
|
10
|
-
import { safeFilename } from './input.js';
|
|
11
11
|
import { profileBaseUrl, sameRemoteInstance } from './locators.js';
|
|
12
12
|
import { TransferReport } from './report.js';
|
|
13
|
+
import { DestinationNameAllocator, preferredDestinationName, verifiedCacheFile } from './resource-reconciliation.js';
|
|
13
14
|
import { resolveStateLocation, StateStore, } from './state.js';
|
|
14
15
|
function sourceIdentity(locator, root) {
|
|
15
16
|
return {
|
|
@@ -62,11 +63,6 @@ async function cacheDocument(cacheDirectory, canonicalId, document) {
|
|
|
62
63
|
async function loadCachedDocument(cacheDirectory, canonicalId) {
|
|
63
64
|
return JSON.parse(await readFile(cachedDocumentPath(cacheDirectory, canonicalId), 'utf8'));
|
|
64
65
|
}
|
|
65
|
-
function placeholder(adapter, page) {
|
|
66
|
-
return adapter.kind === 'confluence'
|
|
67
|
-
? `<p><em>ywtc transfer in progress from <a href="${page.url.replaceAll('&', '&').replaceAll('"', '"')}">${page.provider}</a>.</em></p>`
|
|
68
|
-
: `_ywtc transfer in progress from [${page.provider}](${page.url})._`;
|
|
69
|
-
}
|
|
70
66
|
function collisionTitle(page) {
|
|
71
67
|
const suffix = page.locator.split('/').filter(Boolean).at(-1) ?? page.id;
|
|
72
68
|
return `${page.title} [${suffix}]`;
|
|
@@ -74,7 +70,37 @@ function collisionTitle(page) {
|
|
|
74
70
|
function cachePath(cacheDirectory, page, resource) {
|
|
75
71
|
const pageKey = sha256(page.canonicalId).slice(0, 12);
|
|
76
72
|
const resourceKey = sha256(`${resource.id}:${resource.name}`).slice(0, 12);
|
|
77
|
-
return path.join(cacheDirectory, pageKey, `${resourceKey}-${
|
|
73
|
+
return path.join(cacheDirectory, pageKey, `${resourceKey}-${preferredDestinationName(resource)}`);
|
|
74
|
+
}
|
|
75
|
+
function containedCachePath(cacheDirectory, candidate) {
|
|
76
|
+
if (!candidate)
|
|
77
|
+
return undefined;
|
|
78
|
+
const relative = path.relative(path.resolve(cacheDirectory), path.resolve(candidate));
|
|
79
|
+
return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)
|
|
80
|
+
? path.resolve(candidate)
|
|
81
|
+
: undefined;
|
|
82
|
+
}
|
|
83
|
+
function resetResourceDestinations(resources) {
|
|
84
|
+
return Object.fromEntries(Object.entries(resources).map(([id, resource]) => [id, {
|
|
85
|
+
sourceId: resource.sourceId,
|
|
86
|
+
sourceName: resource.sourceName,
|
|
87
|
+
destinationName: resource.destinationName,
|
|
88
|
+
...(resource.sha256 ? { sha256: resource.sha256 } : {}),
|
|
89
|
+
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
90
|
+
...(resource.cachePath ? { cachePath: resource.cachePath } : {}),
|
|
91
|
+
...(resource.mimeType ? { mimeType: resource.mimeType } : {}),
|
|
92
|
+
status: 'planned',
|
|
93
|
+
updatedAt: new Date().toISOString(),
|
|
94
|
+
}]));
|
|
95
|
+
}
|
|
96
|
+
function sourceDerivedTextMatches(expectedBody, actualBody, provider, page) {
|
|
97
|
+
if (actualBody === undefined)
|
|
98
|
+
return false;
|
|
99
|
+
const result = verifyDestinationContent(expectedBody, actualBody, provider, page);
|
|
100
|
+
if (result.status === 'complete')
|
|
101
|
+
return true;
|
|
102
|
+
const text = (value) => renderedPreviewHtml(value, provider).replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
|
103
|
+
return text(expectedBody) === text(actualBody);
|
|
78
104
|
}
|
|
79
105
|
function destinationFromState(state, adapter) {
|
|
80
106
|
if (!state.destinationId || !state.destinationCanonicalId || !state.destinationLocator) {
|
|
@@ -92,6 +118,9 @@ function destinationFromState(state, adapter) {
|
|
|
92
118
|
};
|
|
93
119
|
}
|
|
94
120
|
function reportPage(input) {
|
|
121
|
+
const transferable = input.resources.filter((resource) => resource.kind !== 'grid');
|
|
122
|
+
const sizes = transferable.map((resource) => input.resourceSizes?.[resource.id]?.size ?? resource.size);
|
|
123
|
+
const knownBytes = sizes.reduce((total, size) => total + (size ?? 0), 0);
|
|
95
124
|
return {
|
|
96
125
|
sourceCanonicalId: input.page.canonicalId,
|
|
97
126
|
sourceId: input.page.id,
|
|
@@ -105,14 +134,17 @@ function reportPage(input) {
|
|
|
105
134
|
...(input.page.projectedParentId ? { parentId: input.page.projectedParentId } : {}),
|
|
106
135
|
depth: input.page.depth,
|
|
107
136
|
status: input.status,
|
|
108
|
-
resources:
|
|
109
|
-
|
|
137
|
+
resources: transferable.length,
|
|
138
|
+
...(transferable.length === 0 || sizes.some((size) => size !== undefined) ? { bytes: knownBytes } : {}),
|
|
139
|
+
bytesUnknown: sizes.filter((size) => size === undefined).length,
|
|
110
140
|
links: input.document.metrics.links,
|
|
111
141
|
images: input.document.metrics.images,
|
|
112
142
|
tables: input.document.metrics.tables,
|
|
113
143
|
codeBlocks: input.document.metrics.codeBlocks,
|
|
114
|
-
degraded: input.document.degraded,
|
|
115
|
-
|
|
144
|
+
degraded: input.document.degraded || (input.resourceFallbacks?.length ?? 0) > 0,
|
|
145
|
+
resourceFallbacks: input.resourceFallbacks ?? [],
|
|
146
|
+
warnings: [...new Set([...input.document.warnings, ...(input.warnings ?? [])])],
|
|
147
|
+
normalizations: [...new Set([...(input.document.normalizations ?? []), ...(input.normalizations ?? [])])],
|
|
116
148
|
...(input.error ? { error: input.error } : {}),
|
|
117
149
|
};
|
|
118
150
|
}
|
|
@@ -121,14 +153,39 @@ function runStatus(errors, warnings) {
|
|
|
121
153
|
return 'partial';
|
|
122
154
|
return warnings > 0 ? 'completed-with-warnings' : 'completed';
|
|
123
155
|
}
|
|
156
|
+
export class MigrationRunError extends Error {
|
|
157
|
+
reportPaths;
|
|
158
|
+
stateFilename;
|
|
159
|
+
constructor(message, reportPaths, stateFilename, options) {
|
|
160
|
+
super(message, options);
|
|
161
|
+
this.reportPaths = reportPaths;
|
|
162
|
+
this.stateFilename = stateFilename;
|
|
163
|
+
this.name = 'MigrationRunError';
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
async function renderRemoteDocument(input) {
|
|
167
|
+
try {
|
|
168
|
+
const rendered = input.destination.kind === 'confluence'
|
|
169
|
+
? await renderConfluenceStorage(input.document, input.page, input.context, input.container.space ?? '')
|
|
170
|
+
: await renderYfm(input.document, input.context);
|
|
171
|
+
if (!rendered.body.trim())
|
|
172
|
+
return rawContentFallback(input.page, input.destination.kind, 'renderer returned an empty body');
|
|
173
|
+
return rendered;
|
|
174
|
+
}
|
|
175
|
+
catch (error) {
|
|
176
|
+
return rawContentFallback(input.page, input.destination.kind, error instanceof Error ? error.message : String(error));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
124
179
|
export class Migrator {
|
|
125
180
|
config;
|
|
126
181
|
logger;
|
|
127
182
|
adapterFactory;
|
|
128
|
-
|
|
183
|
+
consistencyOptions;
|
|
184
|
+
constructor(config, logger, adapterFactory = createProviderAdapter, consistencyOptions = {}) {
|
|
129
185
|
this.config = config;
|
|
130
186
|
this.logger = logger;
|
|
131
187
|
this.adapterFactory = adapterFactory;
|
|
188
|
+
this.consistencyOptions = consistencyOptions;
|
|
132
189
|
}
|
|
133
190
|
async run(options) {
|
|
134
191
|
const source = this.adapterFactory(options.source.profile, this.config);
|
|
@@ -207,12 +264,16 @@ export class Migrator {
|
|
|
207
264
|
report.finish(finalStatus);
|
|
208
265
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
209
266
|
if (!options.dryRun)
|
|
210
|
-
await store.finishRun(run.id, finalStatus, {
|
|
211
|
-
|
|
267
|
+
await store.finishRun(run.id, finalStatus, {
|
|
268
|
+
pages: 0,
|
|
269
|
+
warnings: report.data.summary.warnings,
|
|
270
|
+
errors: report.data.summary.errors,
|
|
271
|
+
});
|
|
272
|
+
throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
|
|
212
273
|
}
|
|
213
274
|
const resourcesByPage = new Map();
|
|
214
275
|
const inventoryQueue = new PQueue({ concurrency: options.concurrency });
|
|
215
|
-
const inventory = { resources: 0, bytes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
|
|
276
|
+
const inventory = { resources: 0, bytes: 0, unknownSizes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
|
|
216
277
|
status(`Inspecting content and resources for ${graph.pages.length} pages`);
|
|
217
278
|
const inventoryErrors = [];
|
|
218
279
|
await Promise.all(graph.pages.map((page) => inventoryQueue.add(async () => {
|
|
@@ -226,6 +287,7 @@ export class Migrator {
|
|
|
226
287
|
await cacheDocument(options.cacheDirectory, page.canonicalId, document);
|
|
227
288
|
inventory.resources += resources.filter((resource) => resource.kind !== 'grid').length;
|
|
228
289
|
inventory.bytes += resources.reduce((total, resource) => total + (resource.size ?? 0), 0);
|
|
290
|
+
inventory.unknownSizes += resources.filter((resource) => resource.kind !== 'grid' && resource.size === undefined).length;
|
|
229
291
|
inventory.grids += resources.filter((resource) => resource.kind === 'grid').length;
|
|
230
292
|
inventory.links += document.metrics.links;
|
|
231
293
|
inventory.images += document.metrics.images;
|
|
@@ -244,23 +306,23 @@ export class Migrator {
|
|
|
244
306
|
report.finish(finalStatus);
|
|
245
307
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
246
308
|
if (!options.dryRun)
|
|
247
|
-
await store.finishRun(run.id, finalStatus, {
|
|
248
|
-
|
|
309
|
+
await store.finishRun(run.id, finalStatus, {
|
|
310
|
+
pages: graph.pages.length,
|
|
311
|
+
warnings: report.data.summary.warnings,
|
|
312
|
+
errors: report.data.summary.errors,
|
|
313
|
+
});
|
|
314
|
+
throw new MigrationRunError('Resource preflight failed before destination mutation', reportPaths, stateLocation.filename);
|
|
249
315
|
}
|
|
250
|
-
const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} bytes) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
|
|
316
|
+
const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} known bytes${inventory.unknownSizes > 0 ? ` · ${inventory.unknownSizes} sizes unknown` : ''}) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
|
|
251
317
|
status(planSummary);
|
|
252
318
|
this.logger.info(planSummary);
|
|
253
319
|
let output;
|
|
254
320
|
let archive;
|
|
255
|
-
let transferErrors = 0;
|
|
256
|
-
let transferWarnings = graph.warnings.length;
|
|
257
321
|
try {
|
|
258
322
|
if (options.destination.type === 'directory') {
|
|
259
323
|
const result = await this.exportFilesystem(source, graph, options.destination, options, stateLocation.id, store, report, sourceFingerprints, resourcesByPage);
|
|
260
324
|
output = result.directory;
|
|
261
325
|
archive = result.archive;
|
|
262
|
-
transferErrors += result.errors.length;
|
|
263
|
-
transferWarnings += result.warnings.length;
|
|
264
326
|
report.output('directory', result.directory);
|
|
265
327
|
if (result.archive)
|
|
266
328
|
report.output('archive', result.archive);
|
|
@@ -268,7 +330,7 @@ export class Migrator {
|
|
|
268
330
|
report.output('backup', result.backup);
|
|
269
331
|
}
|
|
270
332
|
else {
|
|
271
|
-
|
|
333
|
+
await this.transferRemote({
|
|
272
334
|
source,
|
|
273
335
|
destination: destination,
|
|
274
336
|
container: container,
|
|
@@ -279,27 +341,30 @@ export class Migrator {
|
|
|
279
341
|
sourceFingerprints,
|
|
280
342
|
resourcesByPage,
|
|
281
343
|
});
|
|
282
|
-
transferErrors += result.errors;
|
|
283
|
-
transferWarnings += result.warnings;
|
|
284
344
|
output = container.url;
|
|
285
345
|
report.output('destination', container.url);
|
|
286
346
|
}
|
|
287
|
-
|
|
347
|
+
report.finish('completed');
|
|
348
|
+
const finalStatus = runStatus(report.data.summary.errors, report.data.summary.warnings);
|
|
288
349
|
report.finish(finalStatus);
|
|
289
350
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
290
351
|
if (!options.dryRun)
|
|
291
|
-
await store.finishRun(run.id, finalStatus, {
|
|
292
|
-
|
|
352
|
+
await store.finishRun(run.id, finalStatus, {
|
|
353
|
+
pages: graph.pages.length,
|
|
354
|
+
warnings: report.data.summary.warnings,
|
|
355
|
+
errors: report.data.summary.errors,
|
|
356
|
+
});
|
|
357
|
+
status(`${finalStatus}: ${graph.pages.length} pages · ${report.data.summary.errors} errors · ${report.data.summary.warnings} warnings`);
|
|
293
358
|
return {
|
|
294
359
|
status: finalStatus,
|
|
295
|
-
hasErrors:
|
|
360
|
+
hasErrors: report.data.summary.errors > 0,
|
|
296
361
|
reportPaths,
|
|
297
362
|
stateId: stateLocation.id,
|
|
298
363
|
stateFilename: stateLocation.filename,
|
|
299
364
|
...(store.loadInfo.backupFilename ? { stateBackupFilename: store.loadInfo.backupFilename } : {}),
|
|
300
365
|
...(output ? { output } : {}),
|
|
301
366
|
...(archive ? { archive } : {}),
|
|
302
|
-
summary:
|
|
367
|
+
summary: report.data.summary,
|
|
303
368
|
};
|
|
304
369
|
}
|
|
305
370
|
catch (error) {
|
|
@@ -310,9 +375,13 @@ export class Migrator {
|
|
|
310
375
|
report.finish(finalStatus);
|
|
311
376
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
312
377
|
if (!options.dryRun)
|
|
313
|
-
await store.finishRun(run.id, finalStatus, {
|
|
378
|
+
await store.finishRun(run.id, finalStatus, {
|
|
379
|
+
pages: graph.pages.length,
|
|
380
|
+
warnings: report.data.summary.warnings,
|
|
381
|
+
errors: report.data.summary.errors,
|
|
382
|
+
});
|
|
314
383
|
this.logger.error(`${message}. Report: ${reportPaths.html}`);
|
|
315
|
-
throw error;
|
|
384
|
+
throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
|
|
316
385
|
}
|
|
317
386
|
}
|
|
318
387
|
async exportFilesystem(source, graph, destination, options, identity, store, report, sourceFingerprints, resourcesByPage) {
|
|
@@ -341,16 +410,22 @@ export class Migrator {
|
|
|
341
410
|
const document = await loadCachedDocument(options.cacheDirectory, page.canonicalId);
|
|
342
411
|
const manifestPage = result.manifest.pages.find((item) => item.sourceCanonicalId === page.canonicalId);
|
|
343
412
|
const previous = store.value.pages[page.canonicalId];
|
|
344
|
-
const
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
413
|
+
const sourceResources = resourcesByPage.get(page.canonicalId) ?? [];
|
|
414
|
+
const transferableResources = sourceResources.filter((resource) => resource.kind !== 'grid');
|
|
415
|
+
const resources = Object.fromEntries((manifestPage?.resources ?? []).map((resource, index) => {
|
|
416
|
+
const sourceResource = transferableResources[index];
|
|
417
|
+
const sourceId = sourceResource?.id ?? String(index);
|
|
418
|
+
return [sourceId, {
|
|
419
|
+
sourceId,
|
|
420
|
+
sourceName: sourceResource?.name ?? resource.name,
|
|
421
|
+
destinationId: resource.path,
|
|
422
|
+
destinationName: resource.name,
|
|
423
|
+
...(resource.sha256 ? { sha256: resource.sha256 } : {}),
|
|
424
|
+
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
425
|
+
status: 'complete',
|
|
426
|
+
updatedAt: new Date().toISOString(),
|
|
427
|
+
}];
|
|
428
|
+
}));
|
|
354
429
|
await store.setPage({
|
|
355
430
|
sourceId: page.id,
|
|
356
431
|
sourceCanonicalId: page.canonicalId,
|
|
@@ -371,8 +446,9 @@ export class Migrator {
|
|
|
371
446
|
report.addPage(reportPage({
|
|
372
447
|
page,
|
|
373
448
|
status: options.dryRun ? 'planned' : result.errors.length ? 'partial' : 'complete',
|
|
374
|
-
resources:
|
|
449
|
+
resources: sourceResources,
|
|
375
450
|
document,
|
|
451
|
+
resourceSizes: resources,
|
|
376
452
|
...(manifestPage?.warnings ? { warnings: manifestPage.warnings } : {}),
|
|
377
453
|
}), { source: normalizedPreviewHtml(document), destination: normalizedPreviewHtml(document) });
|
|
378
454
|
}
|
|
@@ -385,15 +461,34 @@ export class Migrator {
|
|
|
385
461
|
async transferRemote(input) {
|
|
386
462
|
const { source, destination, container, graph, options, store, report, sourceFingerprints, resourcesByPage } = input;
|
|
387
463
|
const destinationPages = new Map();
|
|
388
|
-
|
|
389
|
-
|
|
464
|
+
const sourcePages = new Map(graph.pages.map((page) => [page.canonicalId, page]));
|
|
465
|
+
const bootstrapBodies = new Map();
|
|
390
466
|
const totalResources = [...resourcesByPage.values()].reduce((total, resources) => total + resources.filter((resource) => resource.kind !== 'grid').length, 0);
|
|
391
|
-
const progress = {
|
|
467
|
+
const progress = { staged: 0, pages: 0, resources: 0 };
|
|
392
468
|
const totalUnits = Math.max(1, graph.pages.length * 2 + totalResources);
|
|
393
469
|
const updateProgress = (active) => {
|
|
394
|
-
const completed = progress.
|
|
470
|
+
const completed = progress.staged + progress.pages + progress.resources;
|
|
395
471
|
options.onStatus?.(`Progress ${Math.min(100, Math.floor((completed / totalUnits) * 100))}% · pages ${progress.pages}/${graph.pages.length} · files ${progress.resources}/${totalResources} · ${active}`);
|
|
396
472
|
};
|
|
473
|
+
for (const page of graph.pages) {
|
|
474
|
+
const [document, cachedPage] = await Promise.all([
|
|
475
|
+
loadCachedDocument(options.cacheDirectory, page.canonicalId),
|
|
476
|
+
loadCachedPage(options.cacheDirectory, page.canonicalId),
|
|
477
|
+
]);
|
|
478
|
+
const resources = resourcesByPage.get(page.canonicalId) ?? [];
|
|
479
|
+
bootstrapBodies.set(page.canonicalId, await renderRemoteDocument({
|
|
480
|
+
destination,
|
|
481
|
+
container,
|
|
482
|
+
document,
|
|
483
|
+
page: cachedPage,
|
|
484
|
+
context: {
|
|
485
|
+
pages: new Map(),
|
|
486
|
+
resources: sourceResourceBindings(cachedPage, resources),
|
|
487
|
+
sourcePages,
|
|
488
|
+
currentPageId: page.canonicalId,
|
|
489
|
+
},
|
|
490
|
+
}));
|
|
491
|
+
}
|
|
397
492
|
for (const page of graph.pages) {
|
|
398
493
|
if (options.signal?.aborted)
|
|
399
494
|
throw options.signal.reason ?? new Error('Transfer cancelled');
|
|
@@ -401,20 +496,88 @@ export class Migrator {
|
|
|
401
496
|
const parent = page.projectedParentId ? destinationPages.get(page.projectedParentId) : container;
|
|
402
497
|
if (!parent) {
|
|
403
498
|
const message = `Destination parent is unavailable for ${page.title}`;
|
|
404
|
-
errors += 1;
|
|
405
499
|
report.error(message);
|
|
406
|
-
progress.
|
|
500
|
+
progress.staged += 1;
|
|
407
501
|
updateProgress(message);
|
|
408
502
|
continue;
|
|
409
503
|
}
|
|
410
504
|
try {
|
|
505
|
+
const cachedPage = await loadCachedPage(options.cacheDirectory, page.canonicalId);
|
|
506
|
+
const bootstrap = bootstrapBodies.get(page.canonicalId);
|
|
411
507
|
let created;
|
|
412
508
|
let title = saved?.destinationTitle ?? page.title;
|
|
509
|
+
let resourcesForState = saved?.resources ?? {};
|
|
510
|
+
let previousDestinationIds = saved?.previousDestinationIds ?? [];
|
|
511
|
+
let mappingRecovered = false;
|
|
413
512
|
if (saved?.destinationId) {
|
|
414
|
-
|
|
415
|
-
|
|
513
|
+
const mapped = destinationFromState(saved, destination);
|
|
514
|
+
try {
|
|
515
|
+
created = await verifyConfirmedDestinationPage(destination, mapped, {
|
|
516
|
+
...this.consistencyOptions,
|
|
517
|
+
onRetry: (attempt, delayMs) => report.issue({
|
|
518
|
+
code: 'destination-consistency-retry',
|
|
519
|
+
severity: 'warning',
|
|
520
|
+
phase: 'verification',
|
|
521
|
+
pageId: page.canonicalId,
|
|
522
|
+
retryable: true,
|
|
523
|
+
message: `${page.title}: destination page ${mapped.id} returned 404; consistency retry ${attempt} in ${delayMs} ms`,
|
|
524
|
+
}),
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
catch (error) {
|
|
528
|
+
if (!(error instanceof ConfirmedDestinationMissingError) || options.dryRun)
|
|
529
|
+
throw error;
|
|
530
|
+
if (page.projectedParentId) {
|
|
531
|
+
await verifyConfirmedDestinationPage(destination, parent, this.consistencyOptions);
|
|
532
|
+
}
|
|
533
|
+
const candidate = await (destination.findRecoverablePage?.(parent, title) ?? destination.findChild(parent, title));
|
|
534
|
+
if (candidate && sourceDerivedTextMatches(bootstrap.body, candidate.content, destination.kind, cachedPage)) {
|
|
535
|
+
created = candidate;
|
|
536
|
+
}
|
|
537
|
+
else {
|
|
538
|
+
let recoveryTitle = title;
|
|
539
|
+
const occupied = candidate ?? await destination.findChild(parent, recoveryTitle);
|
|
540
|
+
if (occupied) {
|
|
541
|
+
const base = collisionTitle(page);
|
|
542
|
+
const hash = sha256(page.canonicalId).slice(0, 8);
|
|
543
|
+
recoveryTitle = base;
|
|
544
|
+
let sequence = 0;
|
|
545
|
+
while (await destination.findChild(parent, recoveryTitle)) {
|
|
546
|
+
sequence += 1;
|
|
547
|
+
recoveryTitle = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
created = await destination.createPage({ container: parent, title: recoveryTitle, body: bootstrap.body });
|
|
551
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
552
|
+
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
553
|
+
}
|
|
554
|
+
previousDestinationIds = [...new Set([...previousDestinationIds, mapped.id])];
|
|
555
|
+
resourcesForState = resetResourceDestinations(saved.resources);
|
|
556
|
+
mappingRecovered = true;
|
|
557
|
+
report.issue({
|
|
558
|
+
code: 'destination-mapping-recovered',
|
|
559
|
+
severity: 'warning',
|
|
560
|
+
phase: 'page',
|
|
561
|
+
pageId: page.canonicalId,
|
|
562
|
+
retryable: false,
|
|
563
|
+
message: `${page.title}: replaced unavailable destination mapping ${mapped.id} with ${created.id}`,
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
const generatedPlaceholder = created.title === saved.destinationTitle
|
|
567
|
+
&& isLegacyTransferPlaceholder(created.content, destination.kind, cachedPage);
|
|
568
|
+
if (!mappingRecovered && ['complete', 'partial'].includes(saved.status) && saved.destinationFingerprint && created.fingerprint !== saved.destinationFingerprint && !generatedPlaceholder) {
|
|
416
569
|
throw new Error(`Destination conflict for ${page.title}: content changed since the last verified run`);
|
|
417
570
|
}
|
|
571
|
+
if (generatedPlaceholder && !options.dryRun) {
|
|
572
|
+
created = await destination.updatePage({
|
|
573
|
+
page: created,
|
|
574
|
+
title: created.title,
|
|
575
|
+
body: bootstrap.body,
|
|
576
|
+
...(created.fingerprint ? { expectedFingerprint: created.fingerprint } : {}),
|
|
577
|
+
});
|
|
578
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
579
|
+
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
580
|
+
}
|
|
418
581
|
}
|
|
419
582
|
else if (options.dryRun) {
|
|
420
583
|
created = {
|
|
@@ -425,6 +588,8 @@ export class Migrator {
|
|
|
425
588
|
title,
|
|
426
589
|
url: container.url,
|
|
427
590
|
locator: `dry-run:${page.id}`,
|
|
591
|
+
content: bootstrap.body,
|
|
592
|
+
contentLength: Buffer.byteLength(bootstrap.body),
|
|
428
593
|
...(container.space ? { space: container.space } : {}),
|
|
429
594
|
};
|
|
430
595
|
}
|
|
@@ -439,7 +604,9 @@ export class Migrator {
|
|
|
439
604
|
title = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
|
|
440
605
|
}
|
|
441
606
|
}
|
|
442
|
-
created = await destination.createPage({ container: parent, title, body:
|
|
607
|
+
created = await destination.createPage({ container: parent, title, body: bootstrap.body });
|
|
608
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
609
|
+
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
443
610
|
}
|
|
444
611
|
destinationPages.set(page.canonicalId, created);
|
|
445
612
|
const next = {
|
|
@@ -452,18 +619,18 @@ export class Migrator {
|
|
|
452
619
|
destinationCanonicalId: created.canonicalId,
|
|
453
620
|
destinationLocator: created.url,
|
|
454
621
|
destinationTitle: created.title,
|
|
622
|
+
...(previousDestinationIds.length > 0 ? { previousDestinationIds } : {}),
|
|
455
623
|
sourceFingerprint: sourceFingerprints.get(page.canonicalId),
|
|
456
624
|
...(created.fingerprint ? { destinationFingerprint: created.fingerprint } : {}),
|
|
457
625
|
status: options.dryRun ? 'planned' : saved?.status ?? 'skeleton-created',
|
|
458
|
-
resources:
|
|
626
|
+
resources: resourcesForState,
|
|
459
627
|
degradations: saved?.degradations ?? [],
|
|
460
628
|
updatedAt: new Date().toISOString(),
|
|
461
629
|
};
|
|
462
630
|
await store.setPage(next);
|
|
463
|
-
options.onStatus?.(`${options.dryRun ? 'Planned' : '
|
|
631
|
+
options.onStatus?.(`${options.dryRun ? 'Planned' : 'Staged content for'} ${page.title}`);
|
|
464
632
|
}
|
|
465
633
|
catch (error) {
|
|
466
|
-
errors += 1;
|
|
467
634
|
const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
|
|
468
635
|
report.error(message);
|
|
469
636
|
const savedState = store.value.pages[page.canonicalId];
|
|
@@ -471,8 +638,8 @@ export class Migrator {
|
|
|
471
638
|
await store.setPage({ ...savedState, status: /conflict/i.test(message) ? 'conflict' : 'failed', lastError: message, updatedAt: new Date().toISOString() });
|
|
472
639
|
}
|
|
473
640
|
finally {
|
|
474
|
-
progress.
|
|
475
|
-
updateProgress(`
|
|
641
|
+
progress.staged += 1;
|
|
642
|
+
updateProgress(`content staged ${page.title}`);
|
|
476
643
|
}
|
|
477
644
|
}
|
|
478
645
|
const contentQueue = new PQueue({ concurrency: options.concurrency });
|
|
@@ -490,74 +657,247 @@ export class Migrator {
|
|
|
490
657
|
status: failedState?.status ?? 'blocked',
|
|
491
658
|
resources,
|
|
492
659
|
document,
|
|
493
|
-
error: failedState?.lastError ?? 'Destination
|
|
660
|
+
error: failedState?.lastError ?? 'Destination page with source-derived content was not created',
|
|
494
661
|
}));
|
|
495
662
|
progress.pages += 1;
|
|
496
663
|
updateProgress(`blocked ${page.title}`);
|
|
497
664
|
return;
|
|
498
665
|
}
|
|
499
666
|
const state = store.value.pages[page.canonicalId];
|
|
500
|
-
const
|
|
667
|
+
const bootstrap = bootstrapBodies.get(page.canonicalId);
|
|
668
|
+
const bindings = sourceResourceBindings(cachedPage, resources);
|
|
669
|
+
const transferableResources = resources.filter((item) => item.kind !== 'grid');
|
|
670
|
+
const resourceProblems = [];
|
|
671
|
+
const failedResourceIds = new Set();
|
|
672
|
+
const recordResourceFailure = async (resource, detail) => {
|
|
673
|
+
if (!failedResourceIds.has(resource.id)) {
|
|
674
|
+
failedResourceIds.add(resource.id);
|
|
675
|
+
resourceProblems.push(detail);
|
|
676
|
+
report.issue({
|
|
677
|
+
code: 'resource-transfer-failed',
|
|
678
|
+
severity: 'error',
|
|
679
|
+
phase: 'resource',
|
|
680
|
+
pageId: page.canonicalId,
|
|
681
|
+
resourceId: resource.id,
|
|
682
|
+
retryable: true,
|
|
683
|
+
message: `${page.title}: ${detail}`,
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
const previous = state.resources[resource.id];
|
|
687
|
+
state.resources[resource.id] = {
|
|
688
|
+
sourceId: resource.id,
|
|
689
|
+
sourceName: resource.name,
|
|
690
|
+
...(previous?.destinationId ? { destinationId: previous.destinationId } : {}),
|
|
691
|
+
destinationName: previous?.destinationName ?? preferredDestinationName(resource),
|
|
692
|
+
...(previous?.sha256 ? { sha256: previous.sha256 } : {}),
|
|
693
|
+
...(previous?.size !== undefined ? { size: previous.size } : resource.size !== undefined ? { size: resource.size } : {}),
|
|
694
|
+
...(previous?.cachePath ? { cachePath: previous.cachePath } : {}),
|
|
695
|
+
...(previous?.mimeType ? { mimeType: previous.mimeType } : {}),
|
|
696
|
+
status: 'failed',
|
|
697
|
+
lastError: detail,
|
|
698
|
+
updatedAt: new Date().toISOString(),
|
|
699
|
+
};
|
|
700
|
+
await store.setPage({ ...state, status: 'partial', lastError: detail, updatedAt: new Date().toISOString() });
|
|
701
|
+
};
|
|
501
702
|
try {
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
if (
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
703
|
+
let remoteResources = options.dryRun ? [] : await destination.listUploadedResources(destinationPage);
|
|
704
|
+
const nameAllocator = new DestinationNameAllocator(remoteResources.map((resource) => resource.name));
|
|
705
|
+
for (const resourceState of Object.values(state.resources)) {
|
|
706
|
+
if (resourceState.status === 'complete')
|
|
707
|
+
nameAllocator.reserve(resourceState.destinationName);
|
|
708
|
+
}
|
|
709
|
+
const fallbackBinding = (resource) => [...sourceResourceBindings(cachedPage, [resource]).values()][0];
|
|
710
|
+
const remoteMatches = async (remote, downloaded, trustedMapping = false) => {
|
|
711
|
+
if (remote.size !== undefined && remote.size !== downloaded.size)
|
|
712
|
+
return false;
|
|
713
|
+
if (!destination.downloadUploadedResource) {
|
|
714
|
+
return remote.fingerprint === downloaded.sha256
|
|
715
|
+
|| (trustedMapping && (remote.size === undefined || remote.size === downloaded.size));
|
|
716
|
+
}
|
|
717
|
+
const comparisonPath = `${downloaded.path}.destination-${sha256(remote.id).slice(0, 8)}`;
|
|
718
|
+
try {
|
|
719
|
+
const actual = await destination.downloadUploadedResource({ page: destinationPage, resource: remote, destination: comparisonPath });
|
|
720
|
+
return actual.size === downloaded.size && actual.sha256 === downloaded.sha256;
|
|
515
721
|
}
|
|
722
|
+
finally {
|
|
723
|
+
await rm(comparisonPath, { force: true });
|
|
724
|
+
}
|
|
725
|
+
};
|
|
726
|
+
for (const resource of transferableResources) {
|
|
727
|
+
const existing = state.resources[resource.id];
|
|
516
728
|
if (options.dryRun) {
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
729
|
+
const destinationName = nameAllocator.allocate(resource, existing?.status === 'complete' ? existing.destinationName : undefined);
|
|
730
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: {
|
|
731
|
+
id: `dry-run:${resource.id}`,
|
|
732
|
+
name: destinationName,
|
|
733
|
+
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
734
|
+
} });
|
|
522
735
|
progress.resources += 1;
|
|
523
736
|
updateProgress(`planned ${resource.name}`);
|
|
524
737
|
continue;
|
|
525
738
|
}
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
739
|
+
try {
|
|
740
|
+
const filename = containedCachePath(options.cacheDirectory, existing?.cachePath)
|
|
741
|
+
?? cachePath(options.cacheDirectory, page, resource);
|
|
742
|
+
const cached = await verifiedCacheFile(filename, existing?.sha256 ? {
|
|
743
|
+
sha256: existing.sha256,
|
|
744
|
+
...(existing.size !== undefined ? { size: existing.size } : {}),
|
|
745
|
+
...(existing.mimeType ? { mimeType: existing.mimeType } : {}),
|
|
746
|
+
} : undefined);
|
|
747
|
+
const downloaded = cached ?? await source.downloadResource(resource, filename);
|
|
748
|
+
const mimeType = downloaded.mimeType ?? existing?.mimeType ?? resource.mimeType;
|
|
749
|
+
const mappedRemote = existing?.destinationId
|
|
750
|
+
? remoteResources.find((item) => item.id === existing.destinationId)
|
|
751
|
+
: undefined;
|
|
752
|
+
if (mappedRemote && await remoteMatches(mappedRemote, downloaded, true)) {
|
|
753
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: mappedRemote });
|
|
754
|
+
state.resources[resource.id] = {
|
|
755
|
+
sourceId: resource.id,
|
|
756
|
+
sourceName: resource.name,
|
|
757
|
+
destinationId: mappedRemote.id,
|
|
758
|
+
destinationName: mappedRemote.name,
|
|
759
|
+
sha256: downloaded.sha256,
|
|
760
|
+
size: downloaded.size,
|
|
761
|
+
cachePath: downloaded.path,
|
|
762
|
+
...(mimeType ? { mimeType } : {}),
|
|
763
|
+
status: 'complete',
|
|
764
|
+
updatedAt: new Date().toISOString(),
|
|
765
|
+
};
|
|
766
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
767
|
+
updateProgress(`reused ${resource.name}`);
|
|
768
|
+
continue;
|
|
769
|
+
}
|
|
770
|
+
const reconciliationNames = new Set([
|
|
771
|
+
preferredDestinationName(resource),
|
|
772
|
+
...(existing?.sha256 ? [existing.destinationName] : []),
|
|
773
|
+
].map((name) => name.toLocaleLowerCase()));
|
|
774
|
+
const reconciled = remoteResources.find((item) => reconciliationNames.has(item.name.toLocaleLowerCase()));
|
|
775
|
+
if (reconciled && await remoteMatches(reconciled, downloaded)) {
|
|
776
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: reconciled });
|
|
777
|
+
state.resources[resource.id] = {
|
|
778
|
+
sourceId: resource.id,
|
|
779
|
+
sourceName: resource.name,
|
|
780
|
+
destinationId: reconciled.id,
|
|
781
|
+
destinationName: reconciled.name,
|
|
782
|
+
sha256: downloaded.sha256,
|
|
783
|
+
size: downloaded.size,
|
|
784
|
+
cachePath: downloaded.path,
|
|
785
|
+
...(mimeType ? { mimeType } : {}),
|
|
786
|
+
status: 'complete',
|
|
787
|
+
updatedAt: new Date().toISOString(),
|
|
788
|
+
};
|
|
789
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
790
|
+
updateProgress(`reconciled ${resource.name}`);
|
|
791
|
+
continue;
|
|
792
|
+
}
|
|
793
|
+
const updateExisting = existing?.status === 'complete' && mappedRemote ? mappedRemote : undefined;
|
|
794
|
+
const destinationName = updateExisting?.name ?? nameAllocator.allocate(resource, existing?.status === 'complete' && !nameAllocator.isOccupied(existing.destinationName) ? existing.destinationName : undefined);
|
|
795
|
+
state.resources[resource.id] = {
|
|
796
|
+
sourceId: resource.id,
|
|
797
|
+
sourceName: resource.name,
|
|
798
|
+
destinationName,
|
|
799
|
+
sha256: downloaded.sha256,
|
|
800
|
+
size: downloaded.size,
|
|
801
|
+
cachePath: downloaded.path,
|
|
802
|
+
...(mimeType ? { mimeType } : {}),
|
|
803
|
+
status: 'planned',
|
|
804
|
+
updatedAt: new Date().toISOString(),
|
|
805
|
+
};
|
|
806
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
807
|
+
const matchingName = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
|
|
808
|
+
let result = matchingName && await remoteMatches(matchingName, downloaded) ? matchingName : undefined;
|
|
809
|
+
if (!result) {
|
|
810
|
+
try {
|
|
811
|
+
result = await destination.uploadResource({
|
|
812
|
+
page: destinationPage,
|
|
813
|
+
name: destinationName,
|
|
814
|
+
filePath: downloaded.path,
|
|
815
|
+
size: downloaded.size,
|
|
816
|
+
...(mimeType ? { mimeType } : {}),
|
|
817
|
+
...(updateExisting ? { existingId: updateExisting.id } : {}),
|
|
818
|
+
});
|
|
819
|
+
}
|
|
820
|
+
catch (uploadError) {
|
|
821
|
+
remoteResources = await destination.listUploadedResources(destinationPage);
|
|
822
|
+
const uncertain = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
|
|
823
|
+
if (uncertain && await remoteMatches(uncertain, downloaded))
|
|
824
|
+
result = uncertain;
|
|
825
|
+
else {
|
|
826
|
+
const retryName = nameAllocator.allocate(resource);
|
|
827
|
+
result = await destination.uploadResource({
|
|
828
|
+
page: destinationPage,
|
|
829
|
+
name: retryName,
|
|
830
|
+
filePath: downloaded.path,
|
|
831
|
+
size: downloaded.size,
|
|
832
|
+
...(mimeType ? { mimeType } : {}),
|
|
833
|
+
}).catch((retryError) => { throw new Error(`${retryError instanceof Error ? retryError.message : String(retryError)} (initial upload: ${uploadError instanceof Error ? uploadError.message : String(uploadError)})`); });
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
if (!remoteResources.some((item) => item.id === result.id))
|
|
838
|
+
remoteResources.push(result);
|
|
839
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: result });
|
|
840
|
+
state.resources[resource.id] = {
|
|
841
|
+
sourceId: resource.id,
|
|
842
|
+
sourceName: resource.name,
|
|
843
|
+
destinationId: result.id,
|
|
844
|
+
destinationName: result.name,
|
|
845
|
+
sha256: downloaded.sha256,
|
|
846
|
+
size: downloaded.size,
|
|
847
|
+
cachePath: downloaded.path,
|
|
848
|
+
...(mimeType ? { mimeType } : {}),
|
|
849
|
+
status: 'complete',
|
|
850
|
+
updatedAt: new Date().toISOString(),
|
|
851
|
+
};
|
|
852
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
853
|
+
updateProgress(`uploaded ${resource.name} → ${page.title}`);
|
|
854
|
+
}
|
|
855
|
+
catch (error) {
|
|
856
|
+
await recordResourceFailure(resource, `Resource ${resource.name} was not transferred: ${error instanceof Error ? error.message : String(error)}`);
|
|
857
|
+
updateProgress(`fallback ${resource.name} → ${page.title}`);
|
|
858
|
+
}
|
|
859
|
+
finally {
|
|
860
|
+
progress.resources += 1;
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
if (!options.dryRun && transferableResources.length > 0) {
|
|
864
|
+
try {
|
|
865
|
+
remoteResources = await destination.listUploadedResources(destinationPage);
|
|
866
|
+
for (const resource of transferableResources) {
|
|
867
|
+
const transferred = state.resources[resource.id];
|
|
868
|
+
if (transferred?.status !== 'complete')
|
|
869
|
+
continue;
|
|
870
|
+
const remote = remoteResources.find((item) => item.id === transferred.destinationId || item.name === transferred.destinationName);
|
|
871
|
+
const sizeMismatch = transferred.size !== undefined
|
|
872
|
+
&& remote?.size !== undefined
|
|
873
|
+
&& transferred.size !== remote.size;
|
|
874
|
+
if (!remote || sizeMismatch) {
|
|
875
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
|
|
876
|
+
await recordResourceFailure(resource, !remote
|
|
877
|
+
? `Verification did not find destination resource for ${resource.name}`
|
|
878
|
+
: `Verification size mismatch for ${resource.name}: ${transferred.size} source bytes, ${remote.size} destination bytes`);
|
|
879
|
+
}
|
|
880
|
+
else {
|
|
881
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: remote });
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
catch (error) {
|
|
886
|
+
for (const resource of transferableResources) {
|
|
887
|
+
if (state.resources[resource.id]?.status !== 'complete')
|
|
888
|
+
continue;
|
|
889
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
|
|
890
|
+
await recordResourceFailure(resource, `Destination resource verification failed for ${resource.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
891
|
+
}
|
|
892
|
+
}
|
|
550
893
|
}
|
|
551
894
|
const context = {
|
|
552
895
|
pages: destinationPages,
|
|
553
|
-
resources:
|
|
554
|
-
sourcePages
|
|
896
|
+
resources: bindings,
|
|
897
|
+
sourcePages,
|
|
555
898
|
currentPageId: page.canonicalId,
|
|
556
899
|
};
|
|
557
|
-
const rendered = destination
|
|
558
|
-
? await renderConfluenceStorage(document, cachedPage, context, container.space ?? '')
|
|
559
|
-
: await renderYfm(document, context);
|
|
560
|
-
warnings += rendered.warnings.length;
|
|
900
|
+
const rendered = await renderRemoteDocument({ destination, container, document, page: cachedPage, context });
|
|
561
901
|
if (rendered.degraded && !options.dryRun) {
|
|
562
902
|
const extension = page.rawFormat === 'yfm' ? 'yfm' : 'storage.xml';
|
|
563
903
|
const snapshotName = `ywtc-source-${sha256(page.canonicalId).slice(0, 8)}.${extension}`;
|
|
@@ -566,92 +906,144 @@ export class Migrator {
|
|
|
566
906
|
await writeFile(snapshotPath, cachedPage.rawContent, { mode: 0o600 });
|
|
567
907
|
const snapshotKey = `snapshot:${page.canonicalId}`;
|
|
568
908
|
const existingSnapshot = state.resources[snapshotKey];
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
909
|
+
try {
|
|
910
|
+
const snapshot = existingSnapshot?.status === 'complete' && existingSnapshot.destinationId
|
|
911
|
+
? {
|
|
912
|
+
id: existingSnapshot.destinationId,
|
|
913
|
+
name: existingSnapshot.destinationName,
|
|
914
|
+
...(existingSnapshot.size !== undefined ? { size: existingSnapshot.size } : {}),
|
|
915
|
+
}
|
|
916
|
+
: await destination.uploadResource({
|
|
917
|
+
page: destinationPage,
|
|
918
|
+
name: snapshotName,
|
|
919
|
+
filePath: snapshotPath,
|
|
920
|
+
size: Buffer.byteLength(cachedPage.rawContent),
|
|
921
|
+
mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
|
|
922
|
+
...(existingSnapshot?.destinationId ? { existingId: existingSnapshot.destinationId } : {}),
|
|
923
|
+
});
|
|
924
|
+
state.resources[snapshotKey] = {
|
|
925
|
+
sourceId: snapshotKey,
|
|
926
|
+
sourceName: snapshotName,
|
|
927
|
+
destinationId: snapshot.id,
|
|
928
|
+
destinationName: snapshot.name,
|
|
929
|
+
...(snapshot.size !== undefined ? { size: snapshot.size } : {}),
|
|
930
|
+
sha256: sha256(cachedPage.rawContent),
|
|
931
|
+
status: 'complete',
|
|
932
|
+
updatedAt: new Date().toISOString(),
|
|
933
|
+
};
|
|
934
|
+
}
|
|
935
|
+
catch (error) {
|
|
936
|
+
const detail = `Raw source snapshot was not transferred: ${error instanceof Error ? error.message : String(error)}`;
|
|
937
|
+
resourceProblems.push(detail);
|
|
938
|
+
report.error(`${page.title}: ${detail}`);
|
|
939
|
+
state.resources[snapshotKey] = {
|
|
940
|
+
sourceId: snapshotKey,
|
|
941
|
+
sourceName: snapshotName,
|
|
942
|
+
destinationName: snapshotName,
|
|
943
|
+
status: 'failed',
|
|
944
|
+
lastError: detail,
|
|
945
|
+
updatedAt: new Date().toISOString(),
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
state.degradations = [...new Set([...rendered.warnings, ...resourceProblems])];
|
|
595
949
|
}
|
|
596
950
|
let verified = destinationPage;
|
|
951
|
+
let verificationDegradations = [];
|
|
597
952
|
if (!options.dryRun) {
|
|
953
|
+
const current = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
|
|
954
|
+
if (destinationPage.contentFingerprint
|
|
955
|
+
&& current.contentFingerprint
|
|
956
|
+
&& current.contentFingerprint !== destinationPage.contentFingerprint) {
|
|
957
|
+
throw new Error(`Destination conflict for ${page.title}: content changed while resources were transferred`);
|
|
958
|
+
}
|
|
598
959
|
verified = await destination.updatePage({
|
|
599
|
-
page:
|
|
600
|
-
title:
|
|
960
|
+
page: current,
|
|
961
|
+
title: current.title,
|
|
601
962
|
body: rendered.body,
|
|
602
|
-
...(
|
|
963
|
+
...(current.fingerprint ? { expectedFingerprint: current.fingerprint } : {}),
|
|
603
964
|
});
|
|
604
965
|
if (verified.fingerprint) {
|
|
605
966
|
state.destinationFingerprint = verified.fingerprint;
|
|
606
967
|
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
607
968
|
}
|
|
608
|
-
verified = await destination.
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
}
|
|
612
|
-
const remoteResources = await destination.listUploadedResources(verified);
|
|
613
|
-
for (const resource of resources.filter((item) => item.kind !== 'grid')) {
|
|
614
|
-
const transferred = state.resources[resource.id];
|
|
615
|
-
const remote = transferred && remoteResources.find((item) => (item.id === transferred.destinationId || item.name === transferred.destinationName));
|
|
616
|
-
if (!transferred || !remote) {
|
|
617
|
-
throw new Error(`Verification did not find destination resource for ${resource.name}`);
|
|
618
|
-
}
|
|
619
|
-
if (destination.kind === 'confluence'
|
|
620
|
-
&& transferred.size !== undefined
|
|
621
|
-
&& remote.size !== undefined
|
|
622
|
-
&& transferred.size !== remote.size) {
|
|
623
|
-
throw new Error(`Verification size mismatch for ${resource.name}: ${transferred.size} source bytes, ${remote.size} destination bytes`);
|
|
624
|
-
}
|
|
625
|
-
}
|
|
969
|
+
verified = await verifyConfirmedDestinationPage(destination, verified, this.consistencyOptions);
|
|
970
|
+
const verification = assertDestinationContent(rendered.body, verified.content, destination.kind, cachedPage);
|
|
971
|
+
verificationDegradations = verification.status === 'partial' ? verification.differences : [];
|
|
626
972
|
}
|
|
973
|
+
const partial = rendered.degraded || resourceProblems.length > 0 || verificationDegradations.length > 0;
|
|
974
|
+
const lastResourceProblem = resourceProblems.at(-1);
|
|
975
|
+
const clearedState = { ...state };
|
|
976
|
+
delete clearedState.lastError;
|
|
627
977
|
const finalState = {
|
|
628
|
-
...
|
|
978
|
+
...clearedState,
|
|
629
979
|
destinationId: verified.id,
|
|
630
980
|
destinationCanonicalId: verified.canonicalId,
|
|
631
981
|
destinationLocator: verified.url,
|
|
632
982
|
destinationTitle: verified.title,
|
|
633
983
|
...(verified.fingerprint ? { destinationFingerprint: verified.fingerprint } : {}),
|
|
634
|
-
status: options.dryRun ? 'planned' :
|
|
635
|
-
degradations: [...new Set([...
|
|
984
|
+
status: options.dryRun ? 'planned' : partial ? 'partial' : 'complete',
|
|
985
|
+
degradations: [...new Set([...rendered.warnings, ...resourceProblems, ...verificationDegradations])],
|
|
986
|
+
...(lastResourceProblem ? { lastError: lastResourceProblem } : {}),
|
|
636
987
|
updatedAt: new Date().toISOString(),
|
|
637
988
|
};
|
|
638
989
|
await store.setPage(finalState);
|
|
990
|
+
const fallbackDetails = resources.flatMap((resource) => {
|
|
991
|
+
const resourceState = finalState.resources[resource.id];
|
|
992
|
+
return resourceState?.status === 'failed'
|
|
993
|
+
? [{ name: resource.name, error: resourceState.lastError ?? 'Resource was not transferred' }]
|
|
994
|
+
: [];
|
|
995
|
+
});
|
|
639
996
|
report.addPage(reportPage({
|
|
640
997
|
page,
|
|
641
998
|
destination: verified,
|
|
642
999
|
status: finalState.status,
|
|
643
1000
|
resources,
|
|
644
1001
|
document,
|
|
645
|
-
|
|
646
|
-
|
|
1002
|
+
resourceSizes: finalState.resources,
|
|
1003
|
+
resourceFallbacks: fallbackDetails,
|
|
1004
|
+
warnings: [...rendered.warnings, ...verificationDegradations],
|
|
1005
|
+
normalizations: rendered.normalizations,
|
|
1006
|
+
}), { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(verified.content ?? rendered.body, destination.kind) });
|
|
647
1007
|
}
|
|
648
1008
|
catch (error) {
|
|
649
|
-
errors += 1;
|
|
650
1009
|
const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
|
|
651
|
-
|
|
1010
|
+
let failureStatus = /conflict/i.test(message) ? 'conflict' : 'failed';
|
|
1011
|
+
let actual;
|
|
1012
|
+
if (failureStatus !== 'conflict' && !options.dryRun) {
|
|
1013
|
+
try {
|
|
1014
|
+
actual = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
|
|
1015
|
+
assertDestinationContent(bootstrap.body, actual.content, destination.kind, cachedPage);
|
|
1016
|
+
failureStatus = 'partial';
|
|
1017
|
+
}
|
|
1018
|
+
catch {
|
|
1019
|
+
failureStatus = 'failed';
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
652
1022
|
report.error(message);
|
|
653
|
-
await store.setPage({
|
|
654
|
-
|
|
1023
|
+
await store.setPage({
|
|
1024
|
+
...state,
|
|
1025
|
+
...(actual?.fingerprint ? { destinationFingerprint: actual.fingerprint } : {}),
|
|
1026
|
+
status: failureStatus,
|
|
1027
|
+
lastError: message,
|
|
1028
|
+
updatedAt: new Date().toISOString(),
|
|
1029
|
+
});
|
|
1030
|
+
const fallbackDetails = resources.flatMap((resource) => {
|
|
1031
|
+
const resourceState = state.resources[resource.id];
|
|
1032
|
+
return resourceState?.status === 'failed'
|
|
1033
|
+
? [{ name: resource.name, error: resourceState.lastError ?? 'Resource was not transferred' }]
|
|
1034
|
+
: [];
|
|
1035
|
+
});
|
|
1036
|
+
report.addPage(reportPage({
|
|
1037
|
+
page,
|
|
1038
|
+
destination: actual ?? destinationPage,
|
|
1039
|
+
status: failureStatus,
|
|
1040
|
+
resources,
|
|
1041
|
+
document,
|
|
1042
|
+
resourceSizes: state.resources,
|
|
1043
|
+
resourceFallbacks: fallbackDetails,
|
|
1044
|
+
warnings: resourceProblems,
|
|
1045
|
+
error: message,
|
|
1046
|
+
}), actual?.content ? { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(actual.content, destination.kind) } : undefined);
|
|
655
1047
|
}
|
|
656
1048
|
finally {
|
|
657
1049
|
progress.pages += 1;
|
|
@@ -663,7 +1055,6 @@ export class Migrator {
|
|
|
663
1055
|
const rejected = settled.find((result) => result.status === 'rejected');
|
|
664
1056
|
if (rejected)
|
|
665
1057
|
throw rejected.reason;
|
|
666
|
-
return { errors, warnings };
|
|
667
1058
|
}
|
|
668
1059
|
}
|
|
669
1060
|
//# sourceMappingURL=migrator.js.map
|