@iwonz/ywtc 3.2.0 → 3.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +10 -8
- package/dist/adapters/confluence.d.ts +6 -0
- package/dist/adapters/confluence.js +22 -5
- package/dist/adapters/confluence.js.map +1 -1
- package/dist/adapters/yandex.d.ts +6 -0
- package/dist/adapters/yandex.js +24 -2
- package/dist/adapters/yandex.js.map +1 -1
- package/dist/cli-arguments.js +2 -2
- package/dist/cli-arguments.js.map +1 -1
- package/dist/cli.js +22 -15
- package/dist/cli.js.map +1 -1
- package/dist/clients/confluence.d.ts +1 -0
- package/dist/clients/confluence.js +20 -0
- package/dist/clients/confluence.js.map +1 -1
- package/dist/consistency.d.ts +11 -0
- package/dist/consistency.js +38 -0
- package/dist/consistency.js.map +1 -0
- package/dist/content-model.d.ts +10 -1
- package/dist/content-model.js +356 -40
- package/dist/content-model.js.map +1 -1
- package/dist/domain.d.ts +24 -1
- package/dist/domain.js.map +1 -1
- package/dist/filesystem-export.js +6 -5
- package/dist/filesystem-export.js.map +1 -1
- package/dist/issues.d.ts +23 -0
- package/dist/issues.js +31 -0
- package/dist/issues.js.map +1 -0
- package/dist/migrator.d.ts +5 -14
- package/dist/migrator.js +494 -127
- package/dist/migrator.js.map +1 -1
- package/dist/report-catalog.d.ts +30 -0
- package/dist/report-catalog.js +57 -7
- package/dist/report-catalog.js.map +1 -1
- package/dist/report.d.ts +79 -16
- package/dist/report.js +281 -27
- package/dist/report.js.map +1 -1
- package/dist/resource-reconciliation.d.ts +21 -0
- package/dist/resource-reconciliation.js +72 -0
- package/dist/resource-reconciliation.js.map +1 -0
- package/dist/state.d.ts +3 -0
- package/dist/state.js +3 -0
- package/dist/state.js.map +1 -1
- package/dist/types.d.ts +1 -0
- package/package.json +1 -1
package/dist/migrator.js
CHANGED
|
@@ -1,15 +1,16 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import PQueue from 'p-queue';
|
|
5
5
|
import { createProviderAdapter } from './adapters/factory.js';
|
|
6
|
-
import { appendGridSnapshots, assertDestinationContent, isLegacyTransferPlaceholder, normalizePageContent, normalizedPreviewHtml, rawContentFallback, renderedPreviewHtml, renderConfluenceStorage, renderYfm, sourceResourceBindings, } from './content-model.js';
|
|
6
|
+
import { appendGridSnapshots, assertDestinationContent, isLegacyTransferPlaceholder, normalizePageContent, normalizedPreviewHtml, rawContentFallback, renderedPreviewHtml, renderConfluenceStorage, renderYfm, setSourceResourceBinding, sourceResourceBindings, verifyDestinationContent, } from './content-model.js';
|
|
7
|
+
import { ConfirmedDestinationMissingError, verifyConfirmedDestinationPage } from './consistency.js';
|
|
7
8
|
import { assertTransferCapabilities } from './domain.js';
|
|
8
9
|
import { FilesystemExporter } from './filesystem-export.js';
|
|
9
10
|
import { discoverPageGraph } from './graph.js';
|
|
10
|
-
import { safeFilename } from './input.js';
|
|
11
11
|
import { profileBaseUrl, sameRemoteInstance } from './locators.js';
|
|
12
|
-
import { TransferReport } from './report.js';
|
|
12
|
+
import { TransferReport, } from './report.js';
|
|
13
|
+
import { DestinationNameAllocator, preferredDestinationName, verifiedCacheFile } from './resource-reconciliation.js';
|
|
13
14
|
import { resolveStateLocation, StateStore, } from './state.js';
|
|
14
15
|
function sourceIdentity(locator, root) {
|
|
15
16
|
return {
|
|
@@ -69,7 +70,37 @@ function collisionTitle(page) {
|
|
|
69
70
|
function cachePath(cacheDirectory, page, resource) {
|
|
70
71
|
const pageKey = sha256(page.canonicalId).slice(0, 12);
|
|
71
72
|
const resourceKey = sha256(`${resource.id}:${resource.name}`).slice(0, 12);
|
|
72
|
-
return path.join(cacheDirectory, pageKey, `${resourceKey}-${
|
|
73
|
+
return path.join(cacheDirectory, pageKey, `${resourceKey}-${preferredDestinationName(resource)}`);
|
|
74
|
+
}
|
|
75
|
+
function containedCachePath(cacheDirectory, candidate) {
|
|
76
|
+
if (!candidate)
|
|
77
|
+
return undefined;
|
|
78
|
+
const relative = path.relative(path.resolve(cacheDirectory), path.resolve(candidate));
|
|
79
|
+
return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)
|
|
80
|
+
? path.resolve(candidate)
|
|
81
|
+
: undefined;
|
|
82
|
+
}
|
|
83
|
+
function resetResourceDestinations(resources) {
|
|
84
|
+
return Object.fromEntries(Object.entries(resources).map(([id, resource]) => [id, {
|
|
85
|
+
sourceId: resource.sourceId,
|
|
86
|
+
sourceName: resource.sourceName,
|
|
87
|
+
destinationName: resource.destinationName,
|
|
88
|
+
...(resource.sha256 ? { sha256: resource.sha256 } : {}),
|
|
89
|
+
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
90
|
+
...(resource.cachePath ? { cachePath: resource.cachePath } : {}),
|
|
91
|
+
...(resource.mimeType ? { mimeType: resource.mimeType } : {}),
|
|
92
|
+
status: 'planned',
|
|
93
|
+
updatedAt: new Date().toISOString(),
|
|
94
|
+
}]));
|
|
95
|
+
}
|
|
96
|
+
function sourceDerivedTextMatches(expectedBody, actualBody, provider, page) {
|
|
97
|
+
if (actualBody === undefined)
|
|
98
|
+
return false;
|
|
99
|
+
const result = verifyDestinationContent(expectedBody, actualBody, provider, page);
|
|
100
|
+
if (result.status === 'complete')
|
|
101
|
+
return true;
|
|
102
|
+
const text = (value) => renderedPreviewHtml(value, provider).replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
|
|
103
|
+
return text(expectedBody) === text(actualBody);
|
|
73
104
|
}
|
|
74
105
|
function destinationFromState(state, adapter) {
|
|
75
106
|
if (!state.destinationId || !state.destinationCanonicalId || !state.destinationLocator) {
|
|
@@ -87,6 +118,25 @@ function destinationFromState(state, adapter) {
|
|
|
87
118
|
};
|
|
88
119
|
}
|
|
89
120
|
function reportPage(input) {
|
|
121
|
+
const transferable = input.resources.filter((resource) => resource.kind !== 'grid');
|
|
122
|
+
const resourceItems = transferable.map((resource) => {
|
|
123
|
+
const state = input.resourceStates?.[resource.id];
|
|
124
|
+
const sourceBytes = state?.size ?? resource.size;
|
|
125
|
+
const status = input.resourceStatusOverride ?? state?.status ?? 'planned';
|
|
126
|
+
return {
|
|
127
|
+
sourceId: resource.id,
|
|
128
|
+
sourceName: resource.name,
|
|
129
|
+
kind: resource.kind,
|
|
130
|
+
status,
|
|
131
|
+
...(sourceBytes !== undefined ? { sourceBytes } : {}),
|
|
132
|
+
...(status === 'complete' && sourceBytes !== undefined ? { transferredBytes: sourceBytes } : {}),
|
|
133
|
+
...(state?.destinationId ? { destinationId: state.destinationId } : {}),
|
|
134
|
+
...(state?.destinationName ? { destinationName: state.destinationName } : {}),
|
|
135
|
+
...(state?.lastError ? { error: state.lastError } : {}),
|
|
136
|
+
};
|
|
137
|
+
});
|
|
138
|
+
const knownBytes = resourceItems.reduce((total, resource) => total + (resource.sourceBytes ?? 0), 0);
|
|
139
|
+
const transferredBytes = resourceItems.reduce((total, resource) => total + (resource.transferredBytes ?? 0), 0);
|
|
90
140
|
return {
|
|
91
141
|
sourceCanonicalId: input.page.canonicalId,
|
|
92
142
|
sourceId: input.page.id,
|
|
@@ -100,22 +150,31 @@ function reportPage(input) {
|
|
|
100
150
|
...(input.page.projectedParentId ? { parentId: input.page.projectedParentId } : {}),
|
|
101
151
|
depth: input.page.depth,
|
|
102
152
|
status: input.status,
|
|
103
|
-
|
|
104
|
-
|
|
153
|
+
inventoryStatus: input.inventoryStatus ?? 'complete',
|
|
154
|
+
resources: transferable.length,
|
|
155
|
+
resourceItems,
|
|
156
|
+
...(transferable.length === 0 || resourceItems.some((resource) => resource.sourceBytes !== undefined) ? { sourceBytes: knownBytes } : {}),
|
|
157
|
+
transferredBytes,
|
|
158
|
+
bytesUnknown: resourceItems.filter((resource) => resource.sourceBytes === undefined).length,
|
|
105
159
|
links: input.document.metrics.links,
|
|
106
160
|
images: input.document.metrics.images,
|
|
107
161
|
tables: input.document.metrics.tables,
|
|
162
|
+
grids: input.resources.filter((resource) => resource.kind === 'grid').length,
|
|
108
163
|
codeBlocks: input.document.metrics.codeBlocks,
|
|
109
164
|
degraded: input.document.degraded || (input.resourceFallbacks?.length ?? 0) > 0,
|
|
110
165
|
resourceFallbacks: input.resourceFallbacks ?? [],
|
|
111
|
-
warnings: [...input.document.warnings, ...(input.warnings ?? [])],
|
|
166
|
+
warnings: [...new Set([...input.document.warnings, ...(input.warnings ?? [])])],
|
|
167
|
+
normalizations: [...new Set([...(input.document.normalizations ?? []), ...(input.normalizations ?? [])])],
|
|
112
168
|
...(input.error ? { error: input.error } : {}),
|
|
113
169
|
};
|
|
114
170
|
}
|
|
115
|
-
function runStatus(errors, warnings) {
|
|
171
|
+
function runStatus(errors, warnings, degradations) {
|
|
116
172
|
if (errors > 0)
|
|
117
173
|
return 'partial';
|
|
118
|
-
return warnings > 0 ? 'completed-with-warnings' : 'completed';
|
|
174
|
+
return warnings + degradations > 0 ? 'completed-with-warnings' : 'completed';
|
|
175
|
+
}
|
|
176
|
+
function attentionCount(summary) {
|
|
177
|
+
return summary.warnings + summary.degradations;
|
|
119
178
|
}
|
|
120
179
|
export class MigrationRunError extends Error {
|
|
121
180
|
reportPaths;
|
|
@@ -144,10 +203,12 @@ export class Migrator {
|
|
|
144
203
|
config;
|
|
145
204
|
logger;
|
|
146
205
|
adapterFactory;
|
|
147
|
-
|
|
206
|
+
consistencyOptions;
|
|
207
|
+
constructor(config, logger, adapterFactory = createProviderAdapter, consistencyOptions = {}) {
|
|
148
208
|
this.config = config;
|
|
149
209
|
this.logger = logger;
|
|
150
210
|
this.adapterFactory = adapterFactory;
|
|
211
|
+
this.consistencyOptions = consistencyOptions;
|
|
151
212
|
}
|
|
152
213
|
async run(options) {
|
|
153
214
|
const source = this.adapterFactory(options.source.profile, this.config);
|
|
@@ -195,6 +256,7 @@ export class Migrator {
|
|
|
195
256
|
...(store.loadInfo.backupFilename ? { backupPath: store.loadInfo.backupFilename } : {}),
|
|
196
257
|
},
|
|
197
258
|
});
|
|
259
|
+
report.registerPages([{ ...root, depth: 0 }]);
|
|
198
260
|
let graph;
|
|
199
261
|
let sourceFingerprints;
|
|
200
262
|
try {
|
|
@@ -206,6 +268,10 @@ export class Migrator {
|
|
|
206
268
|
...(options.signal ? { signal: options.signal } : {}),
|
|
207
269
|
onProgress: (progress) => status(`Discovery: ${progress.pages} pages, ${progress.edges} links${progress.active ? ` · ${progress.active}` : ''}`),
|
|
208
270
|
});
|
|
271
|
+
report.registerPages(graph.pages);
|
|
272
|
+
report.setEdges(graph.edges);
|
|
273
|
+
for (const warning of graph.warnings)
|
|
274
|
+
report.warning(warning);
|
|
209
275
|
if (container
|
|
210
276
|
&& options.destination.type === 'remote'
|
|
211
277
|
&& sameRemoteInstance(options.source.profile, options.destination.profile)
|
|
@@ -215,23 +281,32 @@ export class Migrator {
|
|
|
215
281
|
status(`Caching source content for ${graph.pages.length} pages`);
|
|
216
282
|
sourceFingerprints = await cacheDiscoveredPages(options.cacheDirectory, graph);
|
|
217
283
|
await store.setGraph(graph.rootId, graph.edges);
|
|
218
|
-
report.setEdges(graph.edges);
|
|
219
|
-
for (const warning of graph.warnings)
|
|
220
|
-
report.warning(warning);
|
|
221
284
|
}
|
|
222
285
|
catch (error) {
|
|
223
286
|
const finalStatus = options.signal?.aborted ? 'cancelled' : 'failed';
|
|
224
287
|
const message = error instanceof Error ? error.message : String(error);
|
|
225
|
-
report.error
|
|
288
|
+
report.updatePage(root.canonicalId, { status: 'failed', inventoryStatus: 'failed', error: message });
|
|
289
|
+
report.issue({
|
|
290
|
+
code: 'source-discovery-failed',
|
|
291
|
+
severity: 'error',
|
|
292
|
+
phase: 'discovery',
|
|
293
|
+
pageId: root.canonicalId,
|
|
294
|
+
retryable: true,
|
|
295
|
+
message,
|
|
296
|
+
});
|
|
226
297
|
report.finish(finalStatus);
|
|
227
298
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
228
299
|
if (!options.dryRun)
|
|
229
|
-
await store.finishRun(run.id, finalStatus, {
|
|
300
|
+
await store.finishRun(run.id, finalStatus, {
|
|
301
|
+
pages: 0,
|
|
302
|
+
warnings: attentionCount(report.data.summary),
|
|
303
|
+
errors: report.data.summary.errors,
|
|
304
|
+
});
|
|
230
305
|
throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
|
|
231
306
|
}
|
|
232
307
|
const resourcesByPage = new Map();
|
|
233
308
|
const inventoryQueue = new PQueue({ concurrency: options.concurrency });
|
|
234
|
-
const inventory = { resources: 0, bytes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
|
|
309
|
+
const inventory = { resources: 0, bytes: 0, unknownSizes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
|
|
235
310
|
status(`Inspecting content and resources for ${graph.pages.length} pages`);
|
|
236
311
|
const inventoryErrors = [];
|
|
237
312
|
await Promise.all(graph.pages.map((page) => inventoryQueue.add(async () => {
|
|
@@ -245,17 +320,32 @@ export class Migrator {
|
|
|
245
320
|
await cacheDocument(options.cacheDirectory, page.canonicalId, document);
|
|
246
321
|
inventory.resources += resources.filter((resource) => resource.kind !== 'grid').length;
|
|
247
322
|
inventory.bytes += resources.reduce((total, resource) => total + (resource.size ?? 0), 0);
|
|
323
|
+
inventory.unknownSizes += resources.filter((resource) => resource.kind !== 'grid' && resource.size === undefined).length;
|
|
248
324
|
inventory.grids += resources.filter((resource) => resource.kind === 'grid').length;
|
|
249
325
|
inventory.links += document.metrics.links;
|
|
250
326
|
inventory.images += document.metrics.images;
|
|
251
327
|
inventory.tables += document.metrics.tables;
|
|
252
328
|
inventory.codeBlocks += document.metrics.codeBlocks;
|
|
329
|
+
report.addPage(reportPage({
|
|
330
|
+
page,
|
|
331
|
+
status: 'planned',
|
|
332
|
+
resources,
|
|
333
|
+
document,
|
|
334
|
+
}), undefined, false);
|
|
253
335
|
status(`Inspected ${page.title} · ${resources.length} resources`);
|
|
254
336
|
}
|
|
255
337
|
catch (error) {
|
|
256
338
|
const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
|
|
257
339
|
inventoryErrors.push(message);
|
|
258
|
-
report.error
|
|
340
|
+
report.updatePage(page.canonicalId, { status: 'failed', inventoryStatus: 'failed', error: message });
|
|
341
|
+
report.issue({
|
|
342
|
+
code: 'source-inventory-failed',
|
|
343
|
+
severity: 'error',
|
|
344
|
+
phase: 'inventory',
|
|
345
|
+
pageId: page.canonicalId,
|
|
346
|
+
retryable: true,
|
|
347
|
+
message,
|
|
348
|
+
});
|
|
259
349
|
}
|
|
260
350
|
})));
|
|
261
351
|
if (inventoryErrors.length > 0) {
|
|
@@ -263,23 +353,23 @@ export class Migrator {
|
|
|
263
353
|
report.finish(finalStatus);
|
|
264
354
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
265
355
|
if (!options.dryRun)
|
|
266
|
-
await store.finishRun(run.id, finalStatus, {
|
|
356
|
+
await store.finishRun(run.id, finalStatus, {
|
|
357
|
+
pages: graph.pages.length,
|
|
358
|
+
warnings: attentionCount(report.data.summary),
|
|
359
|
+
errors: report.data.summary.errors,
|
|
360
|
+
});
|
|
267
361
|
throw new MigrationRunError('Resource preflight failed before destination mutation', reportPaths, stateLocation.filename);
|
|
268
362
|
}
|
|
269
|
-
const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} bytes) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
|
|
363
|
+
const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} known bytes${inventory.unknownSizes > 0 ? ` · ${inventory.unknownSizes} sizes unknown` : ''}) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
|
|
270
364
|
status(planSummary);
|
|
271
365
|
this.logger.info(planSummary);
|
|
272
366
|
let output;
|
|
273
367
|
let archive;
|
|
274
|
-
let transferErrors = 0;
|
|
275
|
-
let transferWarnings = graph.warnings.length;
|
|
276
368
|
try {
|
|
277
369
|
if (options.destination.type === 'directory') {
|
|
278
370
|
const result = await this.exportFilesystem(source, graph, options.destination, options, stateLocation.id, store, report, sourceFingerprints, resourcesByPage);
|
|
279
371
|
output = result.directory;
|
|
280
372
|
archive = result.archive;
|
|
281
|
-
transferErrors += result.errors.length;
|
|
282
|
-
transferWarnings += result.warnings.length;
|
|
283
373
|
report.output('directory', result.directory);
|
|
284
374
|
if (result.archive)
|
|
285
375
|
report.output('archive', result.archive);
|
|
@@ -287,7 +377,7 @@ export class Migrator {
|
|
|
287
377
|
report.output('backup', result.backup);
|
|
288
378
|
}
|
|
289
379
|
else {
|
|
290
|
-
|
|
380
|
+
await this.transferRemote({
|
|
291
381
|
source,
|
|
292
382
|
destination: destination,
|
|
293
383
|
container: container,
|
|
@@ -298,27 +388,30 @@ export class Migrator {
|
|
|
298
388
|
sourceFingerprints,
|
|
299
389
|
resourcesByPage,
|
|
300
390
|
});
|
|
301
|
-
transferErrors += result.errors;
|
|
302
|
-
transferWarnings += result.warnings;
|
|
303
391
|
output = container.url;
|
|
304
392
|
report.output('destination', container.url);
|
|
305
393
|
}
|
|
306
|
-
|
|
394
|
+
report.finish('completed');
|
|
395
|
+
const finalStatus = runStatus(report.data.summary.errors, report.data.summary.warnings, report.data.summary.degradations);
|
|
307
396
|
report.finish(finalStatus);
|
|
308
397
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
309
398
|
if (!options.dryRun)
|
|
310
|
-
await store.finishRun(run.id, finalStatus, {
|
|
311
|
-
|
|
399
|
+
await store.finishRun(run.id, finalStatus, {
|
|
400
|
+
pages: graph.pages.length,
|
|
401
|
+
warnings: attentionCount(report.data.summary),
|
|
402
|
+
errors: report.data.summary.errors,
|
|
403
|
+
});
|
|
404
|
+
status(`${finalStatus}: source ${report.data.summary.pages} pages · ${report.data.summary.complete} complete · ${report.data.summary.partial} partial · ${report.data.summary.failed + report.data.summary.blocked} failed/blocked · files ${report.data.summary.resourcesComplete}/${report.data.summary.resources} complete · ${report.data.summary.errors} errors · ${report.data.summary.warnings} warnings · ${report.data.summary.degradations} degradations`);
|
|
312
405
|
return {
|
|
313
406
|
status: finalStatus,
|
|
314
|
-
hasErrors:
|
|
407
|
+
hasErrors: report.data.summary.errors > 0,
|
|
315
408
|
reportPaths,
|
|
316
409
|
stateId: stateLocation.id,
|
|
317
410
|
stateFilename: stateLocation.filename,
|
|
318
411
|
...(store.loadInfo.backupFilename ? { stateBackupFilename: store.loadInfo.backupFilename } : {}),
|
|
319
412
|
...(output ? { output } : {}),
|
|
320
413
|
...(archive ? { archive } : {}),
|
|
321
|
-
summary:
|
|
414
|
+
summary: report.data.summary,
|
|
322
415
|
};
|
|
323
416
|
}
|
|
324
417
|
catch (error) {
|
|
@@ -329,7 +422,11 @@ export class Migrator {
|
|
|
329
422
|
report.finish(finalStatus);
|
|
330
423
|
const reportPaths = await report.write(options.reportsDirectory);
|
|
331
424
|
if (!options.dryRun)
|
|
332
|
-
await store.finishRun(run.id, finalStatus, {
|
|
425
|
+
await store.finishRun(run.id, finalStatus, {
|
|
426
|
+
pages: graph.pages.length,
|
|
427
|
+
warnings: attentionCount(report.data.summary),
|
|
428
|
+
errors: report.data.summary.errors,
|
|
429
|
+
});
|
|
333
430
|
this.logger.error(`${message}. Report: ${reportPaths.html}`);
|
|
334
431
|
throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
|
|
335
432
|
}
|
|
@@ -360,16 +457,23 @@ export class Migrator {
|
|
|
360
457
|
const document = await loadCachedDocument(options.cacheDirectory, page.canonicalId);
|
|
361
458
|
const manifestPage = result.manifest.pages.find((item) => item.sourceCanonicalId === page.canonicalId);
|
|
362
459
|
const previous = store.value.pages[page.canonicalId];
|
|
363
|
-
const
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
460
|
+
const sourceResources = resourcesByPage.get(page.canonicalId) ?? [];
|
|
461
|
+
const transferableResources = sourceResources.filter((resource) => resource.kind !== 'grid');
|
|
462
|
+
const resources = Object.fromEntries((manifestPage?.resources ?? []).map((resource, index) => {
|
|
463
|
+
const sourceResource = transferableResources[index];
|
|
464
|
+
const sourceId = sourceResource?.id ?? String(index);
|
|
465
|
+
return [sourceId, {
|
|
466
|
+
sourceId,
|
|
467
|
+
sourceName: sourceResource?.name ?? resource.name,
|
|
468
|
+
destinationId: resource.path,
|
|
469
|
+
destinationName: resource.name,
|
|
470
|
+
...(resource.sha256 ? { sha256: resource.sha256 } : {}),
|
|
471
|
+
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
472
|
+
status: options.dryRun ? 'planned' : 'complete',
|
|
473
|
+
updatedAt: new Date().toISOString(),
|
|
474
|
+
}];
|
|
475
|
+
}));
|
|
476
|
+
const pageStatus = options.dryRun ? 'planned' : manifestPage?.degraded ? 'partial' : 'complete';
|
|
373
477
|
await store.setPage({
|
|
374
478
|
sourceId: page.id,
|
|
375
479
|
sourceCanonicalId: page.canonicalId,
|
|
@@ -382,16 +486,18 @@ export class Migrator {
|
|
|
382
486
|
destinationTitle: page.title,
|
|
383
487
|
sourceFingerprint: sourceFingerprints.get(page.canonicalId),
|
|
384
488
|
...(manifestPage || previous?.destinationFingerprint ? { destinationFingerprint: manifestPage ? sha256(JSON.stringify(manifestPage)) : previous.destinationFingerprint } : {}),
|
|
385
|
-
status:
|
|
489
|
+
status: pageStatus,
|
|
386
490
|
resources,
|
|
387
491
|
degradations: manifestPage?.warnings ?? [],
|
|
388
492
|
updatedAt: new Date().toISOString(),
|
|
389
493
|
});
|
|
390
494
|
report.addPage(reportPage({
|
|
391
495
|
page,
|
|
392
|
-
status:
|
|
393
|
-
resources:
|
|
496
|
+
status: pageStatus,
|
|
497
|
+
resources: sourceResources,
|
|
394
498
|
document,
|
|
499
|
+
resourceStates: resources,
|
|
500
|
+
...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
|
|
395
501
|
...(manifestPage?.warnings ? { warnings: manifestPage.warnings } : {}),
|
|
396
502
|
}), { source: normalizedPreviewHtml(document), destination: normalizedPreviewHtml(document) });
|
|
397
503
|
}
|
|
@@ -404,10 +510,9 @@ export class Migrator {
|
|
|
404
510
|
async transferRemote(input) {
|
|
405
511
|
const { source, destination, container, graph, options, store, report, sourceFingerprints, resourcesByPage } = input;
|
|
406
512
|
const destinationPages = new Map();
|
|
513
|
+
const stagingFailures = new Map();
|
|
407
514
|
const sourcePages = new Map(graph.pages.map((page) => [page.canonicalId, page]));
|
|
408
515
|
const bootstrapBodies = new Map();
|
|
409
|
-
let errors = 0;
|
|
410
|
-
let warnings = 0;
|
|
411
516
|
const totalResources = [...resourcesByPage.values()].reduce((total, resources) => total + resources.filter((resource) => resource.kind !== 'grid').length, 0);
|
|
412
517
|
const progress = { staged: 0, pages: 0, resources: 0 };
|
|
413
518
|
const totalUnits = Math.max(1, graph.pages.length * 2 + totalResources);
|
|
@@ -441,8 +546,16 @@ export class Migrator {
|
|
|
441
546
|
const parent = page.projectedParentId ? destinationPages.get(page.projectedParentId) : container;
|
|
442
547
|
if (!parent) {
|
|
443
548
|
const message = `Destination parent is unavailable for ${page.title}`;
|
|
444
|
-
|
|
445
|
-
report.error
|
|
549
|
+
stagingFailures.set(page.canonicalId, { status: 'blocked', message });
|
|
550
|
+
report.updatePage(page.canonicalId, { status: 'blocked', error: message });
|
|
551
|
+
report.issue({
|
|
552
|
+
code: 'destination-parent-unavailable',
|
|
553
|
+
severity: 'error',
|
|
554
|
+
phase: 'page',
|
|
555
|
+
pageId: page.canonicalId,
|
|
556
|
+
retryable: true,
|
|
557
|
+
message,
|
|
558
|
+
});
|
|
446
559
|
progress.staged += 1;
|
|
447
560
|
updateProgress(message);
|
|
448
561
|
continue;
|
|
@@ -452,11 +565,99 @@ export class Migrator {
|
|
|
452
565
|
const bootstrap = bootstrapBodies.get(page.canonicalId);
|
|
453
566
|
let created;
|
|
454
567
|
let title = saved?.destinationTitle ?? page.title;
|
|
568
|
+
let resourcesForState = saved?.resources ?? {};
|
|
569
|
+
let previousDestinationIds = saved?.previousDestinationIds ?? [];
|
|
570
|
+
let mappingRecovered = false;
|
|
455
571
|
if (saved?.destinationId) {
|
|
456
|
-
|
|
572
|
+
const mapped = destinationFromState(saved, destination);
|
|
573
|
+
try {
|
|
574
|
+
created = await verifyConfirmedDestinationPage(destination, mapped, {
|
|
575
|
+
...this.consistencyOptions,
|
|
576
|
+
onRetry: (attempt, delayMs) => report.issue({
|
|
577
|
+
code: 'destination-consistency-retry',
|
|
578
|
+
severity: 'warning',
|
|
579
|
+
phase: 'verification',
|
|
580
|
+
pageId: page.canonicalId,
|
|
581
|
+
retryable: true,
|
|
582
|
+
message: `${page.title}: destination page ${mapped.id} returned 404; consistency retry ${attempt} in ${delayMs} ms`,
|
|
583
|
+
}),
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
catch (error) {
|
|
587
|
+
if (!(error instanceof ConfirmedDestinationMissingError) || options.dryRun)
|
|
588
|
+
throw error;
|
|
589
|
+
if (page.projectedParentId) {
|
|
590
|
+
await verifyConfirmedDestinationPage(destination, parent, this.consistencyOptions);
|
|
591
|
+
}
|
|
592
|
+
const findRecoveryChild = (candidateTitle) => destination.findRecoverablePage
|
|
593
|
+
? destination.findRecoverablePage(parent, candidateTitle)
|
|
594
|
+
: destination.findChild(parent, candidateTitle);
|
|
595
|
+
let candidate;
|
|
596
|
+
let recoveryLookupFailed = false;
|
|
597
|
+
try {
|
|
598
|
+
candidate = await findRecoveryChild(title);
|
|
599
|
+
}
|
|
600
|
+
catch (lookupError) {
|
|
601
|
+
recoveryLookupFailed = true;
|
|
602
|
+
report.issue({
|
|
603
|
+
code: 'destination-recovery-lookup-failed',
|
|
604
|
+
severity: 'warning',
|
|
605
|
+
phase: 'page',
|
|
606
|
+
pageId: page.canonicalId,
|
|
607
|
+
retryable: true,
|
|
608
|
+
message: `${page.title}: recovery lookup failed; bootstrap recreation will be attempted: ${lookupError instanceof Error ? lookupError.message : String(lookupError)}`,
|
|
609
|
+
});
|
|
610
|
+
}
|
|
611
|
+
if (candidate && sourceDerivedTextMatches(bootstrap.body, candidate.content, destination.kind, cachedPage)) {
|
|
612
|
+
created = candidate;
|
|
613
|
+
}
|
|
614
|
+
else {
|
|
615
|
+
let recoveryTitle = title;
|
|
616
|
+
if (candidate) {
|
|
617
|
+
const base = collisionTitle(page);
|
|
618
|
+
const hash = sha256(page.canonicalId).slice(0, 8);
|
|
619
|
+
recoveryTitle = base;
|
|
620
|
+
let sequence = 0;
|
|
621
|
+
while (!recoveryLookupFailed) {
|
|
622
|
+
try {
|
|
623
|
+
if (!await findRecoveryChild(recoveryTitle))
|
|
624
|
+
break;
|
|
625
|
+
sequence += 1;
|
|
626
|
+
recoveryTitle = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
|
|
627
|
+
}
|
|
628
|
+
catch (lookupError) {
|
|
629
|
+
recoveryLookupFailed = true;
|
|
630
|
+
recoveryTitle = `${base} [${hash}]`;
|
|
631
|
+
report.issue({
|
|
632
|
+
code: 'destination-recovery-lookup-failed',
|
|
633
|
+
severity: 'warning',
|
|
634
|
+
phase: 'page',
|
|
635
|
+
pageId: page.canonicalId,
|
|
636
|
+
retryable: true,
|
|
637
|
+
message: `${page.title}: suffixed-title lookup failed; bootstrap recreation will be attempted: ${lookupError instanceof Error ? lookupError.message : String(lookupError)}`,
|
|
638
|
+
});
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
created = await destination.createPage({ container: parent, title: recoveryTitle, body: bootstrap.body });
|
|
643
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
644
|
+
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
645
|
+
}
|
|
646
|
+
previousDestinationIds = [...new Set([...previousDestinationIds, mapped.id])];
|
|
647
|
+
resourcesForState = resetResourceDestinations(saved.resources);
|
|
648
|
+
mappingRecovered = true;
|
|
649
|
+
report.issue({
|
|
650
|
+
code: 'destination-mapping-recovered',
|
|
651
|
+
severity: 'warning',
|
|
652
|
+
phase: 'page',
|
|
653
|
+
pageId: page.canonicalId,
|
|
654
|
+
retryable: false,
|
|
655
|
+
message: `${page.title}: replaced unavailable destination mapping ${mapped.id} with ${created.id}`,
|
|
656
|
+
});
|
|
657
|
+
}
|
|
457
658
|
const generatedPlaceholder = created.title === saved.destinationTitle
|
|
458
659
|
&& isLegacyTransferPlaceholder(created.content, destination.kind, cachedPage);
|
|
459
|
-
if (['complete', 'partial'].includes(saved.status) && saved.destinationFingerprint && created.fingerprint !== saved.destinationFingerprint && !generatedPlaceholder) {
|
|
660
|
+
if (!mappingRecovered && ['complete', 'partial'].includes(saved.status) && saved.destinationFingerprint && created.fingerprint !== saved.destinationFingerprint && !generatedPlaceholder) {
|
|
460
661
|
throw new Error(`Destination conflict for ${page.title}: content changed since the last verified run`);
|
|
461
662
|
}
|
|
462
663
|
if (generatedPlaceholder && !options.dryRun) {
|
|
@@ -466,7 +667,7 @@ export class Migrator {
|
|
|
466
667
|
body: bootstrap.body,
|
|
467
668
|
...(created.fingerprint ? { expectedFingerprint: created.fingerprint } : {}),
|
|
468
669
|
});
|
|
469
|
-
created = await destination.
|
|
670
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
470
671
|
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
471
672
|
}
|
|
472
673
|
}
|
|
@@ -496,7 +697,7 @@ export class Migrator {
|
|
|
496
697
|
}
|
|
497
698
|
}
|
|
498
699
|
created = await destination.createPage({ container: parent, title, body: bootstrap.body });
|
|
499
|
-
created = await destination.
|
|
700
|
+
created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
|
|
500
701
|
assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
|
|
501
702
|
}
|
|
502
703
|
destinationPages.set(page.canonicalId, created);
|
|
@@ -510,10 +711,11 @@ export class Migrator {
|
|
|
510
711
|
destinationCanonicalId: created.canonicalId,
|
|
511
712
|
destinationLocator: created.url,
|
|
512
713
|
destinationTitle: created.title,
|
|
714
|
+
...(previousDestinationIds.length > 0 ? { previousDestinationIds } : {}),
|
|
513
715
|
sourceFingerprint: sourceFingerprints.get(page.canonicalId),
|
|
514
716
|
...(created.fingerprint ? { destinationFingerprint: created.fingerprint } : {}),
|
|
515
717
|
status: options.dryRun ? 'planned' : saved?.status ?? 'skeleton-created',
|
|
516
|
-
resources:
|
|
718
|
+
resources: resourcesForState,
|
|
517
719
|
degradations: saved?.degradations ?? [],
|
|
518
720
|
updatedAt: new Date().toISOString(),
|
|
519
721
|
};
|
|
@@ -521,12 +723,21 @@ export class Migrator {
|
|
|
521
723
|
options.onStatus?.(`${options.dryRun ? 'Planned' : 'Staged content for'} ${page.title}`);
|
|
522
724
|
}
|
|
523
725
|
catch (error) {
|
|
524
|
-
errors += 1;
|
|
525
726
|
const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
|
|
526
|
-
|
|
727
|
+
const failureStatus = /conflict/i.test(message) ? 'conflict' : 'failed';
|
|
728
|
+
stagingFailures.set(page.canonicalId, { status: failureStatus, message });
|
|
729
|
+
report.updatePage(page.canonicalId, { status: failureStatus, error: message });
|
|
730
|
+
report.issue({
|
|
731
|
+
code: failureStatus === 'conflict' ? 'destination-conflict' : 'destination-page-staging-failed',
|
|
732
|
+
severity: 'error',
|
|
733
|
+
phase: 'page',
|
|
734
|
+
pageId: page.canonicalId,
|
|
735
|
+
retryable: failureStatus !== 'conflict',
|
|
736
|
+
message,
|
|
737
|
+
});
|
|
527
738
|
const savedState = store.value.pages[page.canonicalId];
|
|
528
739
|
if (savedState)
|
|
529
|
-
await store.setPage({ ...savedState, status:
|
|
740
|
+
await store.setPage({ ...savedState, status: failureStatus, lastError: message, updatedAt: new Date().toISOString() });
|
|
530
741
|
}
|
|
531
742
|
finally {
|
|
532
743
|
progress.staged += 1;
|
|
@@ -542,13 +753,16 @@ export class Migrator {
|
|
|
542
753
|
const cachedPage = await loadCachedPage(options.cacheDirectory, page.canonicalId);
|
|
543
754
|
const resources = resourcesByPage.get(page.canonicalId) ?? [];
|
|
544
755
|
if (!destinationPage) {
|
|
756
|
+
const currentFailure = stagingFailures.get(page.canonicalId);
|
|
545
757
|
const failedState = store.value.pages[page.canonicalId];
|
|
546
758
|
report.addPage(reportPage({
|
|
547
759
|
page,
|
|
548
|
-
status:
|
|
760
|
+
status: currentFailure?.status ?? 'blocked',
|
|
549
761
|
resources,
|
|
550
762
|
document,
|
|
551
|
-
|
|
763
|
+
...(failedState?.resources ? { resourceStates: failedState.resources } : {}),
|
|
764
|
+
...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
|
|
765
|
+
error: currentFailure?.message ?? failedState?.lastError ?? 'Destination page with source-derived content was not created',
|
|
552
766
|
}));
|
|
553
767
|
progress.pages += 1;
|
|
554
768
|
updateProgress(`blocked ${page.title}`);
|
|
@@ -557,23 +771,33 @@ export class Migrator {
|
|
|
557
771
|
const state = store.value.pages[page.canonicalId];
|
|
558
772
|
const bootstrap = bootstrapBodies.get(page.canonicalId);
|
|
559
773
|
const bindings = sourceResourceBindings(cachedPage, resources);
|
|
774
|
+
const transferableResources = resources.filter((item) => item.kind !== 'grid');
|
|
560
775
|
const resourceProblems = [];
|
|
561
776
|
const failedResourceIds = new Set();
|
|
562
777
|
const recordResourceFailure = async (resource, detail) => {
|
|
563
778
|
if (!failedResourceIds.has(resource.id)) {
|
|
564
779
|
failedResourceIds.add(resource.id);
|
|
565
|
-
errors += 1;
|
|
566
780
|
resourceProblems.push(detail);
|
|
567
|
-
report.
|
|
781
|
+
report.issue({
|
|
782
|
+
code: 'resource-transfer-failed',
|
|
783
|
+
severity: 'error',
|
|
784
|
+
phase: 'resource',
|
|
785
|
+
pageId: page.canonicalId,
|
|
786
|
+
resourceId: resource.id,
|
|
787
|
+
retryable: true,
|
|
788
|
+
message: `${page.title}: ${detail}`,
|
|
789
|
+
});
|
|
568
790
|
}
|
|
569
791
|
const previous = state.resources[resource.id];
|
|
570
792
|
state.resources[resource.id] = {
|
|
571
793
|
sourceId: resource.id,
|
|
572
794
|
sourceName: resource.name,
|
|
573
795
|
...(previous?.destinationId ? { destinationId: previous.destinationId } : {}),
|
|
574
|
-
destinationName: previous?.destinationName ??
|
|
796
|
+
destinationName: previous?.destinationName ?? preferredDestinationName(resource),
|
|
575
797
|
...(previous?.sha256 ? { sha256: previous.sha256 } : {}),
|
|
576
798
|
...(previous?.size !== undefined ? { size: previous.size } : resource.size !== undefined ? { size: resource.size } : {}),
|
|
799
|
+
...(previous?.cachePath ? { cachePath: previous.cachePath } : {}),
|
|
800
|
+
...(previous?.mimeType ? { mimeType: previous.mimeType } : {}),
|
|
577
801
|
status: 'failed',
|
|
578
802
|
lastError: detail,
|
|
579
803
|
updatedAt: new Date().toISOString(),
|
|
@@ -581,24 +805,36 @@ export class Migrator {
|
|
|
581
805
|
await store.setPage({ ...state, status: 'partial', lastError: detail, updatedAt: new Date().toISOString() });
|
|
582
806
|
};
|
|
583
807
|
try {
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
if (
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
808
|
+
let remoteResources = options.dryRun ? [] : await destination.listUploadedResources(destinationPage);
|
|
809
|
+
const nameAllocator = new DestinationNameAllocator(remoteResources.map((resource) => resource.name));
|
|
810
|
+
for (const resourceState of Object.values(state.resources)) {
|
|
811
|
+
if (resourceState.status === 'complete')
|
|
812
|
+
nameAllocator.reserve(resourceState.destinationName);
|
|
813
|
+
}
|
|
814
|
+
const fallbackBinding = (resource) => [...sourceResourceBindings(cachedPage, [resource]).values()][0];
|
|
815
|
+
const remoteMatches = async (remote, downloaded, trustedMapping = false) => {
|
|
816
|
+
if (remote.size !== undefined && remote.size !== downloaded.size)
|
|
817
|
+
return false;
|
|
818
|
+
if (!destination.downloadUploadedResource) {
|
|
819
|
+
return remote.fingerprint === downloaded.sha256
|
|
820
|
+
|| (trustedMapping && (remote.size === undefined || remote.size === downloaded.size));
|
|
821
|
+
}
|
|
822
|
+
const comparisonPath = `${downloaded.path}.destination-${sha256(remote.id).slice(0, 8)}`;
|
|
823
|
+
try {
|
|
824
|
+
const actual = await destination.downloadUploadedResource({ page: destinationPage, resource: remote, destination: comparisonPath });
|
|
825
|
+
return actual.size === downloaded.size && actual.sha256 === downloaded.sha256;
|
|
826
|
+
}
|
|
827
|
+
finally {
|
|
828
|
+
await rm(comparisonPath, { force: true });
|
|
597
829
|
}
|
|
830
|
+
};
|
|
831
|
+
for (const resource of transferableResources) {
|
|
832
|
+
const existing = state.resources[resource.id];
|
|
598
833
|
if (options.dryRun) {
|
|
599
|
-
|
|
834
|
+
const destinationName = nameAllocator.allocate(resource, existing?.status === 'complete' ? existing.destinationName : undefined);
|
|
835
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: {
|
|
600
836
|
id: `dry-run:${resource.id}`,
|
|
601
|
-
name:
|
|
837
|
+
name: destinationName,
|
|
602
838
|
...(resource.size !== undefined ? { size: resource.size } : {}),
|
|
603
839
|
} });
|
|
604
840
|
progress.resources += 1;
|
|
@@ -606,17 +842,106 @@ export class Migrator {
|
|
|
606
842
|
continue;
|
|
607
843
|
}
|
|
608
844
|
try {
|
|
609
|
-
const filename =
|
|
610
|
-
|
|
611
|
-
const
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
845
|
+
const filename = containedCachePath(options.cacheDirectory, existing?.cachePath)
|
|
846
|
+
?? cachePath(options.cacheDirectory, page, resource);
|
|
847
|
+
const cached = await verifiedCacheFile(filename, existing?.sha256 ? {
|
|
848
|
+
sha256: existing.sha256,
|
|
849
|
+
...(existing.size !== undefined ? { size: existing.size } : {}),
|
|
850
|
+
...(existing.mimeType ? { mimeType: existing.mimeType } : {}),
|
|
851
|
+
} : undefined);
|
|
852
|
+
const downloaded = cached ?? await source.downloadResource(resource, filename);
|
|
853
|
+
const mimeType = downloaded.mimeType ?? existing?.mimeType ?? resource.mimeType;
|
|
854
|
+
const mappedRemote = existing?.destinationId
|
|
855
|
+
? remoteResources.find((item) => item.id === existing.destinationId)
|
|
856
|
+
: undefined;
|
|
857
|
+
if (mappedRemote && await remoteMatches(mappedRemote, downloaded, true)) {
|
|
858
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: mappedRemote });
|
|
859
|
+
state.resources[resource.id] = {
|
|
860
|
+
sourceId: resource.id,
|
|
861
|
+
sourceName: resource.name,
|
|
862
|
+
destinationId: mappedRemote.id,
|
|
863
|
+
destinationName: mappedRemote.name,
|
|
864
|
+
sha256: downloaded.sha256,
|
|
865
|
+
size: downloaded.size,
|
|
866
|
+
cachePath: downloaded.path,
|
|
867
|
+
...(mimeType ? { mimeType } : {}),
|
|
868
|
+
status: 'complete',
|
|
869
|
+
updatedAt: new Date().toISOString(),
|
|
870
|
+
};
|
|
871
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
872
|
+
updateProgress(`reused ${resource.name}`);
|
|
873
|
+
continue;
|
|
874
|
+
}
|
|
875
|
+
const reconciliationNames = new Set([
|
|
876
|
+
preferredDestinationName(resource),
|
|
877
|
+
...(existing?.sha256 ? [existing.destinationName] : []),
|
|
878
|
+
].map((name) => name.toLocaleLowerCase()));
|
|
879
|
+
const reconciled = remoteResources.find((item) => reconciliationNames.has(item.name.toLocaleLowerCase()));
|
|
880
|
+
if (reconciled && await remoteMatches(reconciled, downloaded)) {
|
|
881
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: reconciled });
|
|
882
|
+
state.resources[resource.id] = {
|
|
883
|
+
sourceId: resource.id,
|
|
884
|
+
sourceName: resource.name,
|
|
885
|
+
destinationId: reconciled.id,
|
|
886
|
+
destinationName: reconciled.name,
|
|
887
|
+
sha256: downloaded.sha256,
|
|
888
|
+
size: downloaded.size,
|
|
889
|
+
cachePath: downloaded.path,
|
|
890
|
+
...(mimeType ? { mimeType } : {}),
|
|
891
|
+
status: 'complete',
|
|
892
|
+
updatedAt: new Date().toISOString(),
|
|
893
|
+
};
|
|
894
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
895
|
+
updateProgress(`reconciled ${resource.name}`);
|
|
896
|
+
continue;
|
|
897
|
+
}
|
|
898
|
+
const updateExisting = existing?.status === 'complete' && mappedRemote ? mappedRemote : undefined;
|
|
899
|
+
const destinationName = updateExisting?.name ?? nameAllocator.allocate(resource, existing?.status === 'complete' && !nameAllocator.isOccupied(existing.destinationName) ? existing.destinationName : undefined);
|
|
900
|
+
state.resources[resource.id] = {
|
|
901
|
+
sourceId: resource.id,
|
|
902
|
+
sourceName: resource.name,
|
|
903
|
+
destinationName,
|
|
904
|
+
sha256: downloaded.sha256,
|
|
615
905
|
size: downloaded.size,
|
|
616
|
-
|
|
617
|
-
...(
|
|
618
|
-
|
|
619
|
-
|
|
906
|
+
cachePath: downloaded.path,
|
|
907
|
+
...(mimeType ? { mimeType } : {}),
|
|
908
|
+
status: 'planned',
|
|
909
|
+
updatedAt: new Date().toISOString(),
|
|
910
|
+
};
|
|
911
|
+
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
912
|
+
const matchingName = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
|
|
913
|
+
let result = matchingName && await remoteMatches(matchingName, downloaded) ? matchingName : undefined;
|
|
914
|
+
if (!result) {
|
|
915
|
+
try {
|
|
916
|
+
result = await destination.uploadResource({
|
|
917
|
+
page: destinationPage,
|
|
918
|
+
name: destinationName,
|
|
919
|
+
filePath: downloaded.path,
|
|
920
|
+
size: downloaded.size,
|
|
921
|
+
...(mimeType ? { mimeType } : {}),
|
|
922
|
+
...(updateExisting ? { existingId: updateExisting.id } : {}),
|
|
923
|
+
});
|
|
924
|
+
}
|
|
925
|
+
catch (uploadError) {
|
|
926
|
+
remoteResources = await destination.listUploadedResources(destinationPage);
|
|
927
|
+
const uncertain = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
|
|
928
|
+
if (uncertain && await remoteMatches(uncertain, downloaded))
|
|
929
|
+
result = uncertain;
|
|
930
|
+
else {
|
|
931
|
+
const retryName = nameAllocator.allocate(resource);
|
|
932
|
+
result = await destination.uploadResource({
|
|
933
|
+
page: destinationPage,
|
|
934
|
+
name: retryName,
|
|
935
|
+
filePath: downloaded.path,
|
|
936
|
+
size: downloaded.size,
|
|
937
|
+
...(mimeType ? { mimeType } : {}),
|
|
938
|
+
}).catch((retryError) => { throw new Error(`${retryError instanceof Error ? retryError.message : String(retryError)} (initial upload: ${uploadError instanceof Error ? uploadError.message : String(uploadError)})`); });
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
}
|
|
942
|
+
if (!remoteResources.some((item) => item.id === result.id))
|
|
943
|
+
remoteResources.push(result);
|
|
944
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: result });
|
|
620
945
|
state.resources[resource.id] = {
|
|
621
946
|
sourceId: resource.id,
|
|
622
947
|
sourceName: resource.name,
|
|
@@ -624,6 +949,8 @@ export class Migrator {
|
|
|
624
949
|
destinationName: result.name,
|
|
625
950
|
sha256: downloaded.sha256,
|
|
626
951
|
size: downloaded.size,
|
|
952
|
+
cachePath: downloaded.path,
|
|
953
|
+
...(mimeType ? { mimeType } : {}),
|
|
627
954
|
status: 'complete',
|
|
628
955
|
updatedAt: new Date().toISOString(),
|
|
629
956
|
};
|
|
@@ -638,34 +965,33 @@ export class Migrator {
|
|
|
638
965
|
progress.resources += 1;
|
|
639
966
|
}
|
|
640
967
|
}
|
|
641
|
-
if (!options.dryRun &&
|
|
968
|
+
if (!options.dryRun && transferableResources.length > 0) {
|
|
642
969
|
try {
|
|
643
|
-
|
|
644
|
-
for (const resource of
|
|
970
|
+
remoteResources = await destination.listUploadedResources(destinationPage);
|
|
971
|
+
for (const resource of transferableResources) {
|
|
645
972
|
const transferred = state.resources[resource.id];
|
|
646
973
|
if (transferred?.status !== 'complete')
|
|
647
974
|
continue;
|
|
648
975
|
const remote = remoteResources.find((item) => item.id === transferred.destinationId || item.name === transferred.destinationName);
|
|
649
|
-
const sizeMismatch =
|
|
650
|
-
&& transferred.size !== undefined
|
|
976
|
+
const sizeMismatch = transferred.size !== undefined
|
|
651
977
|
&& remote?.size !== undefined
|
|
652
978
|
&& transferred.size !== remote.size;
|
|
653
979
|
if (!remote || sizeMismatch) {
|
|
654
|
-
bindings
|
|
980
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
|
|
655
981
|
await recordResourceFailure(resource, !remote
|
|
656
982
|
? `Verification did not find destination resource for ${resource.name}`
|
|
657
983
|
: `Verification size mismatch for ${resource.name}: ${transferred.size} source bytes, ${remote.size} destination bytes`);
|
|
658
984
|
}
|
|
659
985
|
else {
|
|
660
|
-
bindings
|
|
986
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: remote });
|
|
661
987
|
}
|
|
662
988
|
}
|
|
663
989
|
}
|
|
664
990
|
catch (error) {
|
|
665
|
-
for (const resource of
|
|
991
|
+
for (const resource of transferableResources) {
|
|
666
992
|
if (state.resources[resource.id]?.status !== 'complete')
|
|
667
993
|
continue;
|
|
668
|
-
bindings
|
|
994
|
+
setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
|
|
669
995
|
await recordResourceFailure(resource, `Destination resource verification failed for ${resource.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
670
996
|
}
|
|
671
997
|
}
|
|
@@ -677,7 +1003,6 @@ export class Migrator {
|
|
|
677
1003
|
currentPageId: page.canonicalId,
|
|
678
1004
|
};
|
|
679
1005
|
const rendered = await renderRemoteDocument({ destination, container, document, page: cachedPage, context });
|
|
680
|
-
warnings += rendered.warnings.length;
|
|
681
1006
|
if (rendered.degraded && !options.dryRun) {
|
|
682
1007
|
const extension = page.rawFormat === 'yfm' ? 'yfm' : 'storage.xml';
|
|
683
1008
|
const snapshotName = `ywtc-source-${sha256(page.canonicalId).slice(0, 8)}.${extension}`;
|
|
@@ -686,37 +1011,64 @@ export class Migrator {
|
|
|
686
1011
|
await writeFile(snapshotPath, cachedPage.rawContent, { mode: 0o600 });
|
|
687
1012
|
const snapshotKey = `snapshot:${page.canonicalId}`;
|
|
688
1013
|
const existingSnapshot = state.resources[snapshotKey];
|
|
1014
|
+
const snapshotFile = {
|
|
1015
|
+
path: snapshotPath,
|
|
1016
|
+
size: Buffer.byteLength(cachedPage.rawContent),
|
|
1017
|
+
sha256: sha256(cachedPage.rawContent),
|
|
1018
|
+
};
|
|
689
1019
|
try {
|
|
690
|
-
const
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
1020
|
+
const knownSnapshot = remoteResources.find((item) => item.id === existingSnapshot?.destinationId)
|
|
1021
|
+
?? remoteResources.find((item) => item.name.toLocaleLowerCase() === snapshotName.toLocaleLowerCase());
|
|
1022
|
+
let snapshot = knownSnapshot && await remoteMatches(knownSnapshot, snapshotFile, true)
|
|
1023
|
+
? knownSnapshot
|
|
1024
|
+
: undefined;
|
|
1025
|
+
if (!snapshot) {
|
|
1026
|
+
try {
|
|
1027
|
+
snapshot = await destination.uploadResource({
|
|
1028
|
+
page: destinationPage,
|
|
1029
|
+
name: snapshotName,
|
|
1030
|
+
filePath: snapshotPath,
|
|
1031
|
+
size: snapshotFile.size,
|
|
1032
|
+
mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
|
|
1033
|
+
...(knownSnapshot ? { existingId: knownSnapshot.id } : {}),
|
|
1034
|
+
});
|
|
695
1035
|
}
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
name
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
}
|
|
1036
|
+
catch (uploadError) {
|
|
1037
|
+
remoteResources = await destination.listUploadedResources(destinationPage);
|
|
1038
|
+
const uncertain = remoteResources.find((item) => item.name.toLocaleLowerCase() === snapshotName.toLocaleLowerCase());
|
|
1039
|
+
if (uncertain && await remoteMatches(uncertain, snapshotFile, true))
|
|
1040
|
+
snapshot = uncertain;
|
|
1041
|
+
else
|
|
1042
|
+
throw uploadError;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
if (!remoteResources.some((item) => item.id === snapshot.id))
|
|
1046
|
+
remoteResources.push(snapshot);
|
|
704
1047
|
state.resources[snapshotKey] = {
|
|
705
1048
|
sourceId: snapshotKey,
|
|
706
1049
|
sourceName: snapshotName,
|
|
707
1050
|
destinationId: snapshot.id,
|
|
708
1051
|
destinationName: snapshot.name,
|
|
709
1052
|
...(snapshot.size !== undefined ? { size: snapshot.size } : {}),
|
|
710
|
-
sha256: sha256
|
|
1053
|
+
sha256: snapshotFile.sha256,
|
|
1054
|
+
cachePath: snapshotPath,
|
|
1055
|
+
mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
|
|
711
1056
|
status: 'complete',
|
|
712
1057
|
updatedAt: new Date().toISOString(),
|
|
713
1058
|
};
|
|
714
1059
|
}
|
|
715
1060
|
catch (error) {
|
|
716
1061
|
const detail = `Raw source snapshot was not transferred: ${error instanceof Error ? error.message : String(error)}`;
|
|
717
|
-
errors += 1;
|
|
718
1062
|
resourceProblems.push(detail);
|
|
719
|
-
report.
|
|
1063
|
+
report.issue({
|
|
1064
|
+
code: 'source-snapshot-transfer-failed',
|
|
1065
|
+
severity: 'error',
|
|
1066
|
+
phase: 'resource',
|
|
1067
|
+
pageId: page.canonicalId,
|
|
1068
|
+
resourceId: snapshotKey,
|
|
1069
|
+
retryable: true,
|
|
1070
|
+
message: `${page.title}: ${detail}`,
|
|
1071
|
+
});
|
|
720
1072
|
state.resources[snapshotKey] = {
|
|
721
1073
|
sourceId: snapshotKey,
|
|
722
1074
|
sourceName: snapshotName,
|
|
@@ -726,15 +1078,20 @@ export class Migrator {
|
|
|
726
1078
|
updatedAt: new Date().toISOString(),
|
|
727
1079
|
};
|
|
728
1080
|
}
|
|
729
|
-
state.degradations = [...new Set([...
|
|
1081
|
+
state.degradations = [...new Set([...rendered.warnings, ...resourceProblems])];
|
|
730
1082
|
}
|
|
731
1083
|
let verified = destinationPage;
|
|
1084
|
+
let verificationDegradations = [];
|
|
732
1085
|
if (!options.dryRun) {
|
|
733
|
-
const current = await destination.
|
|
1086
|
+
const current = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
|
|
734
1087
|
if (destinationPage.contentFingerprint
|
|
735
1088
|
&& current.contentFingerprint
|
|
736
1089
|
&& current.contentFingerprint !== destinationPage.contentFingerprint) {
|
|
737
|
-
|
|
1090
|
+
const normalization = verifyDestinationContent(destinationPage.content ?? bootstrap.body, current.content, destination.kind, cachedPage);
|
|
1091
|
+
if (normalization.status !== 'complete') {
|
|
1092
|
+
throw new Error(`Destination conflict for ${page.title}: content changed while resources were transferred`);
|
|
1093
|
+
}
|
|
1094
|
+
rendered.normalizations.push('Destination normalized page markup without semantic content changes');
|
|
738
1095
|
}
|
|
739
1096
|
verified = await destination.updatePage({
|
|
740
1097
|
page: current,
|
|
@@ -746,10 +1103,11 @@ export class Migrator {
|
|
|
746
1103
|
state.destinationFingerprint = verified.fingerprint;
|
|
747
1104
|
await store.setPage({ ...state, updatedAt: new Date().toISOString() });
|
|
748
1105
|
}
|
|
749
|
-
verified = await destination.
|
|
750
|
-
assertDestinationContent(rendered.body, verified.content, destination.kind, cachedPage);
|
|
1106
|
+
verified = await verifyConfirmedDestinationPage(destination, verified, this.consistencyOptions);
|
|
1107
|
+
const verification = assertDestinationContent(rendered.body, verified.content, destination.kind, cachedPage);
|
|
1108
|
+
verificationDegradations = verification.status === 'partial' ? verification.differences : [];
|
|
751
1109
|
}
|
|
752
|
-
const partial = rendered.degraded || resourceProblems.length > 0;
|
|
1110
|
+
const partial = rendered.degraded || resourceProblems.length > 0 || verificationDegradations.length > 0;
|
|
753
1111
|
const lastResourceProblem = resourceProblems.at(-1);
|
|
754
1112
|
const clearedState = { ...state };
|
|
755
1113
|
delete clearedState.lastError;
|
|
@@ -761,7 +1119,7 @@ export class Migrator {
|
|
|
761
1119
|
destinationTitle: verified.title,
|
|
762
1120
|
...(verified.fingerprint ? { destinationFingerprint: verified.fingerprint } : {}),
|
|
763
1121
|
status: options.dryRun ? 'planned' : partial ? 'partial' : 'complete',
|
|
764
|
-
degradations: [...new Set([...
|
|
1122
|
+
degradations: [...new Set([...rendered.warnings, ...resourceProblems, ...verificationDegradations])],
|
|
765
1123
|
...(lastResourceProblem ? { lastError: lastResourceProblem } : {}),
|
|
766
1124
|
updatedAt: new Date().toISOString(),
|
|
767
1125
|
};
|
|
@@ -778,18 +1136,20 @@ export class Migrator {
|
|
|
778
1136
|
status: finalState.status,
|
|
779
1137
|
resources,
|
|
780
1138
|
document,
|
|
1139
|
+
resourceStates: finalState.resources,
|
|
1140
|
+
...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
|
|
781
1141
|
resourceFallbacks: fallbackDetails,
|
|
782
|
-
warnings: [...rendered.warnings, ...
|
|
1142
|
+
warnings: [...rendered.warnings, ...verificationDegradations],
|
|
1143
|
+
normalizations: rendered.normalizations,
|
|
783
1144
|
}), { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(verified.content ?? rendered.body, destination.kind) });
|
|
784
1145
|
}
|
|
785
1146
|
catch (error) {
|
|
786
|
-
errors += 1;
|
|
787
1147
|
const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
|
|
788
1148
|
let failureStatus = /conflict/i.test(message) ? 'conflict' : 'failed';
|
|
789
1149
|
let actual;
|
|
790
1150
|
if (failureStatus !== 'conflict' && !options.dryRun) {
|
|
791
1151
|
try {
|
|
792
|
-
actual = await destination.
|
|
1152
|
+
actual = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
|
|
793
1153
|
assertDestinationContent(bootstrap.body, actual.content, destination.kind, cachedPage);
|
|
794
1154
|
failureStatus = 'partial';
|
|
795
1155
|
}
|
|
@@ -797,7 +1157,14 @@ export class Migrator {
|
|
|
797
1157
|
failureStatus = 'failed';
|
|
798
1158
|
}
|
|
799
1159
|
}
|
|
800
|
-
report.
|
|
1160
|
+
report.issue({
|
|
1161
|
+
code: failureStatus === 'conflict' ? 'destination-conflict' : 'page-transfer-failed',
|
|
1162
|
+
severity: 'error',
|
|
1163
|
+
phase: 'page',
|
|
1164
|
+
pageId: page.canonicalId,
|
|
1165
|
+
retryable: failureStatus !== 'conflict',
|
|
1166
|
+
message,
|
|
1167
|
+
});
|
|
801
1168
|
await store.setPage({
|
|
802
1169
|
...state,
|
|
803
1170
|
...(actual?.fingerprint ? { destinationFingerprint: actual.fingerprint } : {}),
|
|
@@ -817,8 +1184,9 @@ export class Migrator {
|
|
|
817
1184
|
status: failureStatus,
|
|
818
1185
|
resources,
|
|
819
1186
|
document,
|
|
1187
|
+
resourceStates: state.resources,
|
|
1188
|
+
...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
|
|
820
1189
|
resourceFallbacks: fallbackDetails,
|
|
821
|
-
warnings: resourceProblems,
|
|
822
1190
|
error: message,
|
|
823
1191
|
}), actual?.content ? { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(actual.content, destination.kind) } : undefined);
|
|
824
1192
|
}
|
|
@@ -832,7 +1200,6 @@ export class Migrator {
|
|
|
832
1200
|
const rejected = settled.find((result) => result.status === 'rejected');
|
|
833
1201
|
if (rejected)
|
|
834
1202
|
throw rejected.reason;
|
|
835
|
-
return { errors, warnings };
|
|
836
1203
|
}
|
|
837
1204
|
}
|
|
838
1205
|
//# sourceMappingURL=migrator.js.map
|