@iwonz/ywtc 3.2.0 → 3.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/migrator.js CHANGED
@@ -1,15 +1,16 @@
1
1
  import { createHash } from 'node:crypto';
2
- import { mkdir, readFile, writeFile } from 'node:fs/promises';
2
+ import { mkdir, readFile, rm, writeFile } from 'node:fs/promises';
3
3
  import path from 'node:path';
4
4
  import PQueue from 'p-queue';
5
5
  import { createProviderAdapter } from './adapters/factory.js';
6
- import { appendGridSnapshots, assertDestinationContent, isLegacyTransferPlaceholder, normalizePageContent, normalizedPreviewHtml, rawContentFallback, renderedPreviewHtml, renderConfluenceStorage, renderYfm, sourceResourceBindings, } from './content-model.js';
6
+ import { appendGridSnapshots, assertDestinationContent, isLegacyTransferPlaceholder, normalizePageContent, normalizedPreviewHtml, rawContentFallback, renderedPreviewHtml, renderConfluenceStorage, renderYfm, setSourceResourceBinding, sourceResourceBindings, verifyDestinationContent, } from './content-model.js';
7
+ import { ConfirmedDestinationMissingError, verifyConfirmedDestinationPage } from './consistency.js';
7
8
  import { assertTransferCapabilities } from './domain.js';
8
9
  import { FilesystemExporter } from './filesystem-export.js';
9
10
  import { discoverPageGraph } from './graph.js';
10
- import { safeFilename } from './input.js';
11
11
  import { profileBaseUrl, sameRemoteInstance } from './locators.js';
12
12
  import { TransferReport } from './report.js';
13
+ import { DestinationNameAllocator, preferredDestinationName, verifiedCacheFile } from './resource-reconciliation.js';
13
14
  import { resolveStateLocation, StateStore, } from './state.js';
14
15
  function sourceIdentity(locator, root) {
15
16
  return {
@@ -69,7 +70,37 @@ function collisionTitle(page) {
69
70
  function cachePath(cacheDirectory, page, resource) {
70
71
  const pageKey = sha256(page.canonicalId).slice(0, 12);
71
72
  const resourceKey = sha256(`${resource.id}:${resource.name}`).slice(0, 12);
72
- return path.join(cacheDirectory, pageKey, `${resourceKey}-${safeFilename(resource.name)}`);
73
+ return path.join(cacheDirectory, pageKey, `${resourceKey}-${preferredDestinationName(resource)}`);
74
+ }
75
+ function containedCachePath(cacheDirectory, candidate) {
76
+ if (!candidate)
77
+ return undefined;
78
+ const relative = path.relative(path.resolve(cacheDirectory), path.resolve(candidate));
79
+ return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)
80
+ ? path.resolve(candidate)
81
+ : undefined;
82
+ }
83
+ function resetResourceDestinations(resources) {
84
+ return Object.fromEntries(Object.entries(resources).map(([id, resource]) => [id, {
85
+ sourceId: resource.sourceId,
86
+ sourceName: resource.sourceName,
87
+ destinationName: resource.destinationName,
88
+ ...(resource.sha256 ? { sha256: resource.sha256 } : {}),
89
+ ...(resource.size !== undefined ? { size: resource.size } : {}),
90
+ ...(resource.cachePath ? { cachePath: resource.cachePath } : {}),
91
+ ...(resource.mimeType ? { mimeType: resource.mimeType } : {}),
92
+ status: 'planned',
93
+ updatedAt: new Date().toISOString(),
94
+ }]));
95
+ }
96
+ function sourceDerivedTextMatches(expectedBody, actualBody, provider, page) {
97
+ if (actualBody === undefined)
98
+ return false;
99
+ const result = verifyDestinationContent(expectedBody, actualBody, provider, page);
100
+ if (result.status === 'complete')
101
+ return true;
102
+ const text = (value) => renderedPreviewHtml(value, provider).replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
103
+ return text(expectedBody) === text(actualBody);
73
104
  }
74
105
  function destinationFromState(state, adapter) {
75
106
  if (!state.destinationId || !state.destinationCanonicalId || !state.destinationLocator) {
@@ -87,6 +118,9 @@ function destinationFromState(state, adapter) {
87
118
  };
88
119
  }
89
120
  function reportPage(input) {
121
+ const transferable = input.resources.filter((resource) => resource.kind !== 'grid');
122
+ const sizes = transferable.map((resource) => input.resourceSizes?.[resource.id]?.size ?? resource.size);
123
+ const knownBytes = sizes.reduce((total, size) => total + (size ?? 0), 0);
90
124
  return {
91
125
  sourceCanonicalId: input.page.canonicalId,
92
126
  sourceId: input.page.id,
@@ -100,15 +134,17 @@ function reportPage(input) {
100
134
  ...(input.page.projectedParentId ? { parentId: input.page.projectedParentId } : {}),
101
135
  depth: input.page.depth,
102
136
  status: input.status,
103
- resources: input.resources.filter((resource) => resource.kind !== 'grid').length,
104
- bytes: input.resources.reduce((total, resource) => total + (resource.size ?? 0), 0),
137
+ resources: transferable.length,
138
+ ...(transferable.length === 0 || sizes.some((size) => size !== undefined) ? { bytes: knownBytes } : {}),
139
+ bytesUnknown: sizes.filter((size) => size === undefined).length,
105
140
  links: input.document.metrics.links,
106
141
  images: input.document.metrics.images,
107
142
  tables: input.document.metrics.tables,
108
143
  codeBlocks: input.document.metrics.codeBlocks,
109
144
  degraded: input.document.degraded || (input.resourceFallbacks?.length ?? 0) > 0,
110
145
  resourceFallbacks: input.resourceFallbacks ?? [],
111
- warnings: [...input.document.warnings, ...(input.warnings ?? [])],
146
+ warnings: [...new Set([...input.document.warnings, ...(input.warnings ?? [])])],
147
+ normalizations: [...new Set([...(input.document.normalizations ?? []), ...(input.normalizations ?? [])])],
112
148
  ...(input.error ? { error: input.error } : {}),
113
149
  };
114
150
  }
@@ -144,10 +180,12 @@ export class Migrator {
144
180
  config;
145
181
  logger;
146
182
  adapterFactory;
147
- constructor(config, logger, adapterFactory = createProviderAdapter) {
183
+ consistencyOptions;
184
+ constructor(config, logger, adapterFactory = createProviderAdapter, consistencyOptions = {}) {
148
185
  this.config = config;
149
186
  this.logger = logger;
150
187
  this.adapterFactory = adapterFactory;
188
+ this.consistencyOptions = consistencyOptions;
151
189
  }
152
190
  async run(options) {
153
191
  const source = this.adapterFactory(options.source.profile, this.config);
@@ -226,12 +264,16 @@ export class Migrator {
226
264
  report.finish(finalStatus);
227
265
  const reportPaths = await report.write(options.reportsDirectory);
228
266
  if (!options.dryRun)
229
- await store.finishRun(run.id, finalStatus, { pages: 0, warnings: 0, errors: 1 });
267
+ await store.finishRun(run.id, finalStatus, {
268
+ pages: 0,
269
+ warnings: report.data.summary.warnings,
270
+ errors: report.data.summary.errors,
271
+ });
230
272
  throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
231
273
  }
232
274
  const resourcesByPage = new Map();
233
275
  const inventoryQueue = new PQueue({ concurrency: options.concurrency });
234
- const inventory = { resources: 0, bytes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
276
+ const inventory = { resources: 0, bytes: 0, unknownSizes: 0, grids: 0, links: 0, images: 0, tables: 0, codeBlocks: 0 };
235
277
  status(`Inspecting content and resources for ${graph.pages.length} pages`);
236
278
  const inventoryErrors = [];
237
279
  await Promise.all(graph.pages.map((page) => inventoryQueue.add(async () => {
@@ -245,6 +287,7 @@ export class Migrator {
245
287
  await cacheDocument(options.cacheDirectory, page.canonicalId, document);
246
288
  inventory.resources += resources.filter((resource) => resource.kind !== 'grid').length;
247
289
  inventory.bytes += resources.reduce((total, resource) => total + (resource.size ?? 0), 0);
290
+ inventory.unknownSizes += resources.filter((resource) => resource.kind !== 'grid' && resource.size === undefined).length;
248
291
  inventory.grids += resources.filter((resource) => resource.kind === 'grid').length;
249
292
  inventory.links += document.metrics.links;
250
293
  inventory.images += document.metrics.images;
@@ -263,23 +306,23 @@ export class Migrator {
263
306
  report.finish(finalStatus);
264
307
  const reportPaths = await report.write(options.reportsDirectory);
265
308
  if (!options.dryRun)
266
- await store.finishRun(run.id, finalStatus, { pages: graph.pages.length, warnings: graph.warnings.length, errors: inventoryErrors.length });
309
+ await store.finishRun(run.id, finalStatus, {
310
+ pages: graph.pages.length,
311
+ warnings: report.data.summary.warnings,
312
+ errors: report.data.summary.errors,
313
+ });
267
314
  throw new MigrationRunError('Resource preflight failed before destination mutation', reportPaths, stateLocation.filename);
268
315
  }
269
- const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} bytes) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
316
+ const planSummary = `Plan ready: ${graph.pages.length} pages · ${inventory.resources} files (${inventory.bytes} known bytes${inventory.unknownSizes > 0 ? ` · ${inventory.unknownSizes} sizes unknown` : ''}) · ${inventory.links} links · ${inventory.images} images · ${inventory.tables} tables (${inventory.grids} grids) · ${inventory.codeBlocks} code blocks`;
270
317
  status(planSummary);
271
318
  this.logger.info(planSummary);
272
319
  let output;
273
320
  let archive;
274
- let transferErrors = 0;
275
- let transferWarnings = graph.warnings.length;
276
321
  try {
277
322
  if (options.destination.type === 'directory') {
278
323
  const result = await this.exportFilesystem(source, graph, options.destination, options, stateLocation.id, store, report, sourceFingerprints, resourcesByPage);
279
324
  output = result.directory;
280
325
  archive = result.archive;
281
- transferErrors += result.errors.length;
282
- transferWarnings += result.warnings.length;
283
326
  report.output('directory', result.directory);
284
327
  if (result.archive)
285
328
  report.output('archive', result.archive);
@@ -287,7 +330,7 @@ export class Migrator {
287
330
  report.output('backup', result.backup);
288
331
  }
289
332
  else {
290
- const result = await this.transferRemote({
333
+ await this.transferRemote({
291
334
  source,
292
335
  destination: destination,
293
336
  container: container,
@@ -298,27 +341,30 @@ export class Migrator {
298
341
  sourceFingerprints,
299
342
  resourcesByPage,
300
343
  });
301
- transferErrors += result.errors;
302
- transferWarnings += result.warnings;
303
344
  output = container.url;
304
345
  report.output('destination', container.url);
305
346
  }
306
- const finalStatus = runStatus(transferErrors, transferWarnings);
347
+ report.finish('completed');
348
+ const finalStatus = runStatus(report.data.summary.errors, report.data.summary.warnings);
307
349
  report.finish(finalStatus);
308
350
  const reportPaths = await report.write(options.reportsDirectory);
309
351
  if (!options.dryRun)
310
- await store.finishRun(run.id, finalStatus, { pages: graph.pages.length, warnings: transferWarnings, errors: transferErrors });
311
- status(`${finalStatus}: ${graph.pages.length} pages · ${transferErrors} errors · ${transferWarnings} warnings`);
352
+ await store.finishRun(run.id, finalStatus, {
353
+ pages: graph.pages.length,
354
+ warnings: report.data.summary.warnings,
355
+ errors: report.data.summary.errors,
356
+ });
357
+ status(`${finalStatus}: ${graph.pages.length} pages · ${report.data.summary.errors} errors · ${report.data.summary.warnings} warnings`);
312
358
  return {
313
359
  status: finalStatus,
314
- hasErrors: transferErrors > 0,
360
+ hasErrors: report.data.summary.errors > 0,
315
361
  reportPaths,
316
362
  stateId: stateLocation.id,
317
363
  stateFilename: stateLocation.filename,
318
364
  ...(store.loadInfo.backupFilename ? { stateBackupFilename: store.loadInfo.backupFilename } : {}),
319
365
  ...(output ? { output } : {}),
320
366
  ...(archive ? { archive } : {}),
321
- summary: { ...report.data.summary, warnings: transferWarnings, errors: transferErrors },
367
+ summary: report.data.summary,
322
368
  };
323
369
  }
324
370
  catch (error) {
@@ -329,7 +375,11 @@ export class Migrator {
329
375
  report.finish(finalStatus);
330
376
  const reportPaths = await report.write(options.reportsDirectory);
331
377
  if (!options.dryRun)
332
- await store.finishRun(run.id, finalStatus, { pages: graph.pages.length, warnings: transferWarnings, errors: transferErrors + 1 });
378
+ await store.finishRun(run.id, finalStatus, {
379
+ pages: graph.pages.length,
380
+ warnings: report.data.summary.warnings,
381
+ errors: report.data.summary.errors,
382
+ });
333
383
  this.logger.error(`${message}. Report: ${reportPaths.html}`);
334
384
  throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
335
385
  }
@@ -360,16 +410,22 @@ export class Migrator {
360
410
  const document = await loadCachedDocument(options.cacheDirectory, page.canonicalId);
361
411
  const manifestPage = result.manifest.pages.find((item) => item.sourceCanonicalId === page.canonicalId);
362
412
  const previous = store.value.pages[page.canonicalId];
363
- const resources = Object.fromEntries((manifestPage?.resources ?? []).map((resource, index) => [String(index), {
364
- sourceId: String(index),
365
- sourceName: resource.name,
366
- destinationId: resource.path,
367
- destinationName: resource.name,
368
- ...(resource.sha256 ? { sha256: resource.sha256 } : {}),
369
- ...(resource.size !== undefined ? { size: resource.size } : {}),
370
- status: 'complete',
371
- updatedAt: new Date().toISOString(),
372
- }]));
413
+ const sourceResources = resourcesByPage.get(page.canonicalId) ?? [];
414
+ const transferableResources = sourceResources.filter((resource) => resource.kind !== 'grid');
415
+ const resources = Object.fromEntries((manifestPage?.resources ?? []).map((resource, index) => {
416
+ const sourceResource = transferableResources[index];
417
+ const sourceId = sourceResource?.id ?? String(index);
418
+ return [sourceId, {
419
+ sourceId,
420
+ sourceName: sourceResource?.name ?? resource.name,
421
+ destinationId: resource.path,
422
+ destinationName: resource.name,
423
+ ...(resource.sha256 ? { sha256: resource.sha256 } : {}),
424
+ ...(resource.size !== undefined ? { size: resource.size } : {}),
425
+ status: 'complete',
426
+ updatedAt: new Date().toISOString(),
427
+ }];
428
+ }));
373
429
  await store.setPage({
374
430
  sourceId: page.id,
375
431
  sourceCanonicalId: page.canonicalId,
@@ -390,8 +446,9 @@ export class Migrator {
390
446
  report.addPage(reportPage({
391
447
  page,
392
448
  status: options.dryRun ? 'planned' : result.errors.length ? 'partial' : 'complete',
393
- resources: resourcesByPage.get(page.canonicalId) ?? [],
449
+ resources: sourceResources,
394
450
  document,
451
+ resourceSizes: resources,
395
452
  ...(manifestPage?.warnings ? { warnings: manifestPage.warnings } : {}),
396
453
  }), { source: normalizedPreviewHtml(document), destination: normalizedPreviewHtml(document) });
397
454
  }
@@ -406,8 +463,6 @@ export class Migrator {
406
463
  const destinationPages = new Map();
407
464
  const sourcePages = new Map(graph.pages.map((page) => [page.canonicalId, page]));
408
465
  const bootstrapBodies = new Map();
409
- let errors = 0;
410
- let warnings = 0;
411
466
  const totalResources = [...resourcesByPage.values()].reduce((total, resources) => total + resources.filter((resource) => resource.kind !== 'grid').length, 0);
412
467
  const progress = { staged: 0, pages: 0, resources: 0 };
413
468
  const totalUnits = Math.max(1, graph.pages.length * 2 + totalResources);
@@ -441,7 +496,6 @@ export class Migrator {
441
496
  const parent = page.projectedParentId ? destinationPages.get(page.projectedParentId) : container;
442
497
  if (!parent) {
443
498
  const message = `Destination parent is unavailable for ${page.title}`;
444
- errors += 1;
445
499
  report.error(message);
446
500
  progress.staged += 1;
447
501
  updateProgress(message);
@@ -452,11 +506,66 @@ export class Migrator {
452
506
  const bootstrap = bootstrapBodies.get(page.canonicalId);
453
507
  let created;
454
508
  let title = saved?.destinationTitle ?? page.title;
509
+ let resourcesForState = saved?.resources ?? {};
510
+ let previousDestinationIds = saved?.previousDestinationIds ?? [];
511
+ let mappingRecovered = false;
455
512
  if (saved?.destinationId) {
456
- created = await destination.verifyPage(destinationFromState(saved, destination));
513
+ const mapped = destinationFromState(saved, destination);
514
+ try {
515
+ created = await verifyConfirmedDestinationPage(destination, mapped, {
516
+ ...this.consistencyOptions,
517
+ onRetry: (attempt, delayMs) => report.issue({
518
+ code: 'destination-consistency-retry',
519
+ severity: 'warning',
520
+ phase: 'verification',
521
+ pageId: page.canonicalId,
522
+ retryable: true,
523
+ message: `${page.title}: destination page ${mapped.id} returned 404; consistency retry ${attempt} in ${delayMs} ms`,
524
+ }),
525
+ });
526
+ }
527
+ catch (error) {
528
+ if (!(error instanceof ConfirmedDestinationMissingError) || options.dryRun)
529
+ throw error;
530
+ if (page.projectedParentId) {
531
+ await verifyConfirmedDestinationPage(destination, parent, this.consistencyOptions);
532
+ }
533
+ const candidate = await (destination.findRecoverablePage?.(parent, title) ?? destination.findChild(parent, title));
534
+ if (candidate && sourceDerivedTextMatches(bootstrap.body, candidate.content, destination.kind, cachedPage)) {
535
+ created = candidate;
536
+ }
537
+ else {
538
+ let recoveryTitle = title;
539
+ const occupied = candidate ?? await destination.findChild(parent, recoveryTitle);
540
+ if (occupied) {
541
+ const base = collisionTitle(page);
542
+ const hash = sha256(page.canonicalId).slice(0, 8);
543
+ recoveryTitle = base;
544
+ let sequence = 0;
545
+ while (await destination.findChild(parent, recoveryTitle)) {
546
+ sequence += 1;
547
+ recoveryTitle = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
548
+ }
549
+ }
550
+ created = await destination.createPage({ container: parent, title: recoveryTitle, body: bootstrap.body });
551
+ created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
552
+ assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
553
+ }
554
+ previousDestinationIds = [...new Set([...previousDestinationIds, mapped.id])];
555
+ resourcesForState = resetResourceDestinations(saved.resources);
556
+ mappingRecovered = true;
557
+ report.issue({
558
+ code: 'destination-mapping-recovered',
559
+ severity: 'warning',
560
+ phase: 'page',
561
+ pageId: page.canonicalId,
562
+ retryable: false,
563
+ message: `${page.title}: replaced unavailable destination mapping ${mapped.id} with ${created.id}`,
564
+ });
565
+ }
457
566
  const generatedPlaceholder = created.title === saved.destinationTitle
458
567
  && isLegacyTransferPlaceholder(created.content, destination.kind, cachedPage);
459
- if (['complete', 'partial'].includes(saved.status) && saved.destinationFingerprint && created.fingerprint !== saved.destinationFingerprint && !generatedPlaceholder) {
568
+ if (!mappingRecovered && ['complete', 'partial'].includes(saved.status) && saved.destinationFingerprint && created.fingerprint !== saved.destinationFingerprint && !generatedPlaceholder) {
460
569
  throw new Error(`Destination conflict for ${page.title}: content changed since the last verified run`);
461
570
  }
462
571
  if (generatedPlaceholder && !options.dryRun) {
@@ -466,7 +575,7 @@ export class Migrator {
466
575
  body: bootstrap.body,
467
576
  ...(created.fingerprint ? { expectedFingerprint: created.fingerprint } : {}),
468
577
  });
469
- created = await destination.verifyPage(created);
578
+ created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
470
579
  assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
471
580
  }
472
581
  }
@@ -496,7 +605,7 @@ export class Migrator {
496
605
  }
497
606
  }
498
607
  created = await destination.createPage({ container: parent, title, body: bootstrap.body });
499
- created = await destination.verifyPage(created);
608
+ created = await verifyConfirmedDestinationPage(destination, created, this.consistencyOptions);
500
609
  assertDestinationContent(bootstrap.body, created.content, destination.kind, cachedPage);
501
610
  }
502
611
  destinationPages.set(page.canonicalId, created);
@@ -510,10 +619,11 @@ export class Migrator {
510
619
  destinationCanonicalId: created.canonicalId,
511
620
  destinationLocator: created.url,
512
621
  destinationTitle: created.title,
622
+ ...(previousDestinationIds.length > 0 ? { previousDestinationIds } : {}),
513
623
  sourceFingerprint: sourceFingerprints.get(page.canonicalId),
514
624
  ...(created.fingerprint ? { destinationFingerprint: created.fingerprint } : {}),
515
625
  status: options.dryRun ? 'planned' : saved?.status ?? 'skeleton-created',
516
- resources: saved?.resources ?? {},
626
+ resources: resourcesForState,
517
627
  degradations: saved?.degradations ?? [],
518
628
  updatedAt: new Date().toISOString(),
519
629
  };
@@ -521,7 +631,6 @@ export class Migrator {
521
631
  options.onStatus?.(`${options.dryRun ? 'Planned' : 'Staged content for'} ${page.title}`);
522
632
  }
523
633
  catch (error) {
524
- errors += 1;
525
634
  const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
526
635
  report.error(message);
527
636
  const savedState = store.value.pages[page.canonicalId];
@@ -557,23 +666,33 @@ export class Migrator {
557
666
  const state = store.value.pages[page.canonicalId];
558
667
  const bootstrap = bootstrapBodies.get(page.canonicalId);
559
668
  const bindings = sourceResourceBindings(cachedPage, resources);
669
+ const transferableResources = resources.filter((item) => item.kind !== 'grid');
560
670
  const resourceProblems = [];
561
671
  const failedResourceIds = new Set();
562
672
  const recordResourceFailure = async (resource, detail) => {
563
673
  if (!failedResourceIds.has(resource.id)) {
564
674
  failedResourceIds.add(resource.id);
565
- errors += 1;
566
675
  resourceProblems.push(detail);
567
- report.error(`${page.title}: ${detail}`);
676
+ report.issue({
677
+ code: 'resource-transfer-failed',
678
+ severity: 'error',
679
+ phase: 'resource',
680
+ pageId: page.canonicalId,
681
+ resourceId: resource.id,
682
+ retryable: true,
683
+ message: `${page.title}: ${detail}`,
684
+ });
568
685
  }
569
686
  const previous = state.resources[resource.id];
570
687
  state.resources[resource.id] = {
571
688
  sourceId: resource.id,
572
689
  sourceName: resource.name,
573
690
  ...(previous?.destinationId ? { destinationId: previous.destinationId } : {}),
574
- destinationName: previous?.destinationName ?? safeFilename(resource.name),
691
+ destinationName: previous?.destinationName ?? preferredDestinationName(resource),
575
692
  ...(previous?.sha256 ? { sha256: previous.sha256 } : {}),
576
693
  ...(previous?.size !== undefined ? { size: previous.size } : resource.size !== undefined ? { size: resource.size } : {}),
694
+ ...(previous?.cachePath ? { cachePath: previous.cachePath } : {}),
695
+ ...(previous?.mimeType ? { mimeType: previous.mimeType } : {}),
577
696
  status: 'failed',
578
697
  lastError: detail,
579
698
  updatedAt: new Date().toISOString(),
@@ -581,24 +700,36 @@ export class Migrator {
581
700
  await store.setPage({ ...state, status: 'partial', lastError: detail, updatedAt: new Date().toISOString() });
582
701
  };
583
702
  try {
584
- for (const resource of resources.filter((item) => item.kind !== 'grid')) {
585
- const key = resource.name.toLocaleLowerCase();
586
- const existing = state.resources[resource.id];
587
- if (existing?.status === 'complete' && existing.destinationId) {
588
- bindings.set(key, { kind: 'uploaded', resource: {
589
- id: existing.destinationId,
590
- name: existing.destinationName,
591
- ...(existing.size !== undefined ? { size: existing.size } : {}),
592
- ...(existing.sha256 ? { fingerprint: existing.sha256 } : {}),
593
- } });
594
- progress.resources += 1;
595
- updateProgress(`reused ${resource.name}`);
596
- continue;
703
+ let remoteResources = options.dryRun ? [] : await destination.listUploadedResources(destinationPage);
704
+ const nameAllocator = new DestinationNameAllocator(remoteResources.map((resource) => resource.name));
705
+ for (const resourceState of Object.values(state.resources)) {
706
+ if (resourceState.status === 'complete')
707
+ nameAllocator.reserve(resourceState.destinationName);
708
+ }
709
+ const fallbackBinding = (resource) => [...sourceResourceBindings(cachedPage, [resource]).values()][0];
710
+ const remoteMatches = async (remote, downloaded, trustedMapping = false) => {
711
+ if (remote.size !== undefined && remote.size !== downloaded.size)
712
+ return false;
713
+ if (!destination.downloadUploadedResource) {
714
+ return remote.fingerprint === downloaded.sha256
715
+ || (trustedMapping && (remote.size === undefined || remote.size === downloaded.size));
597
716
  }
717
+ const comparisonPath = `${downloaded.path}.destination-${sha256(remote.id).slice(0, 8)}`;
718
+ try {
719
+ const actual = await destination.downloadUploadedResource({ page: destinationPage, resource: remote, destination: comparisonPath });
720
+ return actual.size === downloaded.size && actual.sha256 === downloaded.sha256;
721
+ }
722
+ finally {
723
+ await rm(comparisonPath, { force: true });
724
+ }
725
+ };
726
+ for (const resource of transferableResources) {
727
+ const existing = state.resources[resource.id];
598
728
  if (options.dryRun) {
599
- bindings.set(key, { kind: 'uploaded', resource: {
729
+ const destinationName = nameAllocator.allocate(resource, existing?.status === 'complete' ? existing.destinationName : undefined);
730
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: {
600
731
  id: `dry-run:${resource.id}`,
601
- name: safeFilename(resource.name),
732
+ name: destinationName,
602
733
  ...(resource.size !== undefined ? { size: resource.size } : {}),
603
734
  } });
604
735
  progress.resources += 1;
@@ -606,17 +737,106 @@ export class Migrator {
606
737
  continue;
607
738
  }
608
739
  try {
609
- const filename = cachePath(options.cacheDirectory, page, resource);
610
- const downloaded = await source.downloadResource(resource, filename);
611
- const result = await destination.uploadResource({
612
- page: destinationPage,
613
- name: safeFilename(resource.name),
614
- filePath: downloaded.path,
740
+ const filename = containedCachePath(options.cacheDirectory, existing?.cachePath)
741
+ ?? cachePath(options.cacheDirectory, page, resource);
742
+ const cached = await verifiedCacheFile(filename, existing?.sha256 ? {
743
+ sha256: existing.sha256,
744
+ ...(existing.size !== undefined ? { size: existing.size } : {}),
745
+ ...(existing.mimeType ? { mimeType: existing.mimeType } : {}),
746
+ } : undefined);
747
+ const downloaded = cached ?? await source.downloadResource(resource, filename);
748
+ const mimeType = downloaded.mimeType ?? existing?.mimeType ?? resource.mimeType;
749
+ const mappedRemote = existing?.destinationId
750
+ ? remoteResources.find((item) => item.id === existing.destinationId)
751
+ : undefined;
752
+ if (mappedRemote && await remoteMatches(mappedRemote, downloaded, true)) {
753
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: mappedRemote });
754
+ state.resources[resource.id] = {
755
+ sourceId: resource.id,
756
+ sourceName: resource.name,
757
+ destinationId: mappedRemote.id,
758
+ destinationName: mappedRemote.name,
759
+ sha256: downloaded.sha256,
760
+ size: downloaded.size,
761
+ cachePath: downloaded.path,
762
+ ...(mimeType ? { mimeType } : {}),
763
+ status: 'complete',
764
+ updatedAt: new Date().toISOString(),
765
+ };
766
+ await store.setPage({ ...state, updatedAt: new Date().toISOString() });
767
+ updateProgress(`reused ${resource.name}`);
768
+ continue;
769
+ }
770
+ const reconciliationNames = new Set([
771
+ preferredDestinationName(resource),
772
+ ...(existing?.sha256 ? [existing.destinationName] : []),
773
+ ].map((name) => name.toLocaleLowerCase()));
774
+ const reconciled = remoteResources.find((item) => reconciliationNames.has(item.name.toLocaleLowerCase()));
775
+ if (reconciled && await remoteMatches(reconciled, downloaded)) {
776
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: reconciled });
777
+ state.resources[resource.id] = {
778
+ sourceId: resource.id,
779
+ sourceName: resource.name,
780
+ destinationId: reconciled.id,
781
+ destinationName: reconciled.name,
782
+ sha256: downloaded.sha256,
783
+ size: downloaded.size,
784
+ cachePath: downloaded.path,
785
+ ...(mimeType ? { mimeType } : {}),
786
+ status: 'complete',
787
+ updatedAt: new Date().toISOString(),
788
+ };
789
+ await store.setPage({ ...state, updatedAt: new Date().toISOString() });
790
+ updateProgress(`reconciled ${resource.name}`);
791
+ continue;
792
+ }
793
+ const updateExisting = existing?.status === 'complete' && mappedRemote ? mappedRemote : undefined;
794
+ const destinationName = updateExisting?.name ?? nameAllocator.allocate(resource, existing?.status === 'complete' && !nameAllocator.isOccupied(existing.destinationName) ? existing.destinationName : undefined);
795
+ state.resources[resource.id] = {
796
+ sourceId: resource.id,
797
+ sourceName: resource.name,
798
+ destinationName,
799
+ sha256: downloaded.sha256,
615
800
  size: downloaded.size,
616
- ...(downloaded.mimeType ?? resource.mimeType ? { mimeType: downloaded.mimeType ?? resource.mimeType } : {}),
617
- ...(existing?.destinationId ? { existingId: existing.destinationId } : {}),
618
- });
619
- bindings.set(key, { kind: 'uploaded', resource: result });
801
+ cachePath: downloaded.path,
802
+ ...(mimeType ? { mimeType } : {}),
803
+ status: 'planned',
804
+ updatedAt: new Date().toISOString(),
805
+ };
806
+ await store.setPage({ ...state, updatedAt: new Date().toISOString() });
807
+ const matchingName = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
808
+ let result = matchingName && await remoteMatches(matchingName, downloaded) ? matchingName : undefined;
809
+ if (!result) {
810
+ try {
811
+ result = await destination.uploadResource({
812
+ page: destinationPage,
813
+ name: destinationName,
814
+ filePath: downloaded.path,
815
+ size: downloaded.size,
816
+ ...(mimeType ? { mimeType } : {}),
817
+ ...(updateExisting ? { existingId: updateExisting.id } : {}),
818
+ });
819
+ }
820
+ catch (uploadError) {
821
+ remoteResources = await destination.listUploadedResources(destinationPage);
822
+ const uncertain = remoteResources.find((item) => item.name.toLocaleLowerCase() === destinationName.toLocaleLowerCase());
823
+ if (uncertain && await remoteMatches(uncertain, downloaded))
824
+ result = uncertain;
825
+ else {
826
+ const retryName = nameAllocator.allocate(resource);
827
+ result = await destination.uploadResource({
828
+ page: destinationPage,
829
+ name: retryName,
830
+ filePath: downloaded.path,
831
+ size: downloaded.size,
832
+ ...(mimeType ? { mimeType } : {}),
833
+ }).catch((retryError) => { throw new Error(`${retryError instanceof Error ? retryError.message : String(retryError)} (initial upload: ${uploadError instanceof Error ? uploadError.message : String(uploadError)})`); });
834
+ }
835
+ }
836
+ }
837
+ if (!remoteResources.some((item) => item.id === result.id))
838
+ remoteResources.push(result);
839
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: result });
620
840
  state.resources[resource.id] = {
621
841
  sourceId: resource.id,
622
842
  sourceName: resource.name,
@@ -624,6 +844,8 @@ export class Migrator {
624
844
  destinationName: result.name,
625
845
  sha256: downloaded.sha256,
626
846
  size: downloaded.size,
847
+ cachePath: downloaded.path,
848
+ ...(mimeType ? { mimeType } : {}),
627
849
  status: 'complete',
628
850
  updatedAt: new Date().toISOString(),
629
851
  };
@@ -638,34 +860,33 @@ export class Migrator {
638
860
  progress.resources += 1;
639
861
  }
640
862
  }
641
- if (!options.dryRun && resources.some((resource) => resource.kind !== 'grid')) {
863
+ if (!options.dryRun && transferableResources.length > 0) {
642
864
  try {
643
- const remoteResources = await destination.listUploadedResources(destinationPage);
644
- for (const resource of resources.filter((item) => item.kind !== 'grid')) {
865
+ remoteResources = await destination.listUploadedResources(destinationPage);
866
+ for (const resource of transferableResources) {
645
867
  const transferred = state.resources[resource.id];
646
868
  if (transferred?.status !== 'complete')
647
869
  continue;
648
870
  const remote = remoteResources.find((item) => item.id === transferred.destinationId || item.name === transferred.destinationName);
649
- const sizeMismatch = destination.kind === 'confluence'
650
- && transferred.size !== undefined
871
+ const sizeMismatch = transferred.size !== undefined
651
872
  && remote?.size !== undefined
652
873
  && transferred.size !== remote.size;
653
874
  if (!remote || sizeMismatch) {
654
- bindings.set(resource.name.toLocaleLowerCase(), sourceResourceBindings(cachedPage, [resource]).get(resource.name.toLocaleLowerCase()));
875
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
655
876
  await recordResourceFailure(resource, !remote
656
877
  ? `Verification did not find destination resource for ${resource.name}`
657
878
  : `Verification size mismatch for ${resource.name}: ${transferred.size} source bytes, ${remote.size} destination bytes`);
658
879
  }
659
880
  else {
660
- bindings.set(resource.name.toLocaleLowerCase(), { kind: 'uploaded', resource: remote });
881
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, { kind: 'uploaded', resource: remote });
661
882
  }
662
883
  }
663
884
  }
664
885
  catch (error) {
665
- for (const resource of resources.filter((item) => item.kind !== 'grid')) {
886
+ for (const resource of transferableResources) {
666
887
  if (state.resources[resource.id]?.status !== 'complete')
667
888
  continue;
668
- bindings.set(resource.name.toLocaleLowerCase(), sourceResourceBindings(cachedPage, [resource]).get(resource.name.toLocaleLowerCase()));
889
+ setSourceResourceBinding(bindings, cachedPage, resources, resource, fallbackBinding(resource));
669
890
  await recordResourceFailure(resource, `Destination resource verification failed for ${resource.name}: ${error instanceof Error ? error.message : String(error)}`);
670
891
  }
671
892
  }
@@ -677,7 +898,6 @@ export class Migrator {
677
898
  currentPageId: page.canonicalId,
678
899
  };
679
900
  const rendered = await renderRemoteDocument({ destination, container, document, page: cachedPage, context });
680
- warnings += rendered.warnings.length;
681
901
  if (rendered.degraded && !options.dryRun) {
682
902
  const extension = page.rawFormat === 'yfm' ? 'yfm' : 'storage.xml';
683
903
  const snapshotName = `ywtc-source-${sha256(page.canonicalId).slice(0, 8)}.${extension}`;
@@ -714,7 +934,6 @@ export class Migrator {
714
934
  }
715
935
  catch (error) {
716
936
  const detail = `Raw source snapshot was not transferred: ${error instanceof Error ? error.message : String(error)}`;
717
- errors += 1;
718
937
  resourceProblems.push(detail);
719
938
  report.error(`${page.title}: ${detail}`);
720
939
  state.resources[snapshotKey] = {
@@ -726,11 +945,12 @@ export class Migrator {
726
945
  updatedAt: new Date().toISOString(),
727
946
  };
728
947
  }
729
- state.degradations = [...new Set([...state.degradations, ...rendered.warnings, ...resourceProblems])];
948
+ state.degradations = [...new Set([...rendered.warnings, ...resourceProblems])];
730
949
  }
731
950
  let verified = destinationPage;
951
+ let verificationDegradations = [];
732
952
  if (!options.dryRun) {
733
- const current = await destination.verifyPage(destinationPage);
953
+ const current = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
734
954
  if (destinationPage.contentFingerprint
735
955
  && current.contentFingerprint
736
956
  && current.contentFingerprint !== destinationPage.contentFingerprint) {
@@ -746,10 +966,11 @@ export class Migrator {
746
966
  state.destinationFingerprint = verified.fingerprint;
747
967
  await store.setPage({ ...state, updatedAt: new Date().toISOString() });
748
968
  }
749
- verified = await destination.verifyPage(verified);
750
- assertDestinationContent(rendered.body, verified.content, destination.kind, cachedPage);
969
+ verified = await verifyConfirmedDestinationPage(destination, verified, this.consistencyOptions);
970
+ const verification = assertDestinationContent(rendered.body, verified.content, destination.kind, cachedPage);
971
+ verificationDegradations = verification.status === 'partial' ? verification.differences : [];
751
972
  }
752
- const partial = rendered.degraded || resourceProblems.length > 0;
973
+ const partial = rendered.degraded || resourceProblems.length > 0 || verificationDegradations.length > 0;
753
974
  const lastResourceProblem = resourceProblems.at(-1);
754
975
  const clearedState = { ...state };
755
976
  delete clearedState.lastError;
@@ -761,7 +982,7 @@ export class Migrator {
761
982
  destinationTitle: verified.title,
762
983
  ...(verified.fingerprint ? { destinationFingerprint: verified.fingerprint } : {}),
763
984
  status: options.dryRun ? 'planned' : partial ? 'partial' : 'complete',
764
- degradations: [...new Set([...state.degradations, ...rendered.warnings, ...resourceProblems])],
985
+ degradations: [...new Set([...rendered.warnings, ...resourceProblems, ...verificationDegradations])],
765
986
  ...(lastResourceProblem ? { lastError: lastResourceProblem } : {}),
766
987
  updatedAt: new Date().toISOString(),
767
988
  };
@@ -778,18 +999,19 @@ export class Migrator {
778
999
  status: finalState.status,
779
1000
  resources,
780
1001
  document,
1002
+ resourceSizes: finalState.resources,
781
1003
  resourceFallbacks: fallbackDetails,
782
- warnings: [...rendered.warnings, ...resourceProblems],
1004
+ warnings: [...rendered.warnings, ...verificationDegradations],
1005
+ normalizations: rendered.normalizations,
783
1006
  }), { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(verified.content ?? rendered.body, destination.kind) });
784
1007
  }
785
1008
  catch (error) {
786
- errors += 1;
787
1009
  const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
788
1010
  let failureStatus = /conflict/i.test(message) ? 'conflict' : 'failed';
789
1011
  let actual;
790
1012
  if (failureStatus !== 'conflict' && !options.dryRun) {
791
1013
  try {
792
- actual = await destination.verifyPage(destinationPage);
1014
+ actual = await verifyConfirmedDestinationPage(destination, destinationPage, this.consistencyOptions);
793
1015
  assertDestinationContent(bootstrap.body, actual.content, destination.kind, cachedPage);
794
1016
  failureStatus = 'partial';
795
1017
  }
@@ -817,6 +1039,7 @@ export class Migrator {
817
1039
  status: failureStatus,
818
1040
  resources,
819
1041
  document,
1042
+ resourceSizes: state.resources,
820
1043
  resourceFallbacks: fallbackDetails,
821
1044
  warnings: resourceProblems,
822
1045
  error: message,
@@ -832,7 +1055,6 @@ export class Migrator {
832
1055
  const rejected = settled.find((result) => result.status === 'rejected');
833
1056
  if (rejected)
834
1057
  throw rejected.reason;
835
- return { errors, warnings };
836
1058
  }
837
1059
  }
838
1060
  //# sourceMappingURL=migrator.js.map