@iwonz/ywtc 3.2.1 → 3.2.2

package/dist/migrator.js

@@ -9,7 +9,7 @@ import { assertTransferCapabilities } from './domain.js';
 import { FilesystemExporter } from './filesystem-export.js';
 import { discoverPageGraph } from './graph.js';
 import { profileBaseUrl, sameRemoteInstance } from './locators.js';
-import { TransferReport } from './report.js';
+import { TransferReport, } from './report.js';
 import { DestinationNameAllocator, preferredDestinationName, verifiedCacheFile } from './resource-reconciliation.js';
 import { resolveStateLocation, StateStore, } from './state.js';
 function sourceIdentity(locator, root) {
@@ -119,8 +119,24 @@ function destinationFromState(state, adapter) {
 }
 function reportPage(input) {
     const transferable = input.resources.filter((resource) => resource.kind !== 'grid');
-    const sizes = transferable.map((resource) => input.resourceSizes?.[resource.id]?.size ?? resource.size);
-    const knownBytes = sizes.reduce((total, size) => total + (size ?? 0), 0);
+    const resourceItems = transferable.map((resource) => {
+        const state = input.resourceStates?.[resource.id];
+        const sourceBytes = state?.size ?? resource.size;
+        const status = input.resourceStatusOverride ?? state?.status ?? 'planned';
+        return {
+            sourceId: resource.id,
+            sourceName: resource.name,
+            kind: resource.kind,
+            status,
+            ...(sourceBytes !== undefined ? { sourceBytes } : {}),
+            ...(status === 'complete' && sourceBytes !== undefined ? { transferredBytes: sourceBytes } : {}),
+            ...(state?.destinationId ? { destinationId: state.destinationId } : {}),
+            ...(state?.destinationName ? { destinationName: state.destinationName } : {}),
+            ...(state?.lastError ? { error: state.lastError } : {}),
+        };
+    });
+    const knownBytes = resourceItems.reduce((total, resource) => total + (resource.sourceBytes ?? 0), 0);
+    const transferredBytes = resourceItems.reduce((total, resource) => total + (resource.transferredBytes ?? 0), 0);
     return {
         sourceCanonicalId: input.page.canonicalId,
         sourceId: input.page.id,
@@ -134,12 +150,16 @@ function reportPage(input) {
         ...(input.page.projectedParentId ? { parentId: input.page.projectedParentId } : {}),
         depth: input.page.depth,
         status: input.status,
+        inventoryStatus: input.inventoryStatus ?? 'complete',
         resources: transferable.length,
-        ...(transferable.length === 0 || sizes.some((size) => size !== undefined) ? { bytes: knownBytes } : {}),
-        bytesUnknown: sizes.filter((size) => size === undefined).length,
+        resourceItems,
+        ...(transferable.length === 0 || resourceItems.some((resource) => resource.sourceBytes !== undefined) ? { sourceBytes: knownBytes } : {}),
+        transferredBytes,
+        bytesUnknown: resourceItems.filter((resource) => resource.sourceBytes === undefined).length,
         links: input.document.metrics.links,
         images: input.document.metrics.images,
         tables: input.document.metrics.tables,
+        grids: input.resources.filter((resource) => resource.kind === 'grid').length,
         codeBlocks: input.document.metrics.codeBlocks,
         degraded: input.document.degraded || (input.resourceFallbacks?.length ?? 0) > 0,
         resourceFallbacks: input.resourceFallbacks ?? [],
@@ -148,10 +168,13 @@ function reportPage(input) {
         ...(input.error ? { error: input.error } : {}),
     };
 }
-function runStatus(errors, warnings) {
+function runStatus(errors, warnings, degradations) {
     if (errors > 0)
         return 'partial';
-    return warnings > 0 ? 'completed-with-warnings' : 'completed';
+    return warnings + degradations > 0 ? 'completed-with-warnings' : 'completed';
+}
+function attentionCount(summary) {
+    return summary.warnings + summary.degradations;
 }
 export class MigrationRunError extends Error {
     reportPaths;
@@ -233,6 +256,7 @@ export class Migrator {
                 ...(store.loadInfo.backupFilename ? { backupPath: store.loadInfo.backupFilename } : {}),
             },
         });
+        report.registerPages([{ ...root, depth: 0 }]);
         let graph;
         let sourceFingerprints;
         try {
@@ -244,6 +268,10 @@ export class Migrator {
                 ...(options.signal ? { signal: options.signal } : {}),
                 onProgress: (progress) => status(`Discovery: ${progress.pages} pages, ${progress.edges} links${progress.active ? ` · ${progress.active}` : ''}`),
             });
+            report.registerPages(graph.pages);
+            report.setEdges(graph.edges);
+            for (const warning of graph.warnings)
+                report.warning(warning);
             if (container
                 && options.destination.type === 'remote'
                 && sameRemoteInstance(options.source.profile, options.destination.profile)
@@ -253,20 +281,25 @@ export class Migrator {
             status(`Caching source content for ${graph.pages.length} pages`);
             sourceFingerprints = await cacheDiscoveredPages(options.cacheDirectory, graph);
             await store.setGraph(graph.rootId, graph.edges);
-            report.setEdges(graph.edges);
-            for (const warning of graph.warnings)
-                report.warning(warning);
         }
         catch (error) {
             const finalStatus = options.signal?.aborted ? 'cancelled' : 'failed';
             const message = error instanceof Error ? error.message : String(error);
-            report.error(message);
+            report.updatePage(root.canonicalId, { status: 'failed', inventoryStatus: 'failed', error: message });
+            report.issue({
+                code: 'source-discovery-failed',
+                severity: 'error',
+                phase: 'discovery',
+                pageId: root.canonicalId,
+                retryable: true,
+                message,
+            });
             report.finish(finalStatus);
             const reportPaths = await report.write(options.reportsDirectory);
             if (!options.dryRun)
                 await store.finishRun(run.id, finalStatus, {
                     pages: 0,
-                    warnings: report.data.summary.warnings,
+                    warnings: attentionCount(report.data.summary),
                     errors: report.data.summary.errors,
                 });
             throw new MigrationRunError(message, reportPaths, stateLocation.filename, { cause: error });
@@ -293,12 +326,26 @@ export class Migrator {
                 inventory.images += document.metrics.images;
                 inventory.tables += document.metrics.tables;
                 inventory.codeBlocks += document.metrics.codeBlocks;
+                report.addPage(reportPage({
+                    page,
+                    status: 'planned',
+                    resources,
+                    document,
+                }), undefined, false);
                 status(`Inspected ${page.title} · ${resources.length} resources`);
             }
             catch (error) {
                 const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
                 inventoryErrors.push(message);
-                report.error(message);
+                report.updatePage(page.canonicalId, { status: 'failed', inventoryStatus: 'failed', error: message });
+                report.issue({
+                    code: 'source-inventory-failed',
+                    severity: 'error',
+                    phase: 'inventory',
+                    pageId: page.canonicalId,
+                    retryable: true,
+                    message,
+                });
             }
         })));
         if (inventoryErrors.length > 0) {
@@ -308,7 +355,7 @@ export class Migrator {
             if (!options.dryRun)
                 await store.finishRun(run.id, finalStatus, {
                     pages: graph.pages.length,
-                    warnings: report.data.summary.warnings,
+                    warnings: attentionCount(report.data.summary),
                     errors: report.data.summary.errors,
                 });
             throw new MigrationRunError('Resource preflight failed before destination mutation', reportPaths, stateLocation.filename);
@@ -345,16 +392,16 @@ export class Migrator {
                 report.output('destination', container.url);
             }
             report.finish('completed');
-            const finalStatus = runStatus(report.data.summary.errors, report.data.summary.warnings);
+            const finalStatus = runStatus(report.data.summary.errors, report.data.summary.warnings, report.data.summary.degradations);
             report.finish(finalStatus);
             const reportPaths = await report.write(options.reportsDirectory);
             if (!options.dryRun)
                 await store.finishRun(run.id, finalStatus, {
                     pages: graph.pages.length,
-                    warnings: report.data.summary.warnings,
+                    warnings: attentionCount(report.data.summary),
                     errors: report.data.summary.errors,
                 });
-            status(`${finalStatus}: ${graph.pages.length} pages · ${report.data.summary.errors} errors · ${report.data.summary.warnings} warnings`);
+            status(`${finalStatus}: source ${report.data.summary.pages} pages · ${report.data.summary.complete} complete · ${report.data.summary.partial} partial · ${report.data.summary.failed + report.data.summary.blocked} failed/blocked · files ${report.data.summary.resourcesComplete}/${report.data.summary.resources} complete · ${report.data.summary.errors} errors · ${report.data.summary.warnings} warnings · ${report.data.summary.degradations} degradations`);
             return {
                 status: finalStatus,
                 hasErrors: report.data.summary.errors > 0,
@@ -377,7 +424,7 @@ export class Migrator {
             if (!options.dryRun)
                 await store.finishRun(run.id, finalStatus, {
                     pages: graph.pages.length,
-                    warnings: report.data.summary.warnings,
+                    warnings: attentionCount(report.data.summary),
                     errors: report.data.summary.errors,
                 });
             this.logger.error(`${message}. Report: ${reportPaths.html}`);
@@ -422,10 +469,11 @@ export class Migrator {
                         destinationName: resource.name,
                         ...(resource.sha256 ? { sha256: resource.sha256 } : {}),
                         ...(resource.size !== undefined ? { size: resource.size } : {}),
-                        status: 'complete',
+                        status: options.dryRun ? 'planned' : 'complete',
                         updatedAt: new Date().toISOString(),
                     }];
             }));
+            const pageStatus = options.dryRun ? 'planned' : manifestPage?.degraded ? 'partial' : 'complete';
             await store.setPage({
                 sourceId: page.id,
                 sourceCanonicalId: page.canonicalId,
@@ -438,17 +486,18 @@ export class Migrator {
                 destinationTitle: page.title,
                 sourceFingerprint: sourceFingerprints.get(page.canonicalId),
                 ...(manifestPage || previous?.destinationFingerprint ? { destinationFingerprint: manifestPage ? sha256(JSON.stringify(manifestPage)) : previous.destinationFingerprint } : {}),
-                status: options.dryRun ? 'planned' : manifestPage?.degraded ? 'partial' : 'complete',
+                status: pageStatus,
                 resources,
                 degradations: manifestPage?.warnings ?? [],
                 updatedAt: new Date().toISOString(),
             });
             report.addPage(reportPage({
                 page,
-                status: options.dryRun ? 'planned' : result.errors.length ? 'partial' : 'complete',
+                status: pageStatus,
                 resources: sourceResources,
                 document,
-                resourceSizes: resources,
+                resourceStates: resources,
+                ...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
                 ...(manifestPage?.warnings ? { warnings: manifestPage.warnings } : {}),
             }), { source: normalizedPreviewHtml(document), destination: normalizedPreviewHtml(document) });
         }
@@ -461,6 +510,7 @@ export class Migrator {
     async transferRemote(input) {
         const { source, destination, container, graph, options, store, report, sourceFingerprints, resourcesByPage } = input;
         const destinationPages = new Map();
+        const stagingFailures = new Map();
         const sourcePages = new Map(graph.pages.map((page) => [page.canonicalId, page]));
         const bootstrapBodies = new Map();
         const totalResources = [...resourcesByPage.values()].reduce((total, resources) => total + resources.filter((resource) => resource.kind !== 'grid').length, 0);
@@ -496,7 +546,16 @@ export class Migrator {
             const parent = page.projectedParentId ? destinationPages.get(page.projectedParentId) : container;
             if (!parent) {
                 const message = `Destination parent is unavailable for ${page.title}`;
-                report.error(message);
+                stagingFailures.set(page.canonicalId, { status: 'blocked', message });
+                report.updatePage(page.canonicalId, { status: 'blocked', error: message });
+                report.issue({
+                    code: 'destination-parent-unavailable',
+                    severity: 'error',
+                    phase: 'page',
+                    pageId: page.canonicalId,
+                    retryable: true,
+                    message,
+                });
                 progress.staged += 1;
                 updateProgress(message);
                 continue;
@@ -530,21 +589,54 @@ export class Migrator {
                         if (page.projectedParentId) {
                             await verifyConfirmedDestinationPage(destination, parent, this.consistencyOptions);
                         }
-                        const candidate = await (destination.findRecoverablePage?.(parent, title) ?? destination.findChild(parent, title));
+                        const findRecoveryChild = (candidateTitle) => destination.findRecoverablePage
+                            ? destination.findRecoverablePage(parent, candidateTitle)
+                            : destination.findChild(parent, candidateTitle);
+                        let candidate;
+                        let recoveryLookupFailed = false;
+                        try {
+                            candidate = await findRecoveryChild(title);
+                        }
+                        catch (lookupError) {
+                            recoveryLookupFailed = true;
+                            report.issue({
+                                code: 'destination-recovery-lookup-failed',
+                                severity: 'warning',
+                                phase: 'page',
+                                pageId: page.canonicalId,
+                                retryable: true,
+                                message: `${page.title}: recovery lookup failed; bootstrap recreation will be attempted: ${lookupError instanceof Error ? lookupError.message : String(lookupError)}`,
+                            });
+                        }
                         if (candidate && sourceDerivedTextMatches(bootstrap.body, candidate.content, destination.kind, cachedPage)) {
                             created = candidate;
                         }
                         else {
                             let recoveryTitle = title;
-                            const occupied = candidate ?? await destination.findChild(parent, recoveryTitle);
-                            if (occupied) {
+                            if (candidate) {
                                 const base = collisionTitle(page);
                                 const hash = sha256(page.canonicalId).slice(0, 8);
                                 recoveryTitle = base;
                                 let sequence = 0;
-                                while (await destination.findChild(parent, recoveryTitle)) {
-                                    sequence += 1;
-                                    recoveryTitle = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
+                                while (!recoveryLookupFailed) {
+                                    try {
+                                        if (!await findRecoveryChild(recoveryTitle))
+                                            break;
+                                        sequence += 1;
+                                        recoveryTitle = `${base} [${hash}${sequence > 1 ? `-${sequence}` : ''}]`;
+                                    }
+                                    catch (lookupError) {
+                                        recoveryLookupFailed = true;
+                                        recoveryTitle = `${base} [${hash}]`;
+                                        report.issue({
+                                            code: 'destination-recovery-lookup-failed',
+                                            severity: 'warning',
+                                            phase: 'page',
+                                            pageId: page.canonicalId,
+                                            retryable: true,
+                                            message: `${page.title}: suffixed-title lookup failed; bootstrap recreation will be attempted: ${lookupError instanceof Error ? lookupError.message : String(lookupError)}`,
+                                        });
+                                    }
                                 }
                             }
                             created = await destination.createPage({ container: parent, title: recoveryTitle, body: bootstrap.body });
@@ -632,10 +724,20 @@ export class Migrator {
             }
             catch (error) {
                 const message = `${page.title}: ${error instanceof Error ? error.message : String(error)}`;
-                report.error(message);
+                const failureStatus = /conflict/i.test(message) ? 'conflict' : 'failed';
+                stagingFailures.set(page.canonicalId, { status: failureStatus, message });
+                report.updatePage(page.canonicalId, { status: failureStatus, error: message });
+                report.issue({
+                    code: failureStatus === 'conflict' ? 'destination-conflict' : 'destination-page-staging-failed',
+                    severity: 'error',
+                    phase: 'page',
+                    pageId: page.canonicalId,
+                    retryable: failureStatus !== 'conflict',
+                    message,
+                });
                 const savedState = store.value.pages[page.canonicalId];
                 if (savedState)
-                    await store.setPage({ ...savedState, status: /conflict/i.test(message) ? 'conflict' : 'failed', lastError: message, updatedAt: new Date().toISOString() });
+                    await store.setPage({ ...savedState, status: failureStatus, lastError: message, updatedAt: new Date().toISOString() });
             }
             finally {
                 progress.staged += 1;
@@ -651,13 +753,16 @@ export class Migrator {
             const cachedPage = await loadCachedPage(options.cacheDirectory, page.canonicalId);
             const resources = resourcesByPage.get(page.canonicalId) ?? [];
             if (!destinationPage) {
+                const currentFailure = stagingFailures.get(page.canonicalId);
                 const failedState = store.value.pages[page.canonicalId];
                 report.addPage(reportPage({
                     page,
-                    status: failedState?.status ?? 'blocked',
+                    status: currentFailure?.status ?? 'blocked',
                     resources,
                     document,
-                    error: failedState?.lastError ?? 'Destination page with source-derived content was not created',
+                    ...(failedState?.resources ? { resourceStates: failedState.resources } : {}),
+                    ...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
+                    error: currentFailure?.message ?? failedState?.lastError ?? 'Destination page with source-derived content was not created',
                 }));
                 progress.pages += 1;
                 updateProgress(`blocked ${page.title}`);
@@ -906,28 +1011,48 @@ export class Migrator {
                     await writeFile(snapshotPath, cachedPage.rawContent, { mode: 0o600 });
                     const snapshotKey = `snapshot:${page.canonicalId}`;
                     const existingSnapshot = state.resources[snapshotKey];
+                    const snapshotFile = {
+                        path: snapshotPath,
+                        size: Buffer.byteLength(cachedPage.rawContent),
+                        sha256: sha256(cachedPage.rawContent),
+                    };
                     try {
-                        const snapshot = existingSnapshot?.status === 'complete' && existingSnapshot.destinationId
-                            ? {
-                                id: existingSnapshot.destinationId,
-                                name: existingSnapshot.destinationName,
-                                ...(existingSnapshot.size !== undefined ? { size: existingSnapshot.size } : {}),
+                        const knownSnapshot = remoteResources.find((item) => item.id === existingSnapshot?.destinationId)
+                            ?? remoteResources.find((item) => item.name.toLocaleLowerCase() === snapshotName.toLocaleLowerCase());
+                        let snapshot = knownSnapshot && await remoteMatches(knownSnapshot, snapshotFile, true)
+                            ? knownSnapshot
+                            : undefined;
+                        if (!snapshot) {
+                            try {
+                                snapshot = await destination.uploadResource({
+                                    page: destinationPage,
+                                    name: snapshotName,
+                                    filePath: snapshotPath,
+                                    size: snapshotFile.size,
+                                    mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
+                                    ...(knownSnapshot ? { existingId: knownSnapshot.id } : {}),
+                                });
                             }
-                            : await destination.uploadResource({
-                                page: destinationPage,
-                                name: snapshotName,
-                                filePath: snapshotPath,
-                                size: Buffer.byteLength(cachedPage.rawContent),
-                                mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
-                                ...(existingSnapshot?.destinationId ? { existingId: existingSnapshot.destinationId } : {}),
-                            });
+                            catch (uploadError) {
+                                remoteResources = await destination.listUploadedResources(destinationPage);
+                                const uncertain = remoteResources.find((item) => item.name.toLocaleLowerCase() === snapshotName.toLocaleLowerCase());
+                                if (uncertain && await remoteMatches(uncertain, snapshotFile, true))
+                                    snapshot = uncertain;
+                                else
+                                    throw uploadError;
+                            }
+                        }
+                        if (!remoteResources.some((item) => item.id === snapshot.id))
+                            remoteResources.push(snapshot);
                         state.resources[snapshotKey] = {
                             sourceId: snapshotKey,
                             sourceName: snapshotName,
                             destinationId: snapshot.id,
                             destinationName: snapshot.name,
                             ...(snapshot.size !== undefined ? { size: snapshot.size } : {}),
-                            sha256: sha256(cachedPage.rawContent),
+                            sha256: snapshotFile.sha256,
+                            cachePath: snapshotPath,
+                            mimeType: page.rawFormat === 'yfm' ? 'text/markdown' : 'application/xml',
                             status: 'complete',
                             updatedAt: new Date().toISOString(),
                         };
@@ -935,7 +1060,15 @@ export class Migrator {
                     catch (error) {
                         const detail = `Raw source snapshot was not transferred: ${error instanceof Error ? error.message : String(error)}`;
                         resourceProblems.push(detail);
-                        report.error(`${page.title}: ${detail}`);
+                        report.issue({
+                            code: 'source-snapshot-transfer-failed',
+                            severity: 'error',
+                            phase: 'resource',
+                            pageId: page.canonicalId,
+                            resourceId: snapshotKey,
+                            retryable: true,
+                            message: `${page.title}: ${detail}`,
+                        });
                         state.resources[snapshotKey] = {
                             sourceId: snapshotKey,
                             sourceName: snapshotName,
@@ -954,7 +1087,11 @@ export class Migrator {
                     if (destinationPage.contentFingerprint
                         && current.contentFingerprint
                         && current.contentFingerprint !== destinationPage.contentFingerprint) {
-                        throw new Error(`Destination conflict for ${page.title}: content changed while resources were transferred`);
+                        const normalization = verifyDestinationContent(destinationPage.content ?? bootstrap.body, current.content, destination.kind, cachedPage);
+                        if (normalization.status !== 'complete') {
+                            throw new Error(`Destination conflict for ${page.title}: content changed while resources were transferred`);
+                        }
+                        rendered.normalizations.push('Destination normalized page markup without semantic content changes');
                     }
                     verified = await destination.updatePage({
                         page: current,
@@ -999,7 +1136,8 @@ export class Migrator {
                     status: finalState.status,
                     resources,
                     document,
-                    resourceSizes: finalState.resources,
+                    resourceStates: finalState.resources,
+                    ...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
                     resourceFallbacks: fallbackDetails,
                     warnings: [...rendered.warnings, ...verificationDegradations],
                     normalizations: rendered.normalizations,
@@ -1019,7 +1157,14 @@ export class Migrator {
                         failureStatus = 'failed';
                     }
                 }
-                report.error(message);
+                report.issue({
+                    code: failureStatus === 'conflict' ? 'destination-conflict' : 'page-transfer-failed',
+                    severity: 'error',
+                    phase: 'page',
+                    pageId: page.canonicalId,
+                    retryable: failureStatus !== 'conflict',
+                    message,
+                });
                 await store.setPage({
                     ...state,
                     ...(actual?.fingerprint ? { destinationFingerprint: actual.fingerprint } : {}),
@@ -1039,9 +1184,9 @@ export class Migrator {
                     status: failureStatus,
                     resources,
                     document,
-                    resourceSizes: state.resources,
+                    resourceStates: state.resources,
+                    ...(options.dryRun ? { resourceStatusOverride: 'planned' } : {}),
                     resourceFallbacks: fallbackDetails,
-                    warnings: resourceProblems,
                     error: message,
                 }), actual?.content ? { source: normalizedPreviewHtml(document), destination: renderedPreviewHtml(actual.content, destination.kind) } : undefined);
             }