@kontent-ai/migration-toolkit 2.9.1 → 3.0.0-0

package/lib/toolkit/import.ts

@@ -1,37 +1,37 @@
-import chalk from 'chalk';
-import { writeFile } from 'fs/promises';
-import { executeWithTrackingAsync } from '../core/index.js';
-import type { ImportConfig, ImportResult } from '../import/index.js';
-import { importManager as _importManager } from '../import/index.js';
-import { libMetadata } from '../metadata.js';
+import { writeFile } from "node:fs/promises";
+import chalk from "chalk";
+import { executeWithTrackingAsync } from "../core/utils/global.utils.js";
+import type { ImportConfig, ImportResult } from "../import/import.models.js";
+import { importManager as _importManager } from "../import/import-manager.js";
+import { libMetadata } from "../metadata.js";
 
 export async function importAsync(config: ImportConfig): Promise<ImportResult> {
-    return await executeWithTrackingAsync({
-        event: {
-            tool: 'migrationToolkit',
-            package: {
-                name: libMetadata.name,
-                version: libMetadata.version
-            },
-            action: 'import',
-            relatedEnvironmentId: undefined,
-            details: {}
-        },
-        func: async () => {
-            const importManager = _importManager(config);
-            const importResult = await importManager.importAsync();
+	return await executeWithTrackingAsync({
+		event: {
+			tool: "migrationToolkit",
+			package: {
+				name: libMetadata.name,
+				version: libMetadata.version,
+			},
+			action: "import",
+			relatedEnvironmentId: undefined,
+			details: {},
+		},
+		func: async () => {
+			const importManager = _importManager(config);
+			const importResult = await importManager.importAsync();
 
-            if (config.createReportFile) {
-                const reportFile = importManager.getReportFile(importResult);
-                await writeFile(reportFile.filename, reportFile.content);
-                config.logger?.log({
-                    type: 'writeFs',
-                    message: `Report '${chalk.yellow(reportFile.filename)}' was created`
-                });
-            }
+			if (config.createReportFile) {
+				const reportFile = importManager.getReportFile(importResult);
+				await writeFile(reportFile.filename, reportFile.content);
+				config.logger?.log({
+					type: "writeFs",
+					message: `Report '${chalk.yellow(reportFile.filename)}' was created`,
+				});
+			}
 
-            return importResult;
-        },
-        logger: config.logger
-    });
+			return importResult;
+		},
+		logger: config.logger,
+	});
 }

package/lib/toolkit/migrate.ts

@@ -1,64 +1,75 @@
-import type { IRetryStrategyOptions } from '@kontent-ai/core-sdk';
-import type { ExternalIdGenerator, Logger, ManagementClientConfig, MigrationData } from '../core/index.js';
-import { executeWithTrackingAsync, getDefaultLogger } from '../core/index.js';
-import type { SourceExportItem } from '../export/index.js';
-import type { ImportResult } from '../import/index.js';
-import { libMetadata } from '../metadata.js';
-import { exportAsync } from './export.js';
-import { importAsync } from './import.js';
+import type { IRetryStrategyOptions } from "@kontent-ai/core-sdk";
+import { getDefaultLogger } from "../core/logs/loggers.js";
+import type { Logger } from "../core/models/log.models.js";
+import type { MigrationData } from "../core/models/migration.models.js";
+import type { ExternalIdGenerator } from "../core/utils/external-id.utils.js";
+import { executeWithTrackingAsync } from "../core/utils/global.utils.js";
+import type { ManagementClientConfig } from "../core/utils/management-client-utils.js";
+import type { SourceExportItem } from "../export/export.models.js";
+import type { ImportResult } from "../import/import.models.js";
+import { libMetadata } from "../metadata.js";
+import { exportAsync } from "./export.js";
+import { importAsync } from "./import.js";
 
 export interface MigrationSource extends ManagementClientConfig {
-    readonly items: readonly SourceExportItem[];
+	readonly items: readonly SourceExportItem[];
+	/**
+	 * When enabled, the export process will skip missing items and assets instead of throwing errors.
+	 * Missing references will be filtered out from the exported data.
+	 * Default: false
+	 */
+	readonly skipMissingReferences?: boolean;
 }
 
 export interface MigrationConfig {
-    readonly retryStrategy?: IRetryStrategyOptions;
-    readonly externalIdGenerator?: ExternalIdGenerator;
-    readonly logger?: Logger;
-    readonly sourceEnvironment: MigrationSource;
-    readonly targetEnvironment: ManagementClientConfig;
+	readonly retryStrategy?: IRetryStrategyOptions;
+	readonly externalIdGenerator?: ExternalIdGenerator;
+	readonly logger?: Logger;
+	readonly sourceEnvironment: MigrationSource;
+	readonly targetEnvironment: ManagementClientConfig;
 }
 
 export interface MigrationResult {
-    readonly migrationData: MigrationData;
-    readonly importResult: ImportResult;
+	readonly migrationData: MigrationData;
+	readonly importResult: ImportResult;
 }
 
 export async function migrateAsync(config: MigrationConfig): Promise<MigrationResult> {
-    const logger = config.logger ?? getDefaultLogger();
+	const logger = config.logger ?? getDefaultLogger();
 
-    return await executeWithTrackingAsync({
-        event: {
-            tool: 'migrationToolkit',
-            package: {
-                name: libMetadata.name,
-                version: libMetadata.version
-            },
-            action: 'migrate',
-            relatedEnvironmentId: undefined,
-            details: {
-                itemsCount: config.sourceEnvironment.items.length
-            }
-        },
-        func: async () => {
-            const migrationData = await exportAsync({
-                ...config.sourceEnvironment,
-                logger: logger,
-                exportItems: config.sourceEnvironment.items
-            });
+	return await executeWithTrackingAsync({
+		event: {
+			tool: "migrationToolkit",
+			package: {
+				name: libMetadata.name,
+				version: libMetadata.version,
+			},
+			action: "migrate",
+			relatedEnvironmentId: undefined,
+			details: {
+				itemsCount: config.sourceEnvironment.items.length,
+			},
+		},
+		func: async () => {
+			const migrationData = await exportAsync({
+				...config.sourceEnvironment,
+				logger: logger,
+				exportItems: config.sourceEnvironment.items,
+				skipMissingReferences: config.sourceEnvironment.skipMissingReferences,
+			});
 
-            const importResult = await importAsync({
-                ...config.targetEnvironment,
-                logger: logger,
-                data: migrationData,
-                externalIdGenerator: config.externalIdGenerator
-            });
+			const importResult = await importAsync({
+				...config.targetEnvironment,
+				logger: logger,
+				data: migrationData,
+				externalIdGenerator: config.externalIdGenerator,
+			});
 
-            return {
-                importResult,
-                migrationData
-            };
-        },
-        logger: config.logger
-    });
+			return {
+				importResult,
+				migrationData,
+			};
+		},
+		logger: config.logger,
+	});
 }

package/lib/translation/extraction/items-extraction.processor.ts

@@ -1,188 +1,218 @@
-import type { ElementModels } from '@kontent-ai/management-sdk';
-import { match, P } from 'ts-pattern';
-import type {
-    MigrationElementModels,
-    MigrationElements,
-    ReferencedDataInLanguageVariants,
-    ReferencedDataInMigrationItems
-} from '../../core/index.js';
-import { isArray, isString, parseAsMigrationReferencesArray } from '../../core/index.js';
-import type { GetFlattenedElementByIds } from '../../export/export.models.js';
-import type { GetFlattenedElementByCodenames } from '../../import/index.js';
-import { richTextProcessor } from '../index.js';
+import { isArray } from "node:util";
+import type { ElementModels } from "@kontent-ai/management-sdk";
+import { match, P } from "ts-pattern";
+import type { ReferencedDataInLanguageVariants, ReferencedDataInMigrationItems } from "../../core/models/core.models.js";
+import type { MigrationElementModels, MigrationElements } from "../../core/models/migration.models.js";
+import { parseAsMigrationReferencesArray } from "../../core/utils/array.utils.js";
+import { isString } from "../../core/utils/global.utils.js";
+import type { GetFlattenedElementByIds } from "../../export/export.models.js";
+import type { GetFlattenedElementByCodenames } from "../../import/import.models.js";
+import { richTextProcessor } from "../helpers/rich-text.processor.js";
 
 export interface ExtractItemById {
-    readonly elements: readonly Readonly<ElementModels.ContentItemElement>[];
-    readonly contentTypeId: string;
+	readonly elements: readonly Readonly<ElementModels.ContentItemElement>[];
+	readonly contentTypeId: string;
 }
 
 export interface ExtractItemByCodename {
-    readonly elements: MigrationElements;
-    readonly contentTypeCodename: string;
+	readonly elements: MigrationElements;
+	readonly contentTypeCodename: string;
 }
 
 export interface ReferencedDataInMigrationItemsLocal {
-    readonly itemCodenames: Set<string>;
-    readonly assetCodenames: Set<string>;
+	readonly itemCodenames: Set<string>;
+	readonly assetCodenames: Set<string>;
 }
 
 export interface ReferencedDataInLanguageVariantsLocal {
-    readonly itemIds: Set<string>;
-    readonly assetIds: Set<string>;
+	readonly itemIds: Set<string>;
+	readonly assetIds: Set<string>;
 }
 
 export function itemsExtractionProcessor() {
-    const extractReferencedDataFromExtractItems = (
-        items: readonly ExtractItemById[],
-        getElement: GetFlattenedElementByIds
-    ): ReferencedDataInLanguageVariants => {
-        const extractedIds = items.reduce<ReferencedDataInLanguageVariantsLocal>(
-            (extractedIds, item) => {
-                return item.elements.reduce<ReferencedDataInLanguageVariantsLocal>((childExtractedIds, itemElement) => {
-                    const typeElement = getElement(item.contentTypeId, itemElement.element.id ?? '');
-
-                    match(typeElement.type)
-                        .with('rich_text', () => {
-                            const rteValue = itemElement.value && isString(itemElement.value) ? itemElement.value : '';
-
-                            // extract referenced items
-                            richTextProcessor()
-                                .processDataIds(rteValue)
-                                .ids.forEach((id) => extractedIds.itemIds.add(id));
-
-                            richTextProcessor()
-                                .processLinkItemIds(rteValue)
-                                .ids.forEach((id) => extractedIds.itemIds.add(id));
-
-                            // extract referenced assets
-                            richTextProcessor()
-                                .processAssetIds(rteValue)
-                                .ids.forEach((id) => extractedIds.assetIds.add(id));
-
-                            richTextProcessor()
-                                .processLinkAssetIds(rteValue)
-                                .ids.forEach((id) => extractedIds.assetIds.add(id));
-
-                            // recursively extract data from components as well because they may reference additional assets & content items
-                            const extractedComponents = extractReferencedDataFromExtractItems(
-                                itemElement.components.map((component) => {
-                                    return {
-                                        contentTypeId: component.type.id ?? '',
-                                        elements: component.elements
-                                    };
-                                }),
-                                getElement
-                            );
-
-                            extractedComponents.itemIds.forEach((id) => extractedIds.itemIds.add(id));
-                            extractedComponents.assetIds.forEach((id) => extractedIds.assetIds.add(id));
-                        })
-                        .with(P.union('modular_content', 'subpages'), () => {
-                            if (itemElement.value && isArray(itemElement.value)) {
-                                itemElement.value.forEach((value) => (value.id ? extractedIds.itemIds.add(value.id) : {}));
-                            }
-                        })
-                        .with('asset', () => {
-                            if (itemElement.value && isArray(itemElement.value)) {
-                                itemElement.value.forEach((value) => (value.id ? extractedIds.assetIds.add(value.id) : {}));
-                            }
-                        })
-                        .otherwise(() => {});
-
-                    return childExtractedIds;
-                }, extractedIds);
-            },
-            { itemIds: new Set(), assetIds: new Set() }
-        );
-
-        return {
-            itemIds: extractedIds.itemIds,
-            assetIds: extractedIds.assetIds
-        };
-    };
-
-    const extractReferencedItemsFromMigrationItems = (
-        items: readonly ExtractItemByCodename[],
-        getElement: GetFlattenedElementByCodenames
-    ): ReferencedDataInMigrationItems => {
-        const extractedCodenames = items.reduce<ReferencedDataInMigrationItemsLocal>(
-            (extractedCodenames, item) => {
-                return Object.entries(item.elements).reduce<ReferencedDataInMigrationItemsLocal>(
-                    (childExtractedCodenames, [elementCodename, element]) => {
-                        const flattenedElement = getElement(item.contentTypeCodename, elementCodename, element.type);
-                        match(flattenedElement.type)
-                            .with('rich_text', () => {
-                                const richTextElementValue = element as MigrationElementModels.RichTextElement;
-                                const richTextHtml = richTextElementValue?.value ?? '';
-
-                                // items
-                                richTextProcessor()
-                                    .processItemCodenames(richTextHtml)
-                                    .codenames.forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
-
-                                richTextProcessor()
-                                    .processLinkItemCodenames(richTextHtml)
-                                    .codenames.forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
-
-                                // assets
-                                richTextProcessor()
-                                    .processAssetCodenames(richTextHtml)
-                                    .codenames.forEach((codename) => childExtractedCodenames.assetCodenames.add(codename));
-
-                                richTextProcessor()
-                                    .processLinkAssetCodenames(richTextHtml)
-                                    .codenames.forEach((codename) => childExtractedCodenames.assetCodenames.add(codename));
-
-                                // recursively extract data from components as well because they may reference additional assets & content items
-                                const extractedComponents = extractReferencedItemsFromMigrationItems(
-                                    (element.components ?? []).map((component) => {
-                                        const extractionItem: ExtractItemByCodename = {
-                                            contentTypeCodename: component.system.type.codename,
-                                            elements: component.elements
-                                        };
-
-                                        return extractionItem;
-                                    }),
-                                    getElement
-                                );
-
-                                extractedComponents.itemCodenames.forEach((codename) =>
-                                    childExtractedCodenames.itemCodenames.add(codename)
-                                );
-                                extractedComponents.assetCodenames.forEach((codename) =>
-                                    childExtractedCodenames.assetCodenames.add(codename)
-                                );
-                            })
-                            .with(P.union('modular_content', 'subpages'), () => {
-                                parseAsMigrationReferencesArray(element.value)
-                                    .map((m) => m.codename)
-                                    .forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
-                            })
-                            .with('asset', () => {
-                                parseAsMigrationReferencesArray(element.value)
-                                    .map((m) => m.codename)
-                                    .forEach((codename) => childExtractedCodenames.assetCodenames.add(codename));
-                            });
-
-                        return childExtractedCodenames;
-                    },
-                    extractedCodenames
-                );
-            },
-            {
-                itemCodenames: new Set(),
-                assetCodenames: new Set()
-            }
-        );
-
-        return {
-            itemCodenames: extractedCodenames.itemCodenames,
-            assetCodenames: extractedCodenames.assetCodenames
-        };
-    };
-
-    return {
-        extractReferencedDataFromExtractItems,
-        extractReferencedItemsFromMigrationItems
-    };
+	const extractReferencedDataFromExtractItems = (
+		items: readonly ExtractItemById[],
+		getElement: GetFlattenedElementByIds,
+	): ReferencedDataInLanguageVariants => {
+		const extractedIds = items.reduce<ReferencedDataInLanguageVariantsLocal>(
+			(extractedIds, item) => {
+				return item.elements.reduce<ReferencedDataInLanguageVariantsLocal>((childExtractedIds, itemElement) => {
+					const typeElement = getElement(item.contentTypeId, itemElement.element.id ?? "");
+
+					match(typeElement.type)
+						.with("rich_text", () => {
+							const rteValue = itemElement.value && isString(itemElement.value) ? itemElement.value : "";
+
+							// extract referenced items
+							richTextProcessor()
+								.processDataIds(rteValue)
+								.ids.forEach((id) => {
+									extractedIds.itemIds.add(id);
+								});
+
+							richTextProcessor()
+								.processLinkItemIds(rteValue)
+								.ids.forEach((id) => {
+									extractedIds.itemIds.add(id);
+								});
+
+							// extract referenced assets
+							richTextProcessor()
+								.processAssetIds(rteValue)
+								.ids.forEach((id) => {
+									extractedIds.assetIds.add(id);
+								});
+
+							richTextProcessor()
+								.processLinkAssetIds(rteValue)
+								.ids.forEach((id) => {
+									extractedIds.assetIds.add(id);
+								});
+
+							// recursively extract data from components as well because they may reference additional assets & content items
+							const extractedComponents = extractReferencedDataFromExtractItems(
+								itemElement.components.map((component) => {
+									return {
+										contentTypeId: component.type.id ?? "",
+										elements: component.elements,
+									};
+								}),
+								getElement,
+							);
+
+							extractedComponents.itemIds.forEach((id) => {
+								extractedIds.itemIds.add(id);
+							});
+							extractedComponents.assetIds.forEach((id) => {
+								extractedIds.assetIds.add(id);
+							});
+						})
+						.with(P.union("modular_content", "subpages"), () => {
+							if (itemElement.value && isArray(itemElement.value)) {
+								itemElement.value.forEach((value) => {
+									if (value.id) {
+										extractedIds.itemIds.add(value.id);
+									}
+								});
+							}
+						})
+						.with("asset", () => {
+							if (itemElement.value && isArray(itemElement.value)) {
+								itemElement.value.forEach((value) => {
+									if (value.id) {
+										extractedIds.assetIds.add(value.id);
+									}
+								});
+							}
+						})
+						.otherwise(() => {});
+
+					return childExtractedIds;
+				}, extractedIds);
+			},
+			{ itemIds: new Set(), assetIds: new Set() },
+		);
+
+		return {
+			itemIds: extractedIds.itemIds,
+			assetIds: extractedIds.assetIds,
+		};
+	};
+
+	const extractReferencedItemsFromMigrationItems = (
+		items: readonly ExtractItemByCodename[],
+		getElement: GetFlattenedElementByCodenames,
+	): ReferencedDataInMigrationItems => {
+		const extractedCodenames = items.reduce<ReferencedDataInMigrationItemsLocal>(
+			(extractedCodenames, item) => {
+				return Object.entries(item.elements).reduce<ReferencedDataInMigrationItemsLocal>(
+					(childExtractedCodenames, [elementCodename, element]) => {
+						const flattenedElement = getElement(item.contentTypeCodename, elementCodename, element.type);
+						match(flattenedElement.type)
+							.with("rich_text", () => {
+								const richTextElementValue = element as MigrationElementModels.RichTextElement;
+								const richTextHtml = richTextElementValue.value ?? "";
+
+								// items
+								richTextProcessor()
+									.processItemCodenames(richTextHtml)
+									.codenames.forEach((codename) => {
+										childExtractedCodenames.itemCodenames.add(codename);
+									});
+
+								richTextProcessor()
+									.processLinkItemCodenames(richTextHtml)
+									.codenames.forEach((codename) => {
+										childExtractedCodenames.itemCodenames.add(codename);
+									});
+
+								// assets
+								richTextProcessor()
+									.processAssetCodenames(richTextHtml)
+									.codenames.forEach((codename) => {
+										childExtractedCodenames.assetCodenames.add(codename);
+									});
+
+								richTextProcessor()
+									.processLinkAssetCodenames(richTextHtml)
+									.codenames.forEach((codename) => {
+										childExtractedCodenames.assetCodenames.add(codename);
+									});
+
+								// recursively extract data from components as well because they may reference additional assets & content items
+								const extractedComponents = extractReferencedItemsFromMigrationItems(
+									(element.components ?? []).map((component) => {
+										const extractionItem: ExtractItemByCodename = {
+											contentTypeCodename: component.system.type.codename,
+											elements: component.elements,
+										};
+
+										return extractionItem;
+									}),
+									getElement,
+								);
+
+								extractedComponents.itemCodenames.forEach((codename) => {
+									childExtractedCodenames.itemCodenames.add(codename);
+								});
+								extractedComponents.assetCodenames.forEach((codename) => {
+									childExtractedCodenames.assetCodenames.add(codename);
+								});
+							})
+							.with(P.union("modular_content", "subpages"), () => {
+								parseAsMigrationReferencesArray(element.value)
+									.map((m) => m.codename)
+									.forEach((codename) => {
+										childExtractedCodenames.itemCodenames.add(codename);
+									});
+							})
+							.with("asset", () => {
+								parseAsMigrationReferencesArray(element.value)
+									.map((m) => m.codename)
+									.forEach((codename) => {
+										childExtractedCodenames.assetCodenames.add(codename);
+									});
+							});
+
+						return childExtractedCodenames;
+					},
+					extractedCodenames,
+				);
+			},
+			{
+				itemCodenames: new Set(),
+				assetCodenames: new Set(),
+			},
+		);
+
+		return {
+			itemCodenames: extractedCodenames.itemCodenames,
+			assetCodenames: extractedCodenames.assetCodenames,
+		};
+	};
+
+	return {
+		extractReferencedDataFromExtractItems,
+		extractReferencedItemsFromMigrationItems,
+	};
 }

package/lib/translation/helpers/export-placeholder.utils.ts

@@ -0,0 +1,10 @@
+/**
+ * Returns a placeholder that indicates missing reference, this placeholder is not actually used anywhere
+ * as missing items will simply be skipped, but may help when debugging in future
+ * @param type - The type of the reference
+ * @param id - The id of the reference
+ * @returns A placeholder that indicates missing reference
+ */
+export function getMissingReferencePlaceholder({ type, id }: { readonly type: "item" | "asset"; readonly id: string }): string {
+	return `#missing:${type}:${id}#`;
+}