@kontent-ai/migration-toolkit 1.0.0

package/lib/import/context/import-context-fetcher.ts

@@ -0,0 +1,367 @@
+import { AssetModels, ContentItemModels, LanguageVariantModels } from '@kontent-ai/management-sdk';
+import {
+    AssetStateInTargetEnvironmentByCodename,
+    ItemStateInTargetEnvironmentByCodename,
+    LanguageVariantStateInTargetEnvironmentByCodename,
+    MigrationItem,
+    findRequired,
+    is404Error,
+    isNotUndefined,
+    managementClientUtils,
+    processItemsAsync,
+    runMapiRequestAsync,
+    workflowHelper
+} from '../../core/index.js';
+import {
+    GetFlattenedElementByCodenames,
+    ImportContext,
+    ImportContextConfig,
+    ImportContextEnvironmentData
+} from '../import.models.js';
+import { ExtractItemByCodename, itemsExtractionProcessor } from '../../translation/index.js';
+import chalk from 'chalk';
+
+interface LanguageVariantWrapper {
+    readonly languageVariant: Readonly<LanguageVariantModels.ContentItemLanguageVariant>;
+    readonly migrationItem: MigrationItem;
+}
+
+export async function importContextFetcherAsync(config: ImportContextConfig) {
+    const getEnvironmentDataAsync = async (): Promise<ImportContextEnvironmentData> => {
+        const mapiUtils = managementClientUtils(config.managementClient, config.logger);
+
+        return await config.logger.logWithSpinnerAsync(async (spinnerData) => {
+            spinnerData({ type: 'info', message: `Loading environment data` });
+
+            const environmentData: ImportContextEnvironmentData = {
+                collections: await mapiUtils.getAllCollectionsAsync(spinnerData),
+                languages: await mapiUtils.getAllLanguagesAsync(spinnerData),
+                workflows: await mapiUtils.getAllWorkflowsAsync(spinnerData),
+                types: await mapiUtils.getFlattenedContentTypesAsync(spinnerData)
+            };
+
+            spinnerData({ type: 'info', message: `Environmental data loaded` });
+
+            return environmentData;
+        });
+    };
+
+    const environmentData = await getEnvironmentDataAsync();
+
+    const getElement = () => {
+        const getFlattenedElement: GetFlattenedElementByCodenames = (
+            contentTypeCodename,
+            elementCodename,
+            sourceType
+        ) => {
+            const contentType = findRequired(
+                environmentData.types,
+                (type) => type.contentTypeCodename === contentTypeCodename,
+                `Content type with codename '${chalk.red(contentTypeCodename)}' was not found.`
+            );
+
+            const element = findRequired(
+                contentType.elements,
+                (element) => element.codename === elementCodename,
+                `Element type with codename '${chalk.red(elementCodename)}' was not found in content type '${chalk.red(
+                    contentTypeCodename
+                )}'. Available elements are '${contentType.elements
+                    .map((element) => chalk.yellow(element.codename))
+                    .join(', ')}'`
+            );
+
+            if (sourceType !== element.type) {
+                throw Error(
+                    `Element '${chalk.red(element.codename)}' in content type '${chalk.yellow(
+                        contentType.contentTypeCodename
+                    )}' is of type '${chalk.red(element.type)}', but source type is '${chalk.yellow(sourceType)}'.`
+                );
+            }
+
+            return element;
+        };
+
+        return getFlattenedElement;
+    };
+
+    const getLanguageVariantsAsync = async (
+        migrationItems: readonly MigrationItem[]
+    ): Promise<readonly LanguageVariantWrapper[]> => {
+        return (
+            await processItemsAsync<MigrationItem, LanguageVariantWrapper | undefined>({
+                action: 'Fetching language variants',
+                logger: config.logger,
+                parallelLimit: 1,
+                items: migrationItems,
+                itemInfo: (item) => {
+                    return {
+                        itemType: 'languageVariant',
+                        title: `${item.system.codename} (${item.system.language.codename})`
+                    };
+                },
+                processAsync: async (item, logSpinner) => {
+                    try {
+                        const languageVariant = await runMapiRequestAsync({
+                            logger: config.logger,
+                            func: async () =>
+                                (
+                                    await config.managementClient
+                                        .viewLanguageVariant()
+                                        .byItemCodename(item.system.codename)
+                                        .byLanguageCodename(item.system.language.codename)
+                                        .toPromise()
+                                ).data,
+                            action: 'view',
+                            type: 'languageVariant',
+                            logSpinner: logSpinner,
+                            itemName: `codename -> ${item.system.codename} (${item.system.language.codename})`
+                        });
+
+                        return {
+                            languageVariant: languageVariant,
+                            migrationItem: item
+                        };
+                    } catch (error) {
+                        if (!is404Error(error)) {
+                            throw error;
+                        }
+
+                        return undefined;
+                    }
+                }
+            })
+        ).filter(isNotUndefined);
+    };
+
+    const getContentItemsByCodenamesAsync = async (
+        itemCodenames: ReadonlySet<string>
+    ): Promise<readonly ContentItemModels.ContentItem[]> => {
+        return (
+            await processItemsAsync<string, Readonly<ContentItemModels.ContentItem> | undefined>({
+                action: 'Fetching content items',
+                logger: config.logger,
+                parallelLimit: 1,
+                items: Array.from(itemCodenames),
+                itemInfo: (codename) => {
+                    return {
+                        itemType: 'contentItem',
+                        title: codename
+                    };
+                },
+                processAsync: async (codename, logSpinner) => {
+                    try {
+                        return await runMapiRequestAsync({
+                            logger: config.logger,
+                            func: async () =>
+                                (
+                                    await config.managementClient.viewContentItem().byItemCodename(codename).toPromise()
+                                ).data,
+                            action: 'view',
+                            type: 'contentItem',
+                            logSpinner: logSpinner,
+                            itemName: `codename -> ${codename}`
+                        });
+                    } catch (error) {
+                        if (!is404Error(error)) {
+                            throw error;
+                        }
+
+                        return undefined;
+                    }
+                }
+            })
+        ).filter(isNotUndefined);
+    };
+
+    const getAssetsByCodenamesAsync = async (
+        assetCodenames: ReadonlySet<string>
+    ): Promise<readonly AssetModels.Asset[]> => {
+        return (
+            await processItemsAsync<string, Readonly<AssetModels.Asset> | undefined>({
+                action: 'Fetching assets',
+                logger: config.logger,
+                parallelLimit: 1,
+                items: Array.from(assetCodenames),
+                itemInfo: (codename) => {
+                    return {
+                        itemType: 'asset',
+                        title: codename
+                    };
+                },
+                processAsync: async (codename, logSpinner) => {
+                    try {
+                        return await runMapiRequestAsync({
+                            logger: config.logger,
+                            func: async () =>
+                                (
+                                    await config.managementClient.viewAsset().byAssetCodename(codename).toPromise()
+                                ).data,
+                            action: 'view',
+                            type: 'asset',
+                            logSpinner: logSpinner,
+                            itemName: `codename -> ${codename}`
+                        });
+                    } catch (error) {
+                        if (!is404Error(error)) {
+                            throw error;
+                        }
+
+                        return undefined;
+                    }
+                }
+            })
+        ).filter(isNotUndefined);
+    };
+
+    const getVariantStatesAsync = async (
+        migrationItems: readonly MigrationItem[]
+    ): Promise<readonly LanguageVariantStateInTargetEnvironmentByCodename[]> => {
+        const variants = await getLanguageVariantsAsync(migrationItems);
+
+        return migrationItems.map<LanguageVariantStateInTargetEnvironmentByCodename>((migrationItem) => {
+            const variant = variants.find(
+                (m) =>
+                    m.migrationItem.system.codename === migrationItem.system.codename &&
+                    m.migrationItem.system.language === migrationItem.system.language
+            );
+
+            const variantWorkflowId = variant?.languageVariant?.workflow?.workflowIdentifier?.id;
+            const variantStepId = variant?.languageVariant?.workflow?.stepIdentifier?.id;
+
+            const workflow = environmentData.workflows.find((workflow) => workflow.id === variantWorkflowId);
+
+            return {
+                itemCodename: migrationItem.system.codename,
+                languageCodename: migrationItem.system.language.codename,
+                languageVariant: variant?.languageVariant,
+                workflow: workflow,
+                step: workflow
+                    ? workflowHelper(environmentData.workflows).getWorkflowStep(workflow, {
+                          errorMessage: `Could not workflow step with id '${chalk.red(
+                              variantStepId
+                          )}' in workflow '${chalk.yellow(workflow.codename)}'`,
+                          match: (step) => step.id == variantStepId
+                      })
+                    : undefined,
+                state: variant ? 'exists' : 'doesNotExists'
+            };
+        });
+    };
+
+    const getItemStatesAsync = async (
+        itemCodenames: ReadonlySet<string>
+    ): Promise<readonly ItemStateInTargetEnvironmentByCodename[]> => {
+        const items = await getContentItemsByCodenamesAsync(itemCodenames);
+
+        return Array.from(itemCodenames).map<ItemStateInTargetEnvironmentByCodename>((codename) => {
+            const item = items.find((m) => m.codename === codename);
+            return {
+                itemCodename: codename,
+                item: item,
+                state: item ? 'exists' : 'doesNotExists',
+                externalIdToUse: config.externalIdGenerator.contentItemExternalId(codename)
+            };
+        });
+    };
+
+    const getAssetStatesAsync = async (
+        assetCodenames: ReadonlySet<string>
+    ): Promise<readonly AssetStateInTargetEnvironmentByCodename[]> => {
+        const assets = await getAssetsByCodenamesAsync(assetCodenames);
+
+        return Array.from(assetCodenames).map<AssetStateInTargetEnvironmentByCodename>((codename) => {
+            const asset = assets.find((m) => m.codename === codename);
+            return {
+                assetCodename: codename,
+                asset: asset,
+                state: asset ? 'exists' : 'doesNotExists',
+                externalIdToUse: config.externalIdGenerator.assetExternalId(codename)
+            };
+        });
+    };
+
+    const getImportContextAsync = async (): Promise<ImportContext> => {
+        const getElementByCodenames: GetFlattenedElementByCodenames = getElement();
+
+        const referencedData = itemsExtractionProcessor().extractReferencedItemsFromMigrationItems(
+            config.migrationData.items.reduce<ExtractItemByCodename[]>((items, item) => {
+                return [
+                    ...items,
+                    ...item.versions.map((version) => {
+                        const extractionItem: ExtractItemByCodename = {
+                            contentTypeCodename: item.system.type.codename,
+                            elements: version.elements
+                        };
+                        return extractionItem;
+                    })
+                ];
+            }, []),
+            getElementByCodenames
+        );
+        // fetch all items, including referenced items in content
+        const itemCodenamesToCheckInTargetEnv: ReadonlySet<string> = new Set<string>([
+            ...referencedData.itemCodenames,
+            ...config.migrationData.items.map((m) => m.system.codename)
+        ]);
+
+        // fetch all assets, including referenced assets in content
+        const assetCodenamesToCheckInTargetEnv: ReadonlySet<string> = new Set<string>([
+            ...referencedData.assetCodenames,
+            ...config.migrationData.assets.map((m) => m.codename)
+        ]);
+
+        // prepare state of objects in target environment
+        const itemStates: readonly ItemStateInTargetEnvironmentByCodename[] = await getItemStatesAsync(
+            itemCodenamesToCheckInTargetEnv
+        );
+
+        const variantStates: readonly LanguageVariantStateInTargetEnvironmentByCodename[] = await getVariantStatesAsync(
+            config.migrationData.items
+        );
+
+        const assetStates: readonly AssetStateInTargetEnvironmentByCodename[] = await getAssetStatesAsync(
+            assetCodenamesToCheckInTargetEnv
+        );
+
+        return {
+            environmentData,
+            referencedData,
+            categorizedImportData: {
+                assets: config.migrationData.assets,
+                contentItems: config.migrationData.items
+            },
+            getItemStateInTargetEnvironment: (itemCodename) => {
+                return findRequired(
+                    itemStates,
+                    (state) => state.itemCodename === itemCodename,
+                    `Invalid state for item '${chalk.red(
+                        itemCodename
+                    )}'. It is expected that all item states will be initialized`
+                );
+            },
+            getLanguageVariantStateInTargetEnvironment: (itemCodename, languageCodename) => {
+                return findRequired(
+                    variantStates,
+                    (state) => state.itemCodename === itemCodename && state.languageCodename === languageCodename,
+                    `Invalid state for language variant '${chalk.red(itemCodename)}' in language '${chalk.red(
+                        languageCodename
+                    )}'. It is expected that all variant states will be initialized`
+                );
+            },
+            getAssetStateInTargetEnvironment: (assetCodename) => {
+                return findRequired(
+                    assetStates,
+                    (state) => state.assetCodename === assetCodename,
+                    `Invalid state for asset '${chalk.red(
+                        assetCodename
+                    )}'. It is expected that all asset states will be initialized`
+                );
+            },
+            getElement: getElementByCodenames
+        };
+    };
+
+    return {
+        getImportContextAsync
+    };
+}

package/lib/import/import-manager.ts

@@ -0,0 +1,103 @@
+import { ContentItemModels, LanguageVariantModels, ManagementClient } from '@kontent-ai/management-sdk';
+
+import { defaultExternalIdGenerator, Logger, getDefaultLogger, getMigrationManagementClient } from '../core/index.js';
+import { ImportConfig, ImportContext, ImportResult } from './import.models.js';
+import { assetsImporter } from './importers/assets-importer.js';
+import { contentItemsImporter } from './importers/content-items-importer.js';
+import { languageVariantImporter } from './importers/language-variant-importer.js';
+import { importContextFetcherAsync } from './context/import-context-fetcher.js';
+
+export function importManager(config: ImportConfig) {
+    const logger: Logger = config.logger ?? getDefaultLogger();
+    const targetEnvironmentClient: ManagementClient = getMigrationManagementClient(config);
+
+    const importAssetsAsync = async (
+        importContext: ImportContext
+    ): Promise<Pick<ImportResult, 'editedAssets' | 'uploadedAssets'>> => {
+        if (!importContext.categorizedImportData.assets.length) {
+            logger.log({
+                type: 'info',
+                message: `There are no assets to import`
+            });
+            return {
+                editedAssets: [],
+                uploadedAssets: []
+            };
+        }
+
+        return await assetsImporter({
+            client: targetEnvironmentClient,
+            importContext: importContext,
+            logger: logger
+        }).importAsync();
+    };
+    const importContentItemsAsync = async (
+        importContext: ImportContext
+    ): Promise<readonly Readonly<ContentItemModels.ContentItem>[]> => {
+        if (!importContext.categorizedImportData.contentItems.length) {
+            logger.log({
+                type: 'info',
+                message: `There are no content items to import`
+            });
+        }
+
+        return await contentItemsImporter({
+            client: targetEnvironmentClient,
+            importContext: importContext,
+            logger: logger
+        }).importAsync();
+    };
+
+    const importLanguageVariantsAsync = async (
+        importContext: ImportContext,
+        contentItems: readonly Readonly<ContentItemModels.ContentItem>[]
+    ): Promise<readonly Readonly<LanguageVariantModels.ContentItemLanguageVariant>[]> => {
+        if (!importContext.categorizedImportData.contentItems.length) {
+            logger.log({
+                type: 'info',
+                message: `There are no language variants to import`
+            });
+        }
+
+        return await languageVariantImporter({
+            client: targetEnvironmentClient,
+            importContext: importContext,
+            logger: logger,
+            preparedContentItems: contentItems
+        }).importAsync();
+    };
+
+    return {
+        async importAsync(): Promise<ImportResult> {
+            const importContext = await (
+                await importContextFetcherAsync({
+                    migrationData: config.data,
+                    externalIdGenerator: config.externalIdGenerator ?? defaultExternalIdGenerator,
+                    logger: logger,
+                    managementClient: targetEnvironmentClient
+                })
+            ).getImportContextAsync();
+
+            // #1 Assets
+            const { editedAssets, uploadedAssets } = await importAssetsAsync(importContext);
+
+            // #2 Content items
+            const contentItems = await importContentItemsAsync(importContext);
+
+            // #3 Language variants
+            const languageVariants = await importLanguageVariantsAsync(importContext, contentItems);
+
+            logger.log({
+                type: 'completed',
+                message: `Finished import`
+            });
+
+            return {
+                editedAssets,
+                uploadedAssets,
+                contentItems,
+                languageVariants
+            };
+        }
+    };
+}

package/lib/import/import.models.ts

@@ -0,0 +1,84 @@
+import {
+    MigrationItem,
+    MigrationAsset,
+    Logger,
+    AssetStateInTargetEnvironmentByCodename,
+    ItemStateInTargetEnvironmentByCodename,
+    ReferencedDataInMigrationItems,
+    LanguageVariantStateInTargetEnvironmentByCodename,
+    ExternalIdGenerator,
+    FlattenedContentTypeElement,
+    MigrationElementValue,
+    MigrationElementType,
+    MigrationData,
+    ManagementClientConfig,
+    FlattenedContentType
+} from '../core/index.js';
+import {
+    AssetModels,
+    CollectionModels,
+    ContentItemModels,
+    ElementContracts,
+    LanguageModels,
+    LanguageVariantModels,
+    ManagementClient,
+    WorkflowModels
+} from '@kontent-ai/management-sdk';
+
+export interface ImportContextConfig {
+    readonly logger: Logger;
+    readonly managementClient: Readonly<ManagementClient>;
+    readonly externalIdGenerator: ExternalIdGenerator;
+    readonly migrationData: MigrationData;
+}
+
+export interface ImportContextEnvironmentData {
+    readonly languages: readonly Readonly<LanguageModels.LanguageModel>[];
+    readonly collections: readonly Readonly<CollectionModels.Collection>[];
+    readonly workflows: readonly Readonly<WorkflowModels.Workflow>[];
+    readonly types: readonly Readonly<FlattenedContentType>[];
+}
+
+export type GetFlattenedElementByCodenames = (
+    contentTypeCodename: string,
+    elementCodename: string,
+    expectedElementType: MigrationElementType
+) => FlattenedContentTypeElement;
+
+export interface CategorizedImportData {
+    readonly assets: readonly MigrationAsset[];
+    readonly contentItems: readonly MigrationItem[];
+}
+
+export interface ImportContext {
+    readonly categorizedImportData: CategorizedImportData;
+    readonly referencedData: ReferencedDataInMigrationItems;
+    readonly environmentData: ImportContextEnvironmentData;
+    readonly getItemStateInTargetEnvironment: (itemCodename: string) => ItemStateInTargetEnvironmentByCodename;
+    readonly getLanguageVariantStateInTargetEnvironment: (
+        itemCodename: string,
+        languageCodename: string
+    ) => LanguageVariantStateInTargetEnvironmentByCodename;
+    readonly getAssetStateInTargetEnvironment: (assetCodename: string) => AssetStateInTargetEnvironmentByCodename;
+    readonly getElement: GetFlattenedElementByCodenames;
+}
+
+export type ImportTransformFunc = (data: {
+    readonly value: MigrationElementValue;
+    readonly elementCodename: string;
+    readonly importContext: ImportContext;
+    readonly migrationItems: readonly MigrationItem[];
+}) => ElementContracts.IContentItemElementContract;
+
+export interface ImportConfig extends ManagementClientConfig {
+    readonly data: MigrationData;
+    readonly externalIdGenerator?: ExternalIdGenerator;
+    readonly logger?: Logger;
+}
+
+export interface ImportResult {
+    readonly uploadedAssets: readonly Readonly<AssetModels.Asset>[];
+    readonly editedAssets: readonly Readonly<AssetModels.Asset>[];
+    readonly contentItems: readonly Readonly<ContentItemModels.ContentItem>[];
+    readonly languageVariants: readonly Readonly<LanguageVariantModels.ContentItemLanguageVariant>[];
+}