@kontent-ai/migration-toolkit 1.0.0

package/lib/export/export-manager.ts

@@ -0,0 +1,249 @@
+import { ExportContext, ExportConfig, ExportItem } from './export.models.js';
+import chalk from 'chalk';
+import { AssetModels, CollectionModels, ElementModels } from '@kontent-ai/management-sdk';
+import {
+    MigrationAsset,
+    MigrationItem,
+    FlattenedContentTypeElement,
+    extractErrorData,
+    processItemsAsync,
+    getBinaryDataFromUrlAsync,
+    MigrationElements,
+    FlattenedContentType,
+    MigrationComponent,
+    getDefaultLogger,
+    MigrationData,
+    isNotUndefined,
+    MigrationElementValue,
+    getMigrationManagementClient,
+    findRequired
+} from '../core/index.js';
+import { exportTransforms } from '../translation/index.js';
+import { exportContextFetcherAsync } from './context/export-context-fetcher.js';
+
+export function exportManager(config: ExportConfig) {
+    const logger = config.logger ?? getDefaultLogger();
+    const managementClient = getMigrationManagementClient(config);
+
+    const getMigrationItems = (context: ExportContext): readonly MigrationItem[] => {
+        return context.exportItems.map<MigrationItem>((exportItem) => mapToMigrationItem(context, exportItem));
+    };
+
+    const mapToMigrationItem = (context: ExportContext, exportItem: ExportItem): Readonly<MigrationItem> => {
+        const migrationItem: MigrationItem = {
+            system: {
+                name: exportItem.contentItem.name,
+                codename: exportItem.contentItem.codename,
+                language: { codename: exportItem.language.codename },
+                type: { codename: exportItem.contentType.contentTypeCodename },
+                collection: { codename: exportItem.collection.codename },
+                workflow: {
+                    codename: exportItem.workflow.codename
+                }
+            },
+            versions: exportItem.versions.map((version) => {
+                return {
+                    elements: getMigrationElements(context, exportItem.contentType, version.languageVariant.elements),
+                    workflow_step: {
+                        codename: version.workflowStepCodename
+                    }
+                };
+            })
+        };
+
+        return migrationItem;
+    };
+
+    const mapToMigrationComponent = (
+        context: ExportContext,
+        component: Readonly<ElementModels.ContentItemElementComponent>
+    ): MigrationComponent => {
+        const componentType = context.environmentData.contentTypes.find((m) => m.contentTypeId === component.type.id);
+
+        if (!componentType) {
+            throw Error(
+                `Could not find content type with id '${chalk.red(component.type.id)}' for component '${chalk.red(
+                    component.id
+                )}'`
+            );
+        }
+
+        const migrationItem: MigrationComponent = {
+            system: {
+                id: component.id,
+                type: {
+                    codename: componentType.contentTypeCodename
+                }
+            },
+            elements: getMigrationElements(context, componentType, component.elements)
+        };
+
+        return migrationItem;
+    };
+
+    const getMigrationElements = (
+        context: ExportContext,
+        contentType: FlattenedContentType,
+        elements: readonly Readonly<ElementModels.ContentItemElement>[]
+    ): MigrationElements => {
+        return contentType.elements
+            .toSorted((a, b) => {
+                if (a.codename < b.codename) {
+                    return -1;
+                }
+                if (a.codename > b.codename) {
+                    return 1;
+                }
+                return 0;
+            })
+            .reduce<MigrationElements>((model, typeElement) => {
+                const itemElement = findRequired(
+                    elements,
+                    (m) => m.element.id === typeElement.id,
+                    `Could not find element '${chalk.red(typeElement.codename)}'`
+                );
+
+                model[typeElement.codename] = {
+                    type: typeElement.type,
+                    value: getValueToStoreFromElement({
+                        context: context,
+                        contentType: contentType,
+                        exportElement: itemElement,
+                        typeElement: typeElement
+                    })
+                };
+
+                return model;
+            }, {});
+    };
+
+    const getValueToStoreFromElement = (data: {
+        context: ExportContext;
+        contentType: FlattenedContentType;
+        typeElement: FlattenedContentTypeElement;
+        exportElement: ElementModels.ContentItemElement;
+    }): MigrationElementValue => {
+        try {
+            return exportTransforms[data.typeElement.type]({
+                context: data.context,
+                typeElement: data.typeElement,
+                exportElement: {
+                    components: data.exportElement.components.map((component) =>
+                        mapToMigrationComponent(data.context, component)
+                    ),
+                    value: data.exportElement.value,
+                    urlSlugMode: data.exportElement.mode
+                }
+            });
+        } catch (error) {
+            const errorData = extractErrorData(error);
+            let jsonValue = 'n/a';
+
+            try {
+                jsonValue = JSON.stringify(data.exportElement.value);
+            } catch (jsonError) {
+                console.error(`Failed to convert json value`, jsonError);
+            }
+
+            throw new Error(
+                `Failed to map value of element '${chalk.yellow(data.typeElement.codename)}' of type '${chalk.cyan(
+                    data.typeElement.type
+                )}'. Value: ${chalk.bgMagenta(jsonValue)}. Message: ${errorData.message}`
+            );
+        }
+    };
+
+    const exportAssetsAsync = async (context: ExportContext): Promise<readonly Readonly<MigrationAsset>[]> => {
+        const assets = Array.from(context.referencedData.assetIds)
+            .map<Readonly<AssetModels.Asset> | undefined>(
+                (assetId) => context.getAssetStateInSourceEnvironment(assetId).asset
+            )
+            .filter(isNotUndefined);
+
+        return await getMigrationAssetsWithBinaryDataAsync(assets, context);
+    };
+
+    const getMigrationAssetsWithBinaryDataAsync = async (
+        assets: readonly Readonly<AssetModels.Asset>[],
+        context: ExportContext
+    ): Promise<readonly MigrationAsset[]> => {
+        logger.log({
+            type: 'info',
+            message: `Preparing to download '${chalk.yellow(assets.length.toString())}' assets`
+        });
+
+        return await processItemsAsync<Readonly<AssetModels.Asset>, MigrationAsset>({
+            action: 'Downloading assets',
+            logger: logger,
+            parallelLimit: 5,
+            itemInfo: (input) => {
+                return {
+                    title: input.codename,
+                    itemType: 'asset'
+                };
+            },
+            items: assets,
+            processAsync: async (asset, logSpinner) => {
+                const assetCollection: Readonly<CollectionModels.Collection> | undefined =
+                    context.environmentData.collections.find((m) => m.id === asset.collection?.reference?.id);
+
+                logSpinner({
+                    type: 'download',
+                    message: `${asset.url}`
+                });
+
+                const binaryData = await getBinaryDataFromUrlAsync(asset.url);
+
+                const migrationAsset: MigrationAsset = {
+                    filename: asset.fileName,
+                    title: asset.title ?? '',
+                    codename: asset.codename,
+                    binaryData: binaryData.data,
+                    collection: assetCollection ? { codename: assetCollection.codename } : undefined,
+                    descriptions: asset.descriptions.map((description) => {
+                        const language = findRequired(
+                            context.environmentData.languages,
+                            (language) => language.id === description.language.id,
+                            `Could not find language with id '${chalk.red(
+                                description.language.id
+                            )}' requested by asset '${chalk.red(asset.codename)}'`
+                        );
+
+                        return {
+                            description: description.description ?? undefined,
+                            language: {
+                                codename: language.codename
+                            }
+                        };
+                    })
+                };
+
+                return migrationAsset;
+            }
+        });
+    };
+
+    return {
+        async exportAsync(): Promise<MigrationData> {
+            const exportContext = await (
+                await exportContextFetcherAsync({
+                    exportItems: config.exportItems,
+                    logger: logger,
+                    managementClient: managementClient
+                })
+            ).getExportContextAsync();
+
+            const migrationData: MigrationData = {
+                items: getMigrationItems(exportContext),
+                assets: await exportAssetsAsync(exportContext)
+            };
+
+            logger.log({
+                type: 'completed',
+                message: `Finished export`
+            });
+
+            return migrationData;
+        }
+    };
+}

package/lib/export/export.models.ts

@@ -0,0 +1,87 @@
+import {
+    Logger,
+    FlattenedContentType,
+    AssetStateInSourceEnvironmentById,
+    FlattenedContentTypeElement,
+    ItemStateInSourceEnvironmentById,
+    ReferencedDataInLanguageVariants,
+    MigrationElementValue,
+    MigrationComponent,
+    MigrationUrlSlugMode,
+    ManagementClientConfig
+} from '../core/index.js';
+import {
+    LanguageVariantModels,
+    ContentItemModels,
+    WorkflowModels,
+    CollectionModels,
+    LanguageModels,
+    TaxonomyModels,
+    ManagementClient,
+    SharedModels
+} from '@kontent-ai/management-sdk';
+
+export interface ExportContextEnvironmentData {
+    readonly languages: readonly Readonly<LanguageModels.LanguageModel>[];
+    readonly contentTypes: readonly Readonly<FlattenedContentType>[];
+    readonly collections: readonly Readonly<CollectionModels.Collection>[];
+    readonly workflows: readonly Readonly<WorkflowModels.Workflow>[];
+    readonly taxonomies: readonly Readonly<TaxonomyModels.Taxonomy>[];
+}
+
+export type ExportElementValue = string | number | SharedModels.ReferenceObject[] | undefined;
+
+export interface ExportElement {
+    readonly value: ExportElementValue;
+    readonly components: readonly MigrationComponent[];
+    readonly urlSlugMode: MigrationUrlSlugMode | undefined;
+}
+
+export type ExportTransformFunc = (data: {
+    readonly typeElement: FlattenedContentTypeElement;
+    readonly exportElement: ExportElement;
+    readonly context: ExportContext;
+}) => MigrationElementValue;
+
+export interface ExportContext {
+    readonly environmentData: ExportContextEnvironmentData;
+    readonly referencedData: ReferencedDataInLanguageVariants;
+    readonly getItemStateInSourceEnvironment: (id: string) => ItemStateInSourceEnvironmentById;
+    readonly getAssetStateInSourceEnvironment: (id: string) => AssetStateInSourceEnvironmentById;
+    readonly exportItems: readonly ExportItem[];
+    readonly getElement: GetFlattenedElementByIds;
+}
+
+export interface SourceExportItem {
+    readonly itemCodename: string;
+    readonly languageCodename: string;
+}
+
+export interface ExportConfig extends ManagementClientConfig {
+    readonly exportItems: readonly SourceExportItem[];
+    readonly logger?: Logger;
+}
+
+export interface DefaultExportContextConfig {
+    readonly logger: Logger;
+    readonly exportItems: readonly SourceExportItem[];
+    readonly managementClient: Readonly<ManagementClient>;
+}
+
+export type GetFlattenedElementByIds = (contentTypeId: string, elementId: string) => FlattenedContentTypeElement;
+
+export interface ExportItemVersion {
+    readonly workflowStepCodename: string;
+    readonly languageVariant: Readonly<LanguageVariantModels.ContentItemLanguageVariant>;
+}
+
+export interface ExportItem {
+    readonly requestItem: SourceExportItem;
+    readonly versions: readonly ExportItemVersion[];
+    readonly contentItem: Readonly<ContentItemModels.ContentItem>;
+
+    readonly collection: Readonly<CollectionModels.Collection>;
+    readonly language: Readonly<LanguageModels.LanguageModel>;
+    readonly workflow: Readonly<WorkflowModels.Workflow>;
+    readonly contentType: Readonly<FlattenedContentType>;
+}

package/lib/export/index.ts

@@ -0,0 +1,2 @@
+export * from './export.models.js';
+export * from './export-manager.js';

package/lib/export/utils/export.utils.ts

@@ -0,0 +1,10 @@
+import chalk from 'chalk';
+import { SourceExportItem } from '../export.models.js';
+
+export function throwErrorForItemRequest(itemRequest: SourceExportItem, message: string): never {
+    throw Error(
+        `Export failed for item '${chalk.yellow(itemRequest.itemCodename)}' in language '${chalk.cyan(
+            itemRequest.languageCodename
+        )}'. Reason: ${message}`
+    );
+}

package/lib/file/file-manager.ts

@@ -0,0 +1,37 @@
+import chalk from 'chalk';
+import { Logger } from '../core/index.js';
+import { promises } from 'fs';
+
+export function fileManager(logger: Logger) {
+    const getFilePath = (filename: string): string => {
+        return `./${filename}`;
+    };
+
+    const loadFileAsync = async (filename: string): Promise<Buffer> => {
+        const filePath = getFilePath(filename);
+
+        logger.log({
+            type: 'readFs',
+            message: `Reading file '${chalk.yellow(filePath)}'`
+        });
+
+        const file = await promises.readFile(filePath);
+
+        return file;
+    };
+
+    const writeFileAsync = async (fileNameWithoutExtension: string, content: string | Buffer): Promise<void> => {
+        const filePath = getFilePath(fileNameWithoutExtension);
+
+        logger.log({
+            type: 'writeFs',
+            message: `Storing file '${chalk.yellow(filePath)}'`
+        });
+        await promises.writeFile(filePath, content);
+    };
+
+    return {
+        loadFileAsync,
+        writeFileAsync
+    };
+}

package/lib/file/index.ts

@@ -0,0 +1 @@
+export * from './file-manager.js';

package/lib/import/comparers/asset-comparer.ts

@@ -0,0 +1,113 @@
+import { AssetModels, CollectionModels, LanguageModels } from '@kontent-ai/management-sdk';
+import { MigrationAsset, MigrationAssetDescription, findRequired, geSizeInBytes } from '../../core/index.js';
+import deepEqual from 'deep-equal';
+
+export function shouldUpdateAsset(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+    readonly collections: readonly Readonly<CollectionModels.Collection>[];
+    readonly languages: readonly Readonly<LanguageModels.LanguageModel>[];
+}): boolean {
+    if (!isInSameCollection(data)) {
+        return true;
+    }
+
+    if (!areDescriptionsIdentical(data)) {
+        return true;
+    }
+
+    if (!isTitleIdentical(data)) {
+        return true;
+    }
+
+    if (!isBinaryFileIdentical(data)) {
+        return true;
+    }
+
+    return false;
+}
+
+export function shouldReplaceBinaryFile(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+}): boolean {
+    if (!isBinaryFileIdentical(data)) {
+        return true;
+    }
+
+    return false;
+}
+
+function isBinaryFileIdentical(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+}): boolean {
+    const sourceFileSize = geSizeInBytes(data.migrationAsset.binaryData);
+    const targetFileSize = data.targetAsset.size;
+
+    const sourceFilename = data.migrationAsset.filename;
+    const targetFilename = data.targetAsset.fileName;
+
+    return sourceFileSize === targetFileSize && sourceFilename === targetFilename;
+}
+
+function isTitleIdentical(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+}): boolean {
+    const sourceTitle = data.migrationAsset.title?.length ? data.migrationAsset.title : undefined;
+    const targetTitle = data.targetAsset.title?.length ? data.targetAsset.title : undefined;
+
+    return sourceTitle === targetTitle;
+}
+
+function isInSameCollection(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+    readonly collections: readonly Readonly<CollectionModels.Collection>[];
+}): boolean {
+    return (
+        data.collections.find((m) => m.id === data.targetAsset.collection?.reference?.id)?.codename ===
+        data.migrationAsset.collection?.codename
+    );
+}
+
+function areDescriptionsIdentical(data: {
+    readonly migrationAsset: MigrationAsset;
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+    readonly languages: readonly Readonly<LanguageModels.LanguageModel>[];
+}): boolean {
+    const sourceMigrationDescriptions = (data.migrationAsset.descriptions ?? [])
+        .map<MigrationAssetDescription>((description) => {
+            return {
+                description: description.description?.length ? description.description : undefined,
+                language: {
+                    codename: description.language.codename
+                }
+            };
+        })
+        .toSorted();
+    const targetMigrationDescriptions = mapToMigrationDescriptions(data).toSorted();
+
+    return deepEqual(sourceMigrationDescriptions, targetMigrationDescriptions);
+}
+
+function mapToMigrationDescriptions(data: {
+    readonly targetAsset: Readonly<AssetModels.Asset>;
+    readonly languages: readonly Readonly<LanguageModels.LanguageModel>[];
+}): MigrationAssetDescription[] {
+    return data.targetAsset.descriptions.map((description) => {
+        const languageId = description.language.id;
+
+        return {
+            description: description.description?.length ? description.description : undefined,
+            language: {
+                codename: findRequired(
+                    data.languages,
+                    (language) => language.id === languageId,
+                    `Could not find language with id '${languageId}'`
+                ).codename
+            }
+        };
+    });
+}