@kontent-ai/migration-toolkit 1.0.0

package/lib/translation/transforms/export-transforms.ts

@@ -0,0 +1,253 @@
+import {
+    MigrationElementType,
+    MigrationReference,
+    MigrationRichTextElementValue,
+    MigrationUrlSlugElementValue,
+    findRequired,
+    isNotUndefined
+} from '../../core/index.js';
+import { ContentTypeElements, TaxonomyModels } from '@kontent-ai/management-sdk';
+import { ExportTransformFunc, ExportContext, ExportElement } from '../../export/index.js';
+import { richTextProcessor } from '../helpers/rich-text.processor.js';
+import chalk from 'chalk';
+
+export const exportTransforms: Readonly<Record<MigrationElementType, ExportTransformFunc>> = {
+    text: (data) => data.exportElement.value?.toString(),
+    number: (data) => {
+        if (!data.exportElement.value) {
+            return undefined;
+        }
+
+        if (Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be a number, not array`);
+        }
+
+        if (data.exportElement.value === 0) {
+            return 0;
+        }
+
+        return +data.exportElement.value;
+    },
+    date_time: (data) => data.exportElement.value?.toString(),
+    rich_text: (data) => transformRichTextValue(data.exportElement, data.context),
+    asset: (data) => {
+        if (!data.exportElement.value) {
+            return [];
+        }
+
+        if (!Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be an array`);
+        }
+
+        // translate asset id to codename
+        return data.exportElement.value
+            .map((m) => m.id)
+            .filter(isNotUndefined)
+            .map<MigrationReference>((id) => {
+                const assetState = data.context.getAssetStateInSourceEnvironment(id);
+
+                if (assetState.asset) {
+                    // reference asset by codename
+                    return { codename: assetState.asset.codename };
+                } else {
+                    throw Error(`Missing asset with id '${chalk.red(id)}'`);
+                }
+            });
+    },
+    taxonomy: (data) => {
+        if (!data.exportElement.value) {
+            return [];
+        }
+
+        if (!Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be an array`);
+        }
+
+        const taxonomyElement = data.typeElement.element as Readonly<ContentTypeElements.ITaxonomyElement>;
+        const taxonomyGroupId = taxonomyElement.taxonomy_group.id ?? 'n/a';
+
+        // get taxonomy group
+        const taxonomy = findRequired(
+            data.context.environmentData.taxonomies,
+            (taxonomy) => taxonomy.id === taxonomyGroupId,
+            `Could not find taxonomy group with id '${chalk.red(taxonomyGroupId)}'`
+        );
+
+        // translate taxonomy term to codename
+        return data.exportElement.value
+            .map((m) => m.id)
+            .filter(isNotUndefined)
+            .map<MigrationReference>((id) => {
+                const taxonomyTerm = findTaxonomy(id, taxonomy);
+
+                if (taxonomyTerm) {
+                    // reference taxonomy term by codename
+                    return { codename: taxonomyTerm.codename };
+                } else {
+                    throw Error(`Missing taxonomy term with id '${chalk.red(id)}'`);
+                }
+            });
+    },
+    modular_content: (data) => {
+        if (!data.exportElement.value) {
+            return [];
+        }
+
+        if (!Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be an array`);
+        }
+
+        // translate item id to codename
+        return data.exportElement.value
+            .map((m) => m.id)
+            .filter(isNotUndefined)
+            .map<MigrationReference>((id) => {
+                const itemState = data.context.getItemStateInSourceEnvironment(id);
+
+                if (itemState.item) {
+                    // reference item by codename
+                    return { codename: itemState.item.codename };
+                } else {
+                    throw Error(`Missing item with id '${chalk.red(id)}'`);
+                }
+            });
+    },
+    custom: (data) => data.exportElement.value?.toString(),
+    url_slug: (data) => {
+        const urlSlugElementValue: MigrationUrlSlugElementValue = {
+            mode: data.exportElement.urlSlugMode ?? 'autogenerated',
+            value: data.exportElement.value?.toString() ?? undefined
+        };
+
+        return urlSlugElementValue;
+    },
+    multiple_choice: (data) => {
+        if (!data.exportElement.value) {
+            return [];
+        }
+
+        if (!Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be an array`);
+        }
+
+        // translate multiple choice option id to codename
+        const multipleChoiceElement = data.typeElement.element as Readonly<ContentTypeElements.IMultipleChoiceElement>;
+
+        return data.exportElement.value
+            .map((m) => m.id)
+            .filter(isNotUndefined)
+            .map<MigrationReference>((id) => {
+                const option = findRequired(
+                    multipleChoiceElement.options,
+                    (option) => option.id === id,
+                    `Could not find multiple choice element with option id '${chalk.red(id)}'`
+                );
+
+                if (!option.codename) {
+                    throw Error(`Invalid codename for multiple choice option '${chalk.red(option.name)}'`);
+                }
+
+                return { codename: option.codename };
+            });
+    },
+    subpages: (data) => {
+        if (!data.exportElement.value) {
+            return [];
+        }
+        if (!Array.isArray(data.exportElement.value)) {
+            throw Error(`Expected value to be an array`);
+        }
+
+        // translate item id to codename
+        return data.exportElement.value
+            .map((m) => m.id)
+            .filter(isNotUndefined)
+            .map<MigrationReference>((id) => {
+                const itemState = data.context.getItemStateInSourceEnvironment(id);
+
+                if (itemState.item) {
+                    // reference item by codename
+                    return { codename: itemState.item.codename };
+                } else {
+                    throw Error(`Missing item with id '${chalk.red(id)}'`);
+                }
+            });
+    }
+};
+
+function findTaxonomy(
+    termId: string,
+    taxonomy: Readonly<TaxonomyModels.Taxonomy>
+): Readonly<TaxonomyModels.Taxonomy> | undefined {
+    if (taxonomy.id === termId) {
+        return taxonomy;
+    }
+
+    if (taxonomy.terms) {
+        const foundTerm = taxonomy.terms.find((term) => findTaxonomy(term.id, term));
+
+        if (foundTerm) {
+            return foundTerm;
+        }
+    }
+
+    return undefined;
+}
+
+function transformRichTextValue(
+    exportElement: ExportElement | undefined,
+    context: ExportContext
+): MigrationRichTextElementValue | undefined {
+    if (!exportElement || !exportElement.value) {
+        return {
+            components: [],
+            value: ''
+        };
+    }
+
+    let richTextHtml: string = exportElement.value.toString();
+
+    // replace item ids with codenames
+    richTextHtml = richTextProcessor().processDataIds(richTextHtml, (id) => {
+        const itemInEnv = context.getItemStateInSourceEnvironment(id).item;
+
+        if (!itemInEnv) {
+            throw Error(`Failed to get item with id '${chalk.red(id)}'`);
+        }
+
+        return {
+            codename: itemInEnv.codename
+        };
+    }).html;
+
+    // replace link item ids with codenames
+    richTextHtml = richTextProcessor().processLinkItemIds(richTextHtml, (id) => {
+        const itemInEnv = context.getItemStateInSourceEnvironment(id).item;
+
+        if (!itemInEnv) {
+            throw Error(`Failed to get item with id '${chalk.red(id)}'`);
+        }
+
+        return {
+            codename: itemInEnv.codename
+        };
+    }).html;
+
+    // replace asset ids with codenames
+    richTextHtml = richTextProcessor().processAssetIds(richTextHtml, (id) => {
+        const assetInEnv = context.getAssetStateInSourceEnvironment(id).asset;
+
+        if (!assetInEnv) {
+            throw Error(`Failed to get asset with id '${chalk.red(id)}'`);
+        }
+
+        return {
+            codename: assetInEnv.codename
+        };
+    }).html;
+
+    return {
+        components: exportElement.components,
+        value: richTextHtml
+    };
+}

package/lib/translation/transforms/import-transforms.ts

@@ -0,0 +1,215 @@
+import { SharedContracts, LanguageVariantElementsBuilder, LanguageVariantElements } from '@kontent-ai/management-sdk';
+import {
+    parseAsMigrationReferencesArray,
+    MigrationElementType,
+    MigrationRichTextElementValue,
+    MigrationItem,
+    MigrationUrlSlugElementValue
+} from '../../core/index.js';
+import { ImportContext, ImportTransformFunc } from '../../import/index.js';
+import { richTextProcessor } from '../helpers/rich-text.processor.js';
+
+const elementsBuilder = new LanguageVariantElementsBuilder();
+
+/**
+ * The purpose of import transform is to take the exported value and convert it to a value that
+ * Kontent.ai Management API understands
+ */
+export const importTransforms: Readonly<Record<MigrationElementType, ImportTransformFunc>> = {
+    subpages: (data) => {
+        return elementsBuilder.linkedItemsElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: parseAsMigrationReferencesArray(data.value).map((m) => {
+                return {
+                    codename: m.codename
+                };
+            })
+        });
+    },
+    asset: (data) => {
+        const assetReferences = parseAsMigrationReferencesArray(data.value)
+            .map((reference) => reference.codename)
+            .map<Readonly<SharedContracts.IReferenceObjectContract>>((codename) => {
+                const assetState = data.importContext.getAssetStateInTargetEnvironment(codename);
+
+                // only reference with external_id if item does not exist in target env
+                // API currently only supports referencing assets by ids only, not by codenames
+                return {
+                    id: assetState.asset ? assetState.asset.id : undefined,
+                    external_id: assetState.asset ? undefined : assetState.externalIdToUse
+                };
+            });
+
+        return elementsBuilder.assetElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: assetReferences
+        });
+    },
+    custom: (data) => {
+        return elementsBuilder.customElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: data.value?.toString() ?? ''
+        });
+    },
+    date_time: (data) => {
+        return elementsBuilder.dateTimeElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: data.value?.toString() ?? null
+        });
+    },
+    modular_content: (data) => {
+        const linkedItemReferences = parseAsMigrationReferencesArray(data.value)
+            .map((reference) => reference.codename)
+            .map<Readonly<SharedContracts.IReferenceObjectContract>>((codename) => {
+                const itemState = data.importContext.getItemStateInTargetEnvironment(codename);
+
+                // only reference with external_id if item does not exist in target env
+                return {
+                    codename: itemState.item ? itemState.item.codename : undefined,
+                    external_id: itemState.item ? undefined : itemState.externalIdToUse
+                };
+            });
+
+        return elementsBuilder.linkedItemsElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: linkedItemReferences
+        });
+    },
+    multiple_choice: (data) => {
+        return elementsBuilder.multipleChoiceElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: parseAsMigrationReferencesArray(data.value).map((m) => {
+                return {
+                    codename: m.codename
+                };
+            })
+        });
+    },
+    number: (data) => {
+        return elementsBuilder.numberElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: data.value ? +data.value : null
+        });
+    },
+    rich_text: (data) => {
+        const rteElementValue = data.value as MigrationRichTextElementValue;
+
+        return elementsBuilder.richTextElement({
+            element: {
+                codename: data.elementCodename
+            },
+            components: mapComponents({
+                importContext: data.importContext,
+                migrationItems: data.migrationItems,
+                rteValue: rteElementValue
+            }),
+            value:
+                processImportRichTextHtmlValue({
+                    element: rteElementValue,
+                    importContext: data.importContext,
+                    migrationItems: data.migrationItems
+                }) ?? null
+        });
+    },
+    taxonomy: (data) => {
+        return elementsBuilder.taxonomyElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: parseAsMigrationReferencesArray(data.value).map((m) => {
+                return {
+                    codename: m.codename
+                };
+            })
+        });
+    },
+    text: (data) => {
+        return elementsBuilder.textElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: data.value?.toString() ?? null
+        });
+    },
+    url_slug: (data) => {
+        const urlSlugElementValue = data.value as Readonly<MigrationUrlSlugElementValue>;
+
+        return elementsBuilder.urlSlugElement({
+            element: {
+                codename: data.elementCodename
+            },
+            value: urlSlugElementValue.value?.toString() ?? '',
+            mode: urlSlugElementValue.mode
+        });
+    }
+};
+
+function mapComponents(data: {
+    readonly rteValue: MigrationRichTextElementValue;
+    readonly importContext: ImportContext;
+    readonly migrationItems: readonly MigrationItem[];
+}): LanguageVariantElements.IRichTextComponent[] {
+    return data.rteValue.components.map((component) => {
+        const mappedComponent: Readonly<LanguageVariantElements.IRichTextComponent> = {
+            id: component.system.id,
+            type: {
+                codename: component.system.type.codename
+            },
+            elements: Object.entries(component.elements).map(([key, element]) => {
+                const transformedElementValue = importTransforms[element.type]({
+                    elementCodename: key,
+                    importContext: data.importContext,
+                    migrationItems: data.migrationItems,
+                    value: element.value
+                });
+
+                return transformedElementValue;
+            })
+        };
+
+        return mappedComponent;
+    });
+}
+
+function processImportRichTextHtmlValue(data: {
+    readonly element: MigrationRichTextElementValue;
+    readonly importContext: ImportContext;
+    readonly migrationItems: readonly MigrationItem[];
+}): string | undefined {
+    if (!data.element) {
+        return undefined;
+    }
+
+    let richTextHtml: string = data.element.value ?? '';
+
+    // replace item codenames with existing codename or external_id
+    richTextHtml = richTextProcessor().processItemCodenames(richTextHtml, (codename) => {
+        return data.importContext.getItemStateInTargetEnvironment(codename);
+    }).html;
+
+    // replace link item codenames with existing codename or external_id
+    richTextHtml = richTextProcessor().processLinkItemCodenames(richTextHtml, (codename) => {
+        return data.importContext.getItemStateInTargetEnvironment(codename);
+    }).html;
+
+    // replace asset codenames with existing codename or external_id
+    richTextHtml = richTextProcessor().processAssetCodenames(richTextHtml, (codename) => {
+        return data.importContext.getAssetStateInTargetEnvironment(codename);
+    }).html;
+
+    return richTextHtml;
+}

package/lib/zip/index.ts

@@ -0,0 +1,4 @@
+export * from './zip.models.js';
+export * from './zip-manager.js';
+export * from './zip-packager.js';
+export * from './zip-transformer.js';

package/lib/zip/zip-manager.ts

@@ -0,0 +1,33 @@
+import JSZip from 'jszip';
+import { Logger, MigrationData, getDefaultLogger } from '../core/index.js';
+import { zipTransformer } from './zip-transformer.js';
+import { zipPackager } from './zip-packager.js';
+import { FileBinaryData } from './zip.models.js';
+
+export function zipManager(logger?: Logger) {
+    const loggerToUse = logger ?? getDefaultLogger();
+
+    const createZipAsync = async (migrationData: MigrationData): Promise<FileBinaryData> => {
+        loggerToUse.log({
+            type: 'info',
+            message: `Creating zip package`
+        });
+
+        return await zipTransformer(zipPackager(new JSZip()), loggerToUse).transformAsync(migrationData);
+    };
+
+    const parseZipAsync = async (zipFile: Buffer): Promise<MigrationData> => {
+        loggerToUse.log({
+            type: 'info',
+            message: `Parsing zip file`
+        });
+
+        const zipPackage = zipPackager(await JSZip.loadAsync(zipFile, {}));
+        return await zipTransformer(zipPackage, loggerToUse).parseAsync();
+    };
+
+    return {
+        createZipAsync,
+        parseZipAsync
+    };
+}

package/lib/zip/zip-packager.ts

@@ -0,0 +1,86 @@
+import JSZip from 'jszip';
+import chalk from 'chalk';
+import { Logger, formatBytes, getCurrentEnvironment, exitProgram, getDefaultLogger } from '../core/index.js';
+import { FileBinaryData, ZipCompressionLevel, ZipPackager } from './zip.models.js';
+
+export function zipPackager(jsZip: JSZip): ZipPackager {
+    const getZipOutputType = (): 'nodebuffer' | 'blob' => {
+        const currentEnv = getCurrentEnvironment();
+
+        if (currentEnv === 'browser') {
+            return 'blob';
+        }
+        if (currentEnv === 'node') {
+            return 'nodebuffer';
+        }
+
+        exitProgram({
+            message: `Unsupported context`
+        });
+    };
+
+    const getZipSizeInBytes = (zipData: FileBinaryData): number => {
+        if (zipData instanceof Blob) {
+            return zipData.size;
+        } else if (zipData instanceof Buffer) {
+            return zipData.byteLength;
+        }
+
+        exitProgram({
+            message: `Unrecognized zip data type '${typeof zipData}'`
+        });
+    };
+
+    return {
+        addFile(filePath: string, data: string | FileBinaryData): JSZip {
+            return jsZip.file(filePath, data);
+        },
+        addFolder(name: string): ZipPackager {
+            const folder = jsZip.folder(name);
+
+            if (!folder) {
+                throw Error(`Failed to add folder '${name}'`);
+            }
+
+            return zipPackager(folder);
+        },
+        async generateZipAsync(config: {
+            logger?: Logger;
+            compressionLevel?: ZipCompressionLevel;
+        }): Promise<FileBinaryData> {
+            const logger = config.logger ?? getDefaultLogger();
+            const zipOutputType = getZipOutputType();
+            const compressionLevel: ZipCompressionLevel = config.compressionLevel ?? 9;
+
+            logger.log({
+                type: 'info',
+                message: `Creating zip file using '${chalk.yellow(
+                    zipOutputType
+                )}' with compression level '${chalk.yellow(compressionLevel.toString())}'`
+            });
+
+            const result = await jsZip.generateAsync({
+                type: zipOutputType,
+                compression: 'DEFLATE',
+                compressionOptions: {
+                    level: compressionLevel
+                },
+                streamFiles: true
+            });
+
+            logger.log({
+                type: 'info',
+                message: `Zip successfully generated (${chalk.yellow(formatBytes(getZipSizeInBytes(result)))})`
+            });
+
+            return result;
+        },
+        async getBinaryDataAsync(filePath: string): Promise<FileBinaryData | undefined> {
+            const binaryData = await jsZip.file(filePath)?.async(getZipOutputType());
+            return binaryData;
+        },
+        async getFileContentAsync(filePath: string): Promise<string | undefined> {
+            return await jsZip.file(filePath)?.async('string');
+        }
+    };
+}

package/lib/zip/zip-transformer.ts

@@ -0,0 +1,105 @@
+import { FileBinaryData, ZipPackager } from './zip.models.js';
+import { Logger, MigrationAsset, MigrationData, MigrationItem, getDefaultLogger, mapAsync } from '../core/index.js';
+import chalk from 'chalk';
+
+type ZipAssetRecord = Omit<MigrationAsset, 'binaryData'> & {
+    _zipFilename: string;
+};
+
+export function zipTransformer(zip: ZipPackager, logger?: Logger) {
+    const loggerToUse: Logger = logger ?? getDefaultLogger();
+    const filename: string = 'items.json';
+    const assetsFilename: string = 'assets.json';
+    const assetsBinaryFolderName: string = 'binary_data';
+
+    const getAssetFolderConfig = (asset: MigrationAsset): { partialColder: string; fullPath: string } => {
+        const codenamePartialFolder: string = asset.codename.slice(0, 2);
+        return {
+            partialColder: codenamePartialFolder,
+            fullPath: `${assetsBinaryFolderName}/${codenamePartialFolder}/${asset.codename}/${asset.filename}`
+        };
+    };
+
+    const transformItems = (items: readonly MigrationItem[]): void => {
+        zip.addFile(filename, items.length ? JSON.stringify(items) : '[]');
+    };
+
+    const transformAssets = (assets: readonly MigrationAsset[]): void => {
+        const assetRecords: ZipAssetRecord[] = [];
+        const binaryDataFolder = zip.addFolder(assetsBinaryFolderName);
+
+        assets.forEach((asset) => {
+            const folderConfig = getAssetFolderConfig(asset);
+            const partialFolder = binaryDataFolder.addFolder(folderConfig.partialColder);
+            const codenameFolder = partialFolder.addFolder(asset.codename);
+
+            assetRecords.push({
+                _zipFilename: folderConfig.fullPath,
+                filename: asset.filename,
+                title: asset.title,
+                codename: asset.codename,
+                collection: asset.collection,
+                descriptions: asset.descriptions
+            });
+
+            if (asset.binaryData) {
+                codenameFolder.addFile(asset.filename, asset.binaryData);
+            }
+        });
+
+        zip.addFile(assetsFilename, JSON.stringify(assetRecords));
+    };
+
+    const parseItems = async () => {
+        const fileContent = await zip.getFileContentAsync(filename);
+        return fileContent ? (JSON.parse(fileContent) as MigrationItem[]) : [];
+    };
+
+    const parseAssets = async () => {
+        const text = await zip.getFileContentAsync(assetsFilename);
+
+        if (!text) {
+            return [];
+        }
+
+        const assetRecords = JSON.parse(text) as ZipAssetRecord[];
+
+        return await mapAsync(assetRecords, async (assetRecord) => {
+            const binaryFile = await zip.getBinaryDataAsync(`${assetRecord._zipFilename}`);
+
+            if (!binaryFile) {
+                throw Error(`Could not load binary data for file '${chalk.red(assetRecord._zipFilename)}'`);
+            }
+
+            const migrationAsset: MigrationAsset = {
+                codename: assetRecord.codename,
+                filename: assetRecord.filename,
+                collection: assetRecord.collection,
+                title: assetRecord.title,
+                descriptions: assetRecord.descriptions,
+                binaryData: binaryFile
+            };
+
+            return migrationAsset;
+        });
+    };
+
+    return {
+        async transformAsync(data: MigrationData): Promise<FileBinaryData> {
+            transformItems(data.items);
+            transformAssets(data.assets);
+
+            return await zip.generateZipAsync({ logger: loggerToUse });
+        },
+        async parseAsync(): Promise<MigrationData> {
+            const items = await parseItems();
+            const assets = await parseAssets();
+
+            const migrationData: MigrationData = {
+                assets: assets,
+                items: items
+            };
+            return migrationData;
+        }
+    };
+}