@kontent-ai/migration-toolkit 1.0.0

package/lib/translation/extraction/items-extraction.processor.ts

@@ -0,0 +1,181 @@
+import { richTextProcessor } from '../index.js';
+import {
+    MigrationElementModels,
+    MigrationElements,
+    ReferencedDataInLanguageVariants,
+    ReferencedDataInMigrationItems,
+    parseAsMigrationReferencesArray
+} from '../../core/index.js';
+import { GetFlattenedElementByCodenames } from '../../import/index.js';
+import { ElementModels } from '@kontent-ai/management-sdk';
+import { GetFlattenedElementByIds } from 'lib/export/export.models.js';
+
+export interface ExtractItemById {
+    readonly elements: Readonly<ElementModels.ContentItemElement>[];
+    readonly contentTypeId: string;
+}
+
+export interface ExtractItemByCodename {
+    readonly elements: MigrationElements;
+    readonly contentTypeCodename: string;
+}
+
+export interface ReferencedDataInMigrationItemsLocal {
+    readonly itemCodenames: Set<string>;
+    readonly assetCodenames: Set<string>;
+}
+
+export interface ReferencedDataInLanguageVariantsLocal {
+    readonly itemIds: Set<string>;
+    readonly assetIds: Set<string>;
+}
+
+export function itemsExtractionProcessor() {
+    const extractReferencedDataFromExtractItems = (
+        items: readonly ExtractItemById[],
+        getElement: GetFlattenedElementByIds
+    ): ReferencedDataInLanguageVariants => {
+        const extractedIds = items.reduce<ReferencedDataInLanguageVariantsLocal>(
+            (extractedIds, item) => {
+                return item.elements.reduce<ReferencedDataInLanguageVariantsLocal>((childExtractedIds, itemElement) => {
+                    const typeElement = getElement(item.contentTypeId, itemElement.element.id ?? '');
+
+                    if (typeElement.type === 'rich_text') {
+                        const rteValue = itemElement.value?.toString();
+
+                        // extract referenced items
+                        richTextProcessor()
+                            .processDataIds(rteValue ?? '')
+                            .ids.forEach((id) => extractedIds.itemIds.add(id));
+
+                        richTextProcessor()
+                            .processLinkItemIds(rteValue ?? '')
+                            .ids.forEach((id) => extractedIds.itemIds.add(id));
+
+                        // extract referenced assets
+                        richTextProcessor()
+                            .processAssetIds(rteValue ?? '')
+                            .ids.forEach((id) => extractedIds.assetIds.add(id));
+
+                        // recursively extract data from components as well because they may reference additional assets & content items
+                        const extractedComponents = extractReferencedDataFromExtractItems(
+                            itemElement.components.map((component) => {
+                                return {
+                                    contentTypeId: component.type.id ?? '',
+                                    elements: component.elements
+                                };
+                            }),
+                            getElement
+                        );
+
+                        extractedComponents.itemIds.forEach((id) => extractedIds.itemIds.add(id));
+                        extractedComponents.assetIds.forEach((id) => extractedIds.assetIds.add(id));
+                    } else if (typeElement.type === 'modular_content' || typeElement.type === 'subpages') {
+                        if (itemElement.value && Array.isArray(itemElement.value)) {
+                            itemElement.value.forEach((value) => (value.id ? extractedIds.itemIds.add(value.id) : {}));
+                        }
+                    } else if (typeElement.type === 'asset') {
+                        if (itemElement.value && Array.isArray(itemElement.value)) {
+                            itemElement.value.forEach((value) => (value.id ? extractedIds.assetIds.add(value.id) : {}));
+                        }
+                    }
+
+                    return childExtractedIds;
+                }, extractedIds);
+            },
+            { itemIds: new Set(), assetIds: new Set() }
+        );
+
+        return <ReferencedDataInLanguageVariants>{
+            itemIds: extractedIds.itemIds,
+            assetIds: extractedIds.assetIds
+        };
+    };
+
+    const extractReferencedItemsFromMigrationItems = (
+        items: readonly ExtractItemByCodename[],
+        getElement: GetFlattenedElementByCodenames
+    ): ReferencedDataInMigrationItems => {
+        const extractedCodenames = items.reduce<ReferencedDataInMigrationItemsLocal>(
+            (extractedCodenames, item) => {
+                return Object.entries(item.elements).reduce<ReferencedDataInMigrationItemsLocal>(
+                    (childExtractedCodenames, [elementCodename, element]) => {
+                        const flattenedElement = getElement(item.contentTypeCodename, elementCodename, element.type);
+
+                        if (flattenedElement.type === 'rich_text') {
+                            const richTextElementValue = element as MigrationElementModels.RichTextElement;
+
+                            if (!richTextElementValue.value) {
+                                return childExtractedCodenames;
+                            }
+
+                            const richTextHtml = richTextElementValue.value?.value ?? '';
+
+                            // items
+                            richTextProcessor()
+                                .processItemCodenames(richTextHtml)
+                                .codenames.forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
+
+                            richTextProcessor()
+                                .processLinkItemCodenames(richTextHtml)
+                                .codenames.forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
+
+                            // assets
+                            richTextProcessor()
+                                .processAssetCodenames(richTextHtml)
+                                .codenames.forEach((codename) => childExtractedCodenames.assetCodenames.add(codename));
+
+                            // recursively extract data from components as well because they may reference additional assets & content items
+                            const extractedComponents = extractReferencedItemsFromMigrationItems(
+                                richTextElementValue.value.components.map((component) => {
+                                    const extractionItem: ExtractItemByCodename = {
+                                        contentTypeCodename: component.system.type.codename,
+                                        elements: component.elements
+                                    };
+
+                                    return extractionItem;
+                                }),
+                                getElement
+                            );
+
+                            extractedComponents.itemCodenames.forEach((codename) =>
+                                childExtractedCodenames.itemCodenames.add(codename)
+                            );
+                            extractedComponents.assetCodenames.forEach((codename) =>
+                                childExtractedCodenames.assetCodenames.add(codename)
+                            );
+                        } else if (
+                            flattenedElement.type === 'modular_content' ||
+                            flattenedElement.type === 'subpages'
+                        ) {
+                            parseAsMigrationReferencesArray(element.value)
+                                .map((m) => m.codename)
+                                .forEach((codename) => childExtractedCodenames.itemCodenames.add(codename));
+                        } else if (flattenedElement.type === 'asset') {
+                            parseAsMigrationReferencesArray(element.value)
+                                .map((m) => m.codename)
+                                .forEach((codename) => childExtractedCodenames.assetCodenames.add(codename));
+                        }
+
+                        return childExtractedCodenames;
+                    },
+                    extractedCodenames
+                );
+            },
+            {
+                itemCodenames: new Set(),
+                assetCodenames: new Set()
+            }
+        );
+
+        return <ReferencedDataInMigrationItems>{
+            itemCodenames: extractedCodenames.itemCodenames,
+            assetCodenames: extractedCodenames.assetCodenames
+        };
+    };
+
+    return {
+        extractReferencedDataFromExtractItems,
+        extractReferencedItemsFromMigrationItems
+    };
+}

package/lib/translation/helpers/rich-text.processor.ts

@@ -0,0 +1,300 @@
+import { AssetStateInTargetEnvironmentByCodename, ItemStateInTargetEnvironmentByCodename } from 'lib/core/index.js';
+
+type CodenameReplaceFunc<T> = (codename: string) => T;
+type IdReplaceFunc = (id: string) => { codename: string };
+
+interface ProcessCodenamesResult {
+    readonly codenames: ReadonlySet<string>;
+    readonly html: string;
+}
+
+interface ProcessIdsResult {
+    readonly ids: ReadonlySet<string>;
+    readonly html: string;
+}
+
+const attributes = {
+    codenames: {
+        itemCodenameAttribute: 'data-codename',
+        linkItemCodenameAttribute: 'data-item-codename',
+        assetCodenameAttribute: 'data-asset-codename'
+    },
+    ids: {
+        assetIdAttributeName: 'data-asset-id',
+        itemIdAttributeName: 'data-item-id',
+        itemExternalIdAttributeName: 'data-item-external-id',
+        idAttributeName: 'data-id',
+        externalIdAttributeName: 'data-external-id',
+        externalAssetIdAttributeName: 'data-asset-external-id'
+    },
+    componentIdentifierAttribute: 'data-type="component"'
+} as const;
+
+const rteRegexes = {
+    elements: {
+        objectTagRegex: new RegExp(`<object(.+?)</object>`, 'g'),
+        imgTagRegex: new RegExp(`<img(.+?)</img>`, 'g'),
+        figureTagRegex: new RegExp(`<figure(.+?)</figure>`, 'g'),
+        linkTagRegex: new RegExp(`<a(.+?)</a>`, 'g')
+    },
+    ids: {
+        dataItemIdAttrRegex: new RegExp(`${attributes.ids.itemIdAttributeName}=\\"(.+?)\\"`),
+        dataAssetIdAttrRegex: new RegExp(`${attributes.ids.assetIdAttributeName}=\\"(.+?)\\"`),
+        dataIdAttrRegex: new RegExp(`${attributes.ids.idAttributeName}=\\"(.+?)\\"`)
+    },
+    codenames: {
+        rteItemCodenameRegex: new RegExp(`${attributes.codenames.itemCodenameAttribute}=\\"(.+?)\\"`),
+        rteLinkItemCodenameRegex: new RegExp(`${attributes.codenames.linkItemCodenameAttribute}=\\"(.+?)\\"`),
+        rteAssetCodenameRegex: new RegExp(`${attributes.codenames.assetCodenameAttribute}=\\"(.+?)\\"`)
+    }
+} as const;
+
+export function richTextProcessor() {
+    const processDataIds = (richTextHtml: string, replaceFunc?: IdReplaceFunc): ProcessIdsResult => {
+        const itemIds = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                html: richTextHtml,
+                ids: itemIds
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.objectTagRegex, (objectTag) => {
+            // skip processing for components
+            if (objectTag.includes(attributes.componentIdentifierAttribute)) {
+                return objectTag;
+            }
+            const itemIdMatch = objectTag.match(rteRegexes.ids.dataIdAttrRegex);
+            if (itemIdMatch && (itemIdMatch?.length ?? 0) >= 2) {
+                const itemId = itemIdMatch[1];
+
+                itemIds.add(itemId);
+
+                if (replaceFunc) {
+                    const { codename } = replaceFunc(itemId);
+
+                    return objectTag.replaceAll(
+                        `${attributes.ids.idAttributeName}="${itemId}"`,
+                        `${attributes.codenames.itemCodenameAttribute}="${codename}"`
+                    );
+                }
+            }
+
+            return objectTag;
+        });
+
+        return {
+            html: richTextHtml,
+            ids: itemIds
+        };
+    };
+
+    const processAssetIds = (richTextHtml: string, replaceFunc?: IdReplaceFunc): ProcessIdsResult => {
+        const assetIds = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                html: richTextHtml,
+                ids: assetIds
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.figureTagRegex, (figureTag) => {
+            const assetIdMatch = figureTag.match(rteRegexes.ids.dataAssetIdAttrRegex);
+            if (assetIdMatch && (assetIdMatch?.length ?? 0) >= 2) {
+                const assetId = assetIdMatch[1];
+
+                assetIds.add(assetId);
+
+                if (replaceFunc) {
+                    const { codename } = replaceFunc(assetId);
+
+                    return figureTag.replaceAll(
+                        `${attributes.ids.assetIdAttributeName}="${assetId}"`,
+                        `${attributes.codenames.assetCodenameAttribute}="${codename}"`
+                    );
+                }
+            }
+
+            return figureTag;
+        });
+
+        return {
+            html: richTextHtml,
+            ids: assetIds
+        };
+    };
+
+    const processLinkItemIds = (richTextHtml: string, replaceFunc?: IdReplaceFunc): ProcessIdsResult => {
+        const linkItemIds = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                html: richTextHtml,
+                ids: linkItemIds
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.linkTagRegex, (linkTag) => {
+            const itemIdMatch = linkTag.match(rteRegexes.ids.dataItemIdAttrRegex);
+            if (itemIdMatch && (itemIdMatch?.length ?? 0) >= 2) {
+                const itemId = itemIdMatch[1];
+                linkItemIds.add(itemId);
+
+                if (replaceFunc) {
+                    const { codename } = replaceFunc(itemId);
+
+                    return linkTag.replaceAll(
+                        `${attributes.ids.itemIdAttributeName}="${itemId}"`,
+                        `${attributes.codenames.linkItemCodenameAttribute}="${codename}"`
+                    );
+                }
+            }
+
+            return linkTag;
+        });
+
+        return {
+            html: richTextHtml,
+            ids: linkItemIds
+        };
+    };
+
+    const processItemCodenames = (
+        richTextHtml: string,
+        replaceFunc?: CodenameReplaceFunc<ItemStateInTargetEnvironmentByCodename>
+    ): ProcessCodenamesResult => {
+        const itemCodenames = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                codenames: itemCodenames,
+                html: richTextHtml
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.objectTagRegex, (objectTag) => {
+            const codenameMatch = objectTag.match(rteRegexes.codenames.rteItemCodenameRegex);
+            if (codenameMatch && (codenameMatch?.length ?? 0) >= 2) {
+                const codename = codenameMatch[1];
+
+                itemCodenames.add(codename);
+
+                if (replaceFunc) {
+                    const itemState = replaceFunc(codename);
+
+                    if (itemState.item && itemState.state === 'exists') {
+                        // no need to replace codename attribute
+                        return objectTag;
+                    }
+
+                    return objectTag.replaceAll(
+                        `${attributes.codenames.itemCodenameAttribute}="${codename}"`,
+                        `${attributes.ids.externalIdAttributeName}="${itemState.externalIdToUse}"`
+                    );
+                }
+            }
+
+            return objectTag;
+        });
+
+        return {
+            codenames: itemCodenames,
+            html: richTextHtml
+        };
+    };
+
+    const processLinkItemCodenames = (
+        richTextHtml: string,
+        replaceFunc?: CodenameReplaceFunc<ItemStateInTargetEnvironmentByCodename>
+    ): ProcessCodenamesResult => {
+        const itemCodenames = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                codenames: itemCodenames,
+                html: richTextHtml
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.linkTagRegex, (linkTag) => {
+            const codenameMatch = linkTag.match(rteRegexes.codenames.rteLinkItemCodenameRegex);
+            if (codenameMatch && (codenameMatch?.length ?? 0) >= 2) {
+                const codename = codenameMatch[1];
+
+                itemCodenames.add(codename);
+
+                if (replaceFunc) {
+                    const itemState = replaceFunc(codename);
+
+                    if (itemState.item && itemState.state === 'exists') {
+                        return linkTag;
+                    }
+                    return linkTag.replaceAll(
+                        `${attributes.codenames.linkItemCodenameAttribute}="${codename}"`,
+                        `${attributes.ids.itemExternalIdAttributeName}="${itemState.externalIdToUse}"`
+                    );
+                }
+            }
+
+            return linkTag;
+        });
+
+        return {
+            codenames: itemCodenames,
+            html: richTextHtml
+        };
+    };
+
+    const processAssetCodenames = (
+        richTextHtml: string,
+        replaceFunc?: CodenameReplaceFunc<AssetStateInTargetEnvironmentByCodename>
+    ): ProcessCodenamesResult => {
+        const assetCodenames = new Set<string>();
+
+        if (!richTextHtml) {
+            return {
+                codenames: assetCodenames,
+                html: richTextHtml
+            };
+        }
+
+        richTextHtml = richTextHtml.replaceAll(rteRegexes.elements.figureTagRegex, (figureTag) => {
+            const codenameMatch = figureTag.match(rteRegexes.codenames.rteAssetCodenameRegex);
+            if (codenameMatch && (codenameMatch?.length ?? 0) >= 2) {
+                const codename = codenameMatch[1];
+
+                assetCodenames.add(codename);
+
+                if (replaceFunc) {
+                    const assetState = replaceFunc(codename);
+
+                    if (assetState.asset && assetState.state === 'exists') {
+                        return figureTag;
+                    }
+                    return figureTag.replaceAll(
+                        `${attributes.codenames.assetCodenameAttribute}="${codename}"`,
+                        `${attributes.ids.externalAssetIdAttributeName}="${assetState.externalIdToUse}"`
+                    );
+                }
+            }
+
+            return figureTag;
+        });
+
+        return {
+            codenames: assetCodenames,
+            html: richTextHtml
+        };
+    };
+
+    return {
+        processAssetIds,
+        processDataIds,
+        processLinkItemIds,
+        processAssetCodenames,
+        processItemCodenames,
+        processLinkItemCodenames
+    };
+}

package/lib/translation/index.ts

@@ -0,0 +1,4 @@
+export * from './helpers/rich-text.processor.js';
+export * from './extraction/items-extraction.processor.js';
+export * from './transforms/export-transforms.js';
+export * from './transforms/import-transforms.js';