@kontent-ai/migration-toolkit 1.0.0

package/lib/core/utils/http.utils.ts

@@ -0,0 +1,35 @@
+import { HttpService, IRetryStrategyOptions } from '@kontent-ai/core-sdk';
+import { OriginalManagementError } from '../models/core.models.js';
+
+const rateExceededErrorCode: number = 10000;
+const notFoundErrorCode: number = 10000;
+
+export const defaultHttpService: Readonly<HttpService> = new HttpService({
+    logErrorsToConsole: false
+});
+
+export const defaultRetryStrategy: Readonly<IRetryStrategyOptions> = {
+    addJitter: true,
+    canRetryError: (err) => {
+        const originalError = err as OriginalManagementError | undefined;
+        const errorCode: number = originalError?.response?.data?.error_code ?? -1;
+
+        if (errorCode === rateExceededErrorCode) {
+            // retry rate exceeded error
+            return true;
+        }
+
+        if (errorCode === notFoundErrorCode) {
+            // do not retry errors indicating resource does not exist
+            return false;
+        }
+
+        if (errorCode >= 0) {
+            // otherwise if error code is set, do not retry the request
+            return false;
+        }
+        return true;
+    },
+    maxAttempts: 3,
+    deltaBackoffMs: 1000
+};

package/lib/core/utils/management-client-utils.ts

@@ -0,0 +1,182 @@
+import {
+    CollectionModels,
+    ContentTypeModels,
+    ContentTypeSnippetModels,
+    EnvironmentModels,
+    LanguageModels,
+    ManagementClient,
+    TaxonomyModels,
+    WorkflowModels,
+    createManagementClient
+} from '@kontent-ai/management-sdk';
+import { IRetryStrategyOptions } from '@kontent-ai/core-sdk';
+import { runMapiRequestAsync } from './run.utils.js';
+import { LogSpinnerData, Logger } from '../models/log.models.js';
+import { FlattenedContentType, FlattenedContentTypeElement } from '../models/core.models.js';
+import { isNotUndefined } from './global.utils.js';
+import chalk from 'chalk';
+import { defaultHttpService, defaultRetryStrategy } from './http.utils.js';
+import { findRequired } from './array.utils.js';
+
+export interface ManagementClientConfig {
+    readonly environmentId: string;
+    readonly apiKey: string;
+    readonly retryStrategy?: Readonly<IRetryStrategyOptions>;
+    readonly baseUrl?: string;
+}
+
+export function getMigrationManagementClient(config: ManagementClientConfig): ManagementClient {
+    return createManagementClient({
+        environmentId: config.environmentId,
+        retryStrategy: config.retryStrategy ?? defaultRetryStrategy,
+        httpService: defaultHttpService,
+        apiKey: config.apiKey
+    });
+}
+
+export function managementClientUtils(client: Readonly<ManagementClient>, logger: Logger) {
+    const getEnvironmentAsync = async (): Promise<EnvironmentModels.EnvironmentInformationModel> => {
+        return (await client.environmentInformation().toPromise()).data.project;
+    };
+
+    const getAllLanguagesAsync = async (
+        logSpinner: LogSpinnerData
+    ): Promise<readonly LanguageModels.LanguageModel[]> => {
+        return await runMapiRequestAsync({
+            logSpinner: logSpinner,
+            logger: logger,
+            func: async () => (await client.listLanguages().toAllPromise()).data.items,
+            action: 'list',
+            type: 'language'
+        });
+    };
+
+    const getAllCollectionsAsync = async (
+        logSpinner: LogSpinnerData
+    ): Promise<readonly CollectionModels.Collection[]> => {
+        return await runMapiRequestAsync({
+            logger: logger,
+            logSpinner: logSpinner,
+            func: async () => (await client.listCollections().toPromise()).data.collections,
+            action: 'list',
+            type: 'collection'
+        });
+    };
+
+    const getAllWorkflowsAsync = async (logSpinner: LogSpinnerData): Promise<readonly WorkflowModels.Workflow[]> => {
+        return await runMapiRequestAsync({
+            logger: logger,
+            logSpinner: logSpinner,
+            func: async () => (await client.listWorkflows().toPromise()).data,
+            action: 'list',
+            type: 'workflow'
+        });
+    };
+
+    const getAllTaxonomiesAsync = async (logSpinner: LogSpinnerData): Promise<readonly TaxonomyModels.Taxonomy[]> => {
+        return await runMapiRequestAsync({
+            logger: logger,
+            logSpinner: logSpinner,
+            func: async () => (await client.listTaxonomies().toAllPromise()).data.items,
+            action: 'list',
+            type: 'taxonomy'
+        });
+    };
+
+    const getContentTypeElements = (
+        contentType: ContentTypeModels.ContentType,
+        contentTypeSnippets: readonly ContentTypeSnippetModels.ContentTypeSnippet[]
+    ): readonly FlattenedContentTypeElement[] => {
+        return contentType.elements
+            .flatMap((element) => {
+                if (!element.codename || !element.id) {
+                    return undefined;
+                }
+
+                if (element.type === 'guidelines') {
+                    return undefined;
+                }
+
+                if (element.type === 'snippet') {
+                    const snippetElement = element;
+
+                    // replace snippet element with actual elements
+                    const contentTypeSnippet = findRequired(
+                        contentTypeSnippets,
+                        (snippet) => snippet.id === snippetElement.snippet.id,
+                        `Could not find content type snippet for element. This snippet is referenced in type '${chalk.red(
+                            contentType.codename
+                        )}'`
+                    );
+
+                    return contentTypeSnippet.elements.map((snippetElement) => {
+                        if (snippetElement.type === 'guidelines' || snippetElement.type === 'snippet') {
+                            return undefined;
+                        }
+                        if (!snippetElement.codename || !snippetElement.id) {
+                            return undefined;
+                        }
+
+                        const flattenedElement: FlattenedContentTypeElement = {
+                            codename: snippetElement.codename,
+                            type: snippetElement.type,
+                            id: snippetElement.id,
+                            element: element
+                        };
+
+                        return flattenedElement;
+                    });
+                }
+                const flattenedElement: FlattenedContentTypeElement = {
+                    codename: element.codename,
+                    id: element.id,
+                    type: element.type,
+                    element: element
+                };
+
+                return flattenedElement;
+            })
+            .filter(isNotUndefined);
+    };
+
+    const getFlattenedContentTypesAsync = async (
+        logSpinner: LogSpinnerData
+    ): Promise<readonly FlattenedContentType[]> => {
+        const contentTypes = await runMapiRequestAsync({
+            logger: logger,
+            logSpinner: logSpinner,
+            func: async () => (await client.listContentTypes().toAllPromise()).data.items,
+            action: 'list',
+            type: 'contentType'
+        });
+
+        const contentTypeSnippets = await runMapiRequestAsync({
+            logger: logger,
+            logSpinner: logSpinner,
+            func: async () => (await client.listContentTypeSnippets().toAllPromise()).data.items,
+            action: 'list',
+            type: 'contentTypeSnippet'
+        });
+
+        return [
+            ...contentTypes.map((contentType) => {
+                const importType: FlattenedContentType = {
+                    contentTypeCodename: contentType.codename,
+                    contentTypeId: contentType.id,
+                    elements: getContentTypeElements(contentType, contentTypeSnippets)
+                };
+
+                return importType;
+            })
+        ];
+    };
+
+    return {
+        getAllCollectionsAsync,
+        getAllLanguagesAsync,
+        getAllWorkflowsAsync,
+        getAllTaxonomiesAsync,
+        getFlattenedContentTypesAsync,
+        getEnvironmentAsync
+    };
+}

package/lib/core/utils/processing-utils.ts

@@ -0,0 +1,71 @@
+import chalk from 'chalk';
+import pLimit from 'p-limit';
+import { ItemInfo } from '../models/core.models.js';
+import { LogSpinnerData, Logger } from '../models/log.models.js';
+
+type ProcessSetAction =
+    | 'Fetching assets'
+    | 'Downloading assets'
+    | 'Fetching content items'
+    | 'Preparing content items & language variants'
+    | 'Importing content items'
+    | 'Importing language variants'
+    | 'Fetching language variants'
+    | 'Upserting assets'
+    | 'Uploading assets';
+
+export async function processItemsAsync<InputItem, OutputItem>(data: {
+    readonly action: ProcessSetAction;
+    readonly logger: Logger;
+    readonly items: Readonly<InputItem[]>;
+    readonly parallelLimit: number;
+    readonly processAsync: (item: Readonly<InputItem>, logSpinner: LogSpinnerData) => Promise<Readonly<OutputItem>>;
+    readonly itemInfo: (item: Readonly<InputItem>) => ItemInfo;
+}): Promise<readonly OutputItem[]> {
+    if (!data.items.length) {
+        return [];
+    }
+
+    const firstItemInfo = data.itemInfo(data.items[0]);
+
+    return await data.logger.logWithSpinnerAsync(async (logSpinner) => {
+        const limit = pLimit(data.parallelLimit);
+        let processedItemsCount: number = 1;
+
+        const requests: Promise<OutputItem>[] = data.items.map((item) =>
+            limit(() => {
+                return data.processAsync(item, logSpinner).then((output) => {
+                    const itemInfo = data.itemInfo(item);
+                    const prefix = getPercentagePrefix(processedItemsCount, data.items.length);
+
+                    logSpinner({
+                        prefix: prefix,
+                        message: itemInfo.title,
+                        type: itemInfo.itemType
+                    });
+
+                    processedItemsCount++;
+                    return output;
+                });
+            })
+        );
+
+        // log processing of first item as progress is set after each item finishes
+        logSpinner({
+            prefix: getPercentagePrefix(0, data.items.length),
+            message: firstItemInfo.title,
+            type: firstItemInfo.itemType
+        });
+
+        // Only '<parallelLimit>' promises at a time
+        const outputItems = await Promise.all(requests);
+
+        logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}' (${outputItems.length})` });
+
+        return outputItems;
+    });
+}
+
+function getPercentagePrefix(processedItems: number, totalCount: number): string {
+    return chalk.gray(`${Math.round((processedItems / totalCount) * 100)}%`);
+}

package/lib/core/utils/run.utils.ts

@@ -0,0 +1,51 @@
+import chalk from 'chalk';
+import { MapiAction, MapiType } from '../models/core.models.js';
+import { LogMessage, LogSpinnerData, Logger } from '../models/log.models.js';
+
+export async function runMapiRequestAsync<TResult>(data: {
+    readonly logger: Logger;
+    readonly logSpinner?: LogSpinnerData;
+    readonly action: MapiAction;
+    readonly type: MapiType;
+    readonly func: () => Promise<TResult>;
+    readonly itemName?: string;
+}): Promise<TResult> {
+    let logData: LogMessage | undefined;
+
+    if (data.action === 'list') {
+        logData = {
+            message: `Fetching objects of type '${chalk.yellow(data.type)}'`,
+            type: data.action
+        };
+    } else if (data.itemName) {
+        logData = {
+            message: `${data.type} -> ${data.itemName}`,
+            type: data.action
+        };
+    } else {
+        logData = {
+            message: `${data.type}`,
+            type: data.action
+        };
+    }
+
+    logSpinnerOrDefault({
+        logSpinner: data.logSpinner,
+        logger: data.logger,
+        logData: logData
+    });
+
+    return await data.func();
+}
+
+function logSpinnerOrDefault(data: {
+    readonly logSpinner: LogSpinnerData | undefined;
+    readonly logData: LogMessage;
+    readonly logger: Logger;
+}): void {
+    if (data.logSpinner) {
+        data.logSpinner(data.logData);
+    } else {
+        data.logger.log(data.logData);
+    }
+}