@stackbit/sdk 0.2.21 → 0.2.25

package/src/config/config-loader.ts

@@ -9,7 +9,7 @@ import { ConfigError, ConfigLoadError, ConfigValidationError } from './config-er
 import {
     assignLabelFieldIfNeeded,
     extendModelMap,
-    getListItemsField,
+    getListFieldItems,
     isCustomModelField,
     isEnumField,
     isListDataModel,
@@ -20,14 +20,19 @@ import {
     isPageModel,
     isDataModel,
     isReferenceField,
-    iterateModelFieldsRecursively
+    iterateModelFieldsRecursively,
+    mapModelFieldsRecursively,
+    normalizeListFieldInPlace,
+    getModelFieldForModelKeyPath
 } from '../utils';
-import { append, parseFile, readDirRecursively, reducePromise, rename } from '@stackbit/utils';
-import { Config, DataModel, FieldEnum, FieldModel, FieldObjectProps, Model, PageModel, YamlModel } from './config-types';
+import { append, omitByNil, parseFile, readDirRecursively, reducePromise, rename } from '@stackbit/utils';
+import { Config, DataModel, FieldEnum, FieldModel, FieldObjectProps, Model, ModelsSource, PageModel, YamlModel } from './config-types';
 import { loadPresets } from './presets-loader';
 
 export interface ConfigLoaderOptions {
+    [option: string]: any;
     dirPath: string;
+    modelsSource?: ModelsSource;
 }
 
 export interface ConfigLoaderResult {
@@ -43,81 +48,94 @@ export interface NormalizedValidationResult {
 }
 
 export interface TempConfigLoaderResult {
-    config?: any;
+    config?: Record<string, unknown>;
     errors: ConfigLoadError[];
 }
 
-export async function loadConfig({ dirPath }: ConfigLoaderOptions): Promise<ConfigLoaderResult> {
-    let configLoadResult: TempConfigLoaderResult;
-    try {
-        configLoadResult = await loadConfigFromDir(dirPath);
-    } catch (error) {
-        return {
-            valid: false,
-            config: null,
-            errors: [new ConfigLoadError(`Error loading Stackbit configuration: ${error.message}`, { originalError: error })]
-        };
-    }
+export async function loadConfig({ dirPath, ...options }: ConfigLoaderOptions): Promise<ConfigLoaderResult> {
+    const { config, errors: configLoadErrors } = await loadConfigFromDir(dirPath);
 
-    if (!configLoadResult.config) {
+    if (!config) {
         return {
             valid: false,
             config: null,
-            errors: configLoadResult.errors
+            errors: configLoadErrors
         };
     }
 
-    const normalizedResult = validateAndNormalizeConfig(configLoadResult.config);
+    const { models: externalModels, errors: externalModelsLoadErrors } = await loadModelsFromExternalSource(config, dirPath, options);
+
+    const normalizedResult = validateAndNormalizeConfig(config, externalModels);
 
     const presetsResult = await loadPresets(dirPath, normalizedResult.config);
 
     return {
         valid: normalizedResult.valid,
         config: presetsResult.config,
-        errors: [...configLoadResult.errors, ...normalizedResult.errors, ...presetsResult.errors]
+        errors: [...configLoadErrors, ...externalModelsLoadErrors, ...normalizedResult.errors, ...presetsResult.errors]
     };
 }
 
-export function validateAndNormalizeConfig(config: any): NormalizedValidationResult {
-    // validate the "contentModels" and extend config models with "contentModels"
-    // this must be done before main config validation to make it independent of "contentModels".
-    const contentModelsValidationResult = validateAndExtendContentModels(config);
-    config = contentModelsValidationResult.value;
-
+export function validateAndNormalizeConfig(config: any, externalModels?: Model[]): NormalizedValidationResult {
     // extend config models having the "extends" property
-    // this must be done before main config validation as some properties like
+    // this must be done before any validation as some properties like
     // the labelField will not work when validating models without extending them first
-    const { models, errors: extendModelErrors } = extendModelMap(config?.models || {});
-    config.models = models;
+    const { models: extendedModels, errors: extendModelErrors } = extendModelMap(config.models as any);
+    const extendedConfig = {
+        ...config,
+        models: extendedModels
+    };
+
+    const { config: mergedConfig, errors: externalModelsMergeErrors } = mergeConfigWithExternalModels(extendedConfig, externalModels);
+
+    // validate the "contentModels" and extend config models with "contentModels"
+    // this must be done before main config validation to make it independent of "contentModels".
+    const { value: configWithContentModels, errors: contentModelsErrors } = validateAndExtendContentModels(mergedConfig);
 
     // normalize config - backward compatibility updates, adding extra fields like "markdown_content", "type" and "layout",
     // and setting other default values.
-    config = normalizeConfig(config);
+    const normalizedConfig = normalizeConfig(configWithContentModels);
 
     // validate config
-    const configValidationResult = validateConfig(config);
+    const { value: validatedConfig, errors: validationErrors } = validateConfig(normalizedConfig);
+
+    const errors = [...extendModelErrors, ...externalModelsMergeErrors, ...contentModelsErrors, ...validationErrors];
 
-    const errors = [...contentModelsValidationResult.errors, ...extendModelErrors, ...configValidationResult.errors];
     return normalizeValidationResult({
         valid: _.isEmpty(errors),
-        value: configValidationResult.value,
+        value: validatedConfig,
         errors: errors
     });
 }
 
 async function loadConfigFromDir(dirPath: string): Promise<TempConfigLoaderResult> {
-    const { config, error } = await loadConfigFromStackbitYaml(dirPath);
-    if (error) {
-        return { errors: [error] };
+    try {
+        const { config, error } = await loadConfigFromStackbitYaml(dirPath);
+        if (error) {
+            return { errors: [error] };
+        }
+
+        const { models: modelsFromFiles, errors: fileModelsErrors } = await loadModelsFromFiles(dirPath, config);
+
+        const mergedModels = mergeConfigModelsWithModelsFromFiles(config.models ?? {}, modelsFromFiles);
+
+        return {
+            config: {
+                ...config,
+                models: mergedModels
+            },
+            errors: fileModelsErrors
+        };
+    } catch (error) {
+        return {
+            errors: [new ConfigLoadError(`Error loading Stackbit configuration: ${error.message}`, { originalError: error })]
+        };
     }
-    const externalModelsResult = await loadExternalModels(dirPath, config);
-    config.models = _.assign(externalModelsResult.models, config.models);
-    return { config, errors: externalModelsResult.errors };
 }
 
-type LoadConfigFromStackbitYamlResult = { config: any; error?: undefined } | { config?: undefined; error: ConfigLoadError };
+type StackbitYamlConfigResult = { config: any; error?: undefined } | { config?: undefined; error: ConfigLoadError };
 
-async function loadConfigFromStackbitYaml(dirPath: string): Promise<LoadConfigFromStackbitYamlResult> {
+async function loadConfigFromStackbitYaml(dirPath: string): Promise<StackbitYamlConfigResult> {
     const stackbitYamlPath = path.join(dirPath, 'stackbit.yaml');
     const stackbitYamlExists = await fse.pathExists(stackbitYamlPath);
     if (!stackbitYamlExists) {
@@ -135,54 +153,56 @@ async function loadConfigFromStackbitYaml(dirPath: string): Promise<LoadConfigFr
     return { config };
 }
 
-async function loadExternalModels(dirPath: string, config: any) {
+async function loadModelsFromFiles(dirPath: string, config: any): Promise<{ models: Record<string, any>; errors: ConfigLoadError[] }> {
     const modelsSource = _.get(config, 'modelsSource', {});
     const sourceType = _.get(modelsSource, 'type', 'files');
-    if (sourceType === 'files') {
-        const defaultModelDirs = ['node_modules/@stackbit/components/models', '.stackbit/models'];
-        const modelDirs = _.castArray(_.get(modelsSource, 'modelDirs', defaultModelDirs)).map((modelDir: string) => _.trim(modelDir, '/'));
-        const modelFiles = await reducePromise(
-            modelDirs,
-            async (modelFiles: string[], modelDir) => {
-                const absModelsDir = path.join(dirPath, modelDir);
-                const dirExists = await fse.pathExists(absModelsDir);
-                if (!dirExists) {
-                    return modelFiles;
-                }
-                const files = await readModelFilesFromDir(absModelsDir);
-                return modelFiles.concat(files.map((filePath) => path.join(modelDir, filePath)));
-            },
-            []
-        );
-        return reducePromise(
-            modelFiles,
-            async (result: { models: any; errors: ConfigLoadError[] }, modelFile) => {
-                let model;
-                try {
-                    model = await parseFile(path.join(dirPath, modelFile));
-                } catch (error) {
-                    return {
-                        models: result.models,
-                        errors: result.errors.concat(new ConfigLoadError(`error parsing model, file: ${modelFile}`))
-                    };
-                }
-                const modelName = model?.name;
-                if (!modelName) {
-                    return {
-                        models: result.models,
-                        errors: result.errors.concat(new ConfigLoadError(`model does not have a name, file: ${modelFile}`))
-                    };
-                }
-                result.models[modelName] = _.omit(model, 'name');
-                result.models[modelName].__metadata = {
-                    filePath: modelFile
+    const defaultModelDirs = ['node_modules/@stackbit/components/models', '.stackbit/models'];
+    const modelDirs =
+        sourceType === 'files'
+            ? _.castArray(_.get(modelsSource, 'modelDirs', defaultModelDirs)).map((modelDir: string) => _.trim(modelDir, '/'))
+            : defaultModelDirs;
+
+    const modelFiles = await reducePromise(
+        modelDirs,
+        async (modelFiles: string[], modelDir) => {
+            const absModelsDir = path.join(dirPath, modelDir);
+            const dirExists = await fse.pathExists(absModelsDir);
+            if (!dirExists) {
+                return modelFiles;
+            }
+            const files = await readModelFilesFromDir(absModelsDir);
+            return modelFiles.concat(files.map((filePath) => path.join(modelDir, filePath)));
+        },
+        []
+    );
+
+    return reducePromise(
+        modelFiles,
+        async (result: { models: any; errors: ConfigLoadError[] }, modelFile) => {
+            let model;
+            try {
+                model = await parseFile(path.join(dirPath, modelFile));
+            } catch (error) {
+                return {
+                    models: result.models,
+                    errors: result.errors.concat(new ConfigLoadError(`error parsing model, file: ${modelFile}`))
                 };
-                return result;
-            },
-            { models: {}, errors: [] }
-        );
-    }
-    return { models: {}, errors: [] };
+            }
+            const modelName = model?.name;
+            if (!modelName) {
+                return {
+                    models: result.models,
+                    errors: result.errors.concat(new ConfigLoadError(`model does not have a name, file: ${modelFile}`))
+                };
+            }
+            result.models[modelName] = _.omit(model, 'name');
+            result.models[modelName].__metadata = {
+                filePath: modelFile
+            };
+            return result;
+        },
+        { models: {}, errors: [] }
+    );
 }
 
 async function readModelFilesFromDir(modelsDir: string) {
@@ -197,6 +217,38 @@ async function readModelFilesFromDir(modelsDir: string) {
     });
 }
 
+async function loadModelsFromExternalSource(
+    config: any,
+    dirPath: string,
+    options: Omit<ConfigLoaderOptions, 'dirPath'> = {}
+): Promise<{ models: Model[]; errors: ConfigLoadError[] }> {
+    const modelsSource = _.get(config, 'modelsSource', options.modelsSource);
+    const sourceType = _.get(modelsSource, 'type', 'files');
+    if (sourceType === 'files') {
+        return { models: [], errors: [] };
+    } else if (sourceType === 'contentful') {
+        const contentfulModule = _.get(modelsSource, 'module', '@stackbit/cms-contentful');
+        const modulePath = path.resolve(dirPath, 'node_modules', contentfulModule);
+        const module = await import(modulePath);
+        try {
+            const { models } = await module.fetchAndConvertSchema(_.omit(options, 'modelsSource'));
+            return {
+                models: models,
+                errors: []
+            };
+        } catch (error) {
+            return {
+                models: [],
+                errors: [new ConfigLoadError(`Error fetching and converting Contentful schema, error: ${error.message}`, { originalError: error })]
+            };
+        }
+    }
+    return {
+        models: [],
+        errors: [new ConfigLoadError(`modelsSource ${modelsSource} is unsupported`)]
+    };
+}
+
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 async function loadConfigFromDotStackbit(dirPath: string) {
     const stackbitDotPath = path.join(dirPath, '.stackbit');
@@ -231,6 +283,113 @@ async function loadConfigFromDotStackbit(dirPath: string) {
     return _.isEmpty(config) ? null : config;
 }
 
+function mergeConfigModelsWithModelsFromFiles(configModels: any, modelsFromFiles: Record<string, any>) {
+    const mergedModels = _.mapValues(modelsFromFiles, (modelFromFile, modelName) => {
+        // resolve thumbnails of models loaded from files
+        const modelFilePath = modelFromFile.__metadata?.filePath;
+        resolveThumbnailPathForModel(modelFromFile, modelFilePath);
+        iterateModelFieldsRecursively(modelFromFile, (field: any) => {
+            if (isListField(field)) {
+                field = normalizeListFieldInPlace(field);
+                field = field.items;
+            }
+            if (isObjectField(field)) {
+                resolveThumbnailPathForModel(field, modelFilePath);
+            } else if (isEnumField(field)) {
+                resolveThumbnailPathForEnumField(field, modelFilePath);
+            }
+        });
+
+        const configModel = _.get(configModels, modelName);
+        if (!configModel) {
+            return modelFromFile;
+        }
+
+        return _.assign({}, modelFromFile, configModel, {
+            fields: _.unionBy(configModel?.fields ?? [], modelFromFile?.fields ?? [], 'name')
+        });
+    });
+    return Object.assign({}, configModels, mergedModels);
+}
+
+function mergeConfigWithExternalModels(config: any, externalModels?: Model[]): { config: any; errors: ConfigValidationError[] } {
+    if (!externalModels || externalModels.length === 0) {
+        return {
+            config,
+            errors: []
+        };
+    }
+
+    const stackbitModels = config?.models ?? {};
+    const errors: ConfigValidationError[] = [];
+
+    const models = _.reduce(
+        externalModels,
+        (modelMap: Record<string, YamlModel>, externalModel) => {
+            const { name, ...rest } = externalModel;
+            return Object.assign(modelMap, { [name]: rest });
+        },
+        {}
+    );
+
+    _.forEach(stackbitModels, (stackbitModel: any, modelName: any) => {
+        let externalModel = models[modelName];
+        if (!externalModel) {
+            return;
+        }
+
+        const modelType = stackbitModel.type ? (stackbitModel.type === 'config' ? 'data' : stackbitModel.type) : externalModel.type ?? 'object';
+        const urlPath = modelType === 'page' ? stackbitModel?.urlPath ?? '/{slug}' : null;
+
+        externalModel = Object.assign(
+            {},
+            externalModel,
+            _.pick(stackbitModel, ['__metadata', 'label', 'description', 'thumbnail', 'singleInstance', 'readOnly', 'labelField', 'fieldGroups']),
+            omitByNil({
+                type: modelType,
+                urlPath
+            })
+        );
+
+        externalModel = mapModelFieldsRecursively(externalModel as Model, (field, modelKeyPath) => {
+            const stackbitField = getModelFieldForModelKeyPath(stackbitModel, modelKeyPath);
+            if (!stackbitField) {
+                return field;
+            }
+
+            let override = {};
+            if (stackbitField.type === 'style') {
+                override = stackbitField;
+            } else if (field.type === 'enum') {
+                override = _.pick(stackbitField, ['options']);
+            } else if (field.type === 'color') {
+                override = { type: 'color' };
+            } else if (field.type === 'number') {
+                override = _.pick(stackbitField, ['subtype', 'min', 'max', 'step', 'unit']);
+            } else if (field.type === 'object') {
+                override = _.pick(stackbitField, ['labelField', 'thumbnail', 'fieldGroups']);
+            }
+
+            return Object.assign(
+                {},
+                field,
+                _.pick(stackbitField, ['label', 'description', 'required', 'default', 'group', 'const', 'hidden', 'readOnly', 'controlType']),
+                override
+            );
+        }) as YamlModel;
+
+        models[modelName] = externalModel;
+    });
+
+    return {
+        config: {
+            ...config,
+            models: models
+        },
+        errors: errors
+    };
+}
+
 function normalizeConfig(config: any): any {
     const pageLayoutKey = _.get(config, 'pageLayoutKey', 'layout');
     const objectTypeKey = _.get(config, 'objectTypeKey', 'type');
@@ -238,6 +397,7 @@ function normalizeConfig(config: any): any {
     const ver = semver.coerce(stackbitYamlVersion);
     const isStackbitYamlV2 = ver ? semver.satisfies(ver, '<0.3.0') : false;
     const models = config?.models || {};
+    const gitCMS = isGitCMS(config);
     let referencedModelNames: string[] = [];
 
     _.forEach(models, (model, modelName) => {
@@ -245,6 +405,10 @@ function normalizeConfig(config: any): any {
             return;
         }
 
+        if (!_.has(model, 'type')) {
+            model.type = 'object';
+        }
+
         // add model label if not set
         if (!_.has(model, 'label')) {
             model.label = _.startCase(modelName);
@@ -259,13 +423,15 @@ function normalizeConfig(config: any): any {
             rename(model, 'template', 'layout');
 
             updatePageUrlPath(model);
-            updatePageFilePath(model, config);
 
-            addMarkdownContentField(model);
+            if (gitCMS) {
+                updatePageFilePath(model, config);
+                addMarkdownContentField(model);
 
-            // TODO: update schema-editor to not show layout field
-            addLayoutFieldToPageModel(model, pageLayoutKey);
-        } else if (isDataModel(model)) {
+                // TODO: update schema-editor to not show layout field
+                addLayoutFieldToPageModel(model, pageLayoutKey);
+            }
+        } else if (isDataModel(model) && gitCMS) {
             updateDataFilePath(model, config);
         }
 
@@ -281,8 +447,6 @@ function normalizeConfig(config: any): any {
             assignLabelFieldIfNeeded(model);
         }
 
-        resolveThumbnailPathForModel(model, model?.__metadata?.filePath);
-
         iterateModelFieldsRecursively(model, (field: any) => {
             // add field label if label is not set
             if (!_.has(field, 'label')) {
@@ -290,18 +454,12 @@ function normalizeConfig(config: any): any {
             }
 
             if (isListField(field)) {
-                // 'items.type' of list field default to 'string', set it explicitly
-                if (!_.has(field, 'items.type')) {
-                    _.set(field, 'items.type', 'string');
-                }
-                field = getListItemsField(field);
+                field = normalizeListFieldInPlace(field);
+                field = field.items;
             }
 
             if (isObjectField(field)) {
                 assignLabelFieldIfNeeded(field);
-                resolveThumbnailPathForModel(field, model?.__metadata?.filePath);
-            } else if (isEnumField(field)) {
-                resolveThumbnailPathForEnumField(field, model?.__metadata?.filePath);
             } else if (isCustomModelField(field, models)) {
                 // stackbit v0.2.0 compatibility
                 // convert the old custom model field type: { type: 'action' }
@@ -331,7 +489,9 @@ function normalizeConfig(config: any): any {
                 }
             }
 
-            referencedModelNames = _.union(referencedModelNames, getReferencedModelNames(field));
+            if (gitCMS) {
+                referencedModelNames = _.union(referencedModelNames, getReferencedModelNames(field));
+            }
         });
     });
 
@@ -488,7 +648,7 @@ function resolveThumbnailPath(thumbnail: string, modelDirPath: string) {
  */
 function getReferencedModelNames(field: any) {
     if (isListField(field)) {
-        field = getListItemsField(field);
+        field = getListFieldItems(field);
     }
     // TODO: add type field to model fields inside container update/create object logic rather adding type to schema
     // 'object' models referenced by 'model' fields should have 'type' field
@@ -509,7 +669,10 @@ function validateAndExtendContentModels(config: any): ConfigValidationResult {
     const contentModels = config.contentModels ?? {};
     const models = config.models ?? {};
 
-    if (_.isEmpty(contentModels)) {
+    const externalModels = !isGitCMS(config);
+    const emptyContentModels = _.isEmpty(contentModels);
+
+    if (externalModels || emptyContentModels) {
         return {
             valid: true,
             value: config,
@@ -520,17 +683,17 @@ function validateAndExtendContentModels(config: any): ConfigValidationResult {
     const validationResult = validateContentModels(contentModels, models);
 
     if (_.isEmpty(models)) {
-        return validationResult;
+        return {
+            valid: validationResult.valid,
+            value: config,
+            errors: validationResult.errors
+        };
     }
 
     const extendedModels = _.mapValues(models, (model, modelName) => {
         const contentModel = validationResult.value.contentModels[modelName];
         if (!contentModel) {
-            return {
-                // if a model does not define a type, use the default "object" type
-                type: model.type || 'object',
-                ..._.omit(model, 'type')
-            };
+            return model;
         }
         if (_.get(contentModel, '__metadata.invalid')) {
             return model;
@@ -565,10 +728,43 @@ function validateAndExtendContentModels(config: any): ConfigValidationResult {
 }
 
 function normalizeValidationResult(validationResult: ConfigValidationResult): NormalizedValidationResult {
+    validationResult = filterAndOrderConfigFields(validationResult);
     convertModelGroupsToModelList(validationResult);
     return convertModelsToArray(validationResult);
 }
 
+function filterAndOrderConfigFields(validationResult: ConfigValidationResult): ConfigValidationResult {
+    // TODO: see if we move filtering and sorting to Joi
+    return {
+        ...validationResult,
+        value: _.pick(validationResult.value, [
+            'stackbitVersion',
+            'ssgName',
+            'ssgVersion',
+            'cmsName',
+            'import',
+            'buildCommand',
+            'publishDir',
+            'nodeVersion',
+            'devCommand',
+            'staticDir',
+            'uploadDir',
+            'assets',
+            'pagesDir',
+            'dataDir',
+            'pageLayoutKey',
+            'objectTypeKey',
+            'styleObjectModelName',
+            'excludePages',
+            'logicFields',
+            'contentModels',
+            'modelsSource',
+            'models',
+            'presets'
+        ])
+    };
+}
+
 function convertModelGroupsToModelList(validationResult: ConfigValidationResult) {
     const models = validationResult.value?.models ?? {};
 
@@ -598,7 +794,7 @@ function convertModelGroupsToModelList(validationResult: ConfigValidationResult)
     _.forEach(models, (model) => {
         iterateModelFieldsRecursively(model, (field: any) => {
             if (isListField(field)) {
-                field = getListItemsField(field);
+                field = field.items;
             }
             if (field.groups) {
                 let key: string | null = null;
@@ -640,6 +836,10 @@ function convertModelsToArray(validationResult: ConfigValidationResult): Normali
         }
     );
 
+    if (!isGitCMS(config)) {
+        addImageModel(modelArray);
+    }
+
     const convertedErrors = _.map(validationResult.errors, (error: ConfigValidationError) => {
         if (error.fieldPath[0] === 'models' && typeof error.fieldPath[1] == 'string') {
             const modelName = error.fieldPath[1];
@@ -660,3 +860,20 @@ function convertModelsToArray(validationResult: ConfigValidationResult): Normali
         errors: convertedErrors
     };
 }
+
+function addImageModel(models: Model[]) {
+    models.push({
+        type: 'image',
+        name: '__image_model',
+        label: 'Image',
+        labelField: 'title',
+        fields: [
+            { name: 'title', type: 'string' },
+            { name: 'url', type: 'string' }
+        ]
+    });
+}
+
+function isGitCMS(config: any) {
+    return !config.cmsName || config.cmsName === 'git';
+}