@stackbit/cms-core 0.1.25-cloudinary-presets.0 → 0.1.26-gitcms.0

package/src/services/git.ts

@@ -0,0 +1,227 @@
+import _ from 'lodash';
+import os from 'os';
+import path from 'path';
+import { v4 as uuid } from 'uuid';
+import fse from 'fs-extra';
+
+import { GitServiceInterface, GitFileObject, GitAuthor, GitCommitLogEntry, Logger } from '@stackbit/types';
+import { Worker } from '@stackbit/utils';
+
+import { RunService } from './run';
+import { DocumentStatus } from '@stackbit/types';
+
+const GIT_LOG_CHANGE_TYPES: Record<string, DocumentStatus> = {
+    M: 'modified',
+    A: 'added',
+    D: 'deleted'
+};
+
+export class GitService implements GitServiceInterface {
+    private readonly repoUrl: string;
+    private readonly repoDir: string;
+    private readonly repoBranch: string;
+    private readonly repoPublishBranch: string;
+    private readonly worker: Worker;
+    private readonly run: RunService;
+    private readonly logger: Logger;
+    private readonly userLogger: Logger;
+
+    private branchFetched: boolean = false;
+
+    constructor(options: {
+        repoUrl: string;
+        repoDir: string;
+        repoBranch: string;
+        repoPublishBranch: string;
+        worker: Worker;
+        runService: RunService;
+        logger: Logger;
+        userLogger: Logger;
+    }) {
+        this.repoUrl = options.repoUrl;
+        this.repoDir = options.repoDir;
+        this.repoBranch = options.repoBranch;
+        this.repoPublishBranch = options.repoPublishBranch;
+        this.worker = options.worker;
+        this.run = options.runService;
+        this.logger = options.logger;
+        this.userLogger = options.userLogger;
+    }
+
+    private async commit(author: GitAuthor, files: GitFileObject[]) {
+        const filePaths = _.map(files, 'filePath');
+        this.logger.debug('[git] Commit scheduled', filePaths);
+        return this.worker.schedule(async () => {
+            this.logger.debug('[git] Commit running', filePaths);
+            const message = files
+                .reduce((messages: string[], file) => {
+                    messages.push(`${path.parse(file.filePath).base}: ${file.description}`);
+                    return messages;
+                }, [])
+                .join('.\n');
+            await this.run.command('git', ['add', ...filePaths], this.repoDir);
+            await this.run.command(
+                'git',
+                ['commit', '--no-verify', '--author', `${author.name || author.email} <${author.email}>`, '-m', message],
+                this.repoDir
+            );
+            this.logger.debug('[git] Commit done', filePaths);
+        });
+    }
+
+    private push() {
+        this.logger.debug('[git] Push scheduled');
+        return this.worker.schedule(async () => {
+            this.logger.debug('[git] Push running');
+            await this.run.command('git', ['pull', 'origin', '--rebase', '--autostash', '-Xtheirs'], this.repoDir);
+            await this.run.command('git', ['push', 'origin'], this.repoDir);
+            this.logger.debug('[git] Push done');
+        });
+    }
+
+    async commitAndPush(author: GitAuthor, files: GitFileObject[]) {
+        await this.commit(author, files);
+        return this.push();
+    }
+
+    pull() {
+        this.logger.debug('[git] Pull scheduled');
+        return this.worker.schedule(async () => {
+            this.logger.debug('[git] Pull running');
+            await this.run.command('git', ['pull', 'origin', '--rebase', '--autostash', '-Xtheirs'], this.repoDir);
+            this.logger.debug('[git] Pull done');
+        });
+    }
+
+    private async publishAll(author: GitAuthor) {
+        this.logger.debug('[git] Publish all started');
+        const publishDir = path.join(os.tmpdir(), uuid());
+        await this.run.command('git', ['clone', this.repoUrl, '--branch', this.repoPublishBranch, publishDir]);
+        try {
+            await this.run.command('git', ['merge', `origin/${this.repoBranch}`, this.repoPublishBranch, '-Xtheirs'], publishDir);
+            await this.run
+                .command('git', ['commit', '--author', `${author.name || author.email} <${author.email}>`, `-m`, 'Publish'], publishDir)
+                .catch(() => {});
+            await this.run.command('git', ['push', 'origin'], publishDir);
+        } finally {
+            await fse.remove(publishDir);
+        }
+        this.logger.debug('[git] Publish all done');
+    }
+
+    private async publishFiles(author: GitAuthor, filePaths: string[]) {
+        this.logger.debug('[git] Publish files started', filePaths);
+        const publishDir = path.join(os.tmpdir(), uuid());
+        await this.run.command('git', ['clone', this.repoUrl, '--branch', this.repoPublishBranch, publishDir]);
+        try {
+            await Promise.all(
+                filePaths.map(async (filePath) => {
+                    const destFilePath = path.join(publishDir, filePath);
+                    await fse.ensureDir(path.dirname(destFilePath));
+                    return fse.copy(path.join(this.repoDir, filePath), destFilePath);
+                })
+            );
+
+            await this.run.command('git', ['checkout', '-b', 'stackbit-publish-branch'], publishDir);
+            await this.run.command('git', ['add', ...filePaths], publishDir);
+            await this.run.command('git', ['commit', '--author', `${author.name || author.email} <${author.email}>`, `-m`, 'Publish'], publishDir);
+            
+            await this.run.command('git', ['checkout', this.repoBranch], publishDir);
+            await this.run.command('git', ['merge', 'stackbit-publish-branch', this.repoBranch, '-Xtheirs'], publishDir);
+
+            await this.run.command('git', ['checkout', this.repoPublishBranch], publishDir);
+            await this.run.command('git', ['merge', 'stackbit-publish-branch', this.repoPublishBranch, '-Xtheirs'], publishDir);
+
+            await this.run.command('git', ['push', 'origin', this.repoPublishBranch, this.repoBranch], publishDir);
+        } finally {
+            await fse.remove(publishDir);
+        }
+        this.logger.debug('[git] Publish files done', filePaths);
+    }
+
+    publish(author: GitAuthor, filePaths?: string[]): Promise<void> {
+        this.logger.debug('[git] Publish scheduled');
+        return this.worker.schedule(async () => {
+            if (filePaths) {
+                if (!filePaths.length) {
+                    this.logger.debug('[git] Nothing to publish');
+                    return;
+                }
+                return this.publishFiles(author, filePaths);
+            } else {
+                return this.publishAll(author);
+            }
+        });
+    }
+
+    private parseGitCommitAuthor(author?: string) {
+        if (!author) {
+            return author;
+        }
+        const regex = /(.*)\((.*)\)/;
+        const match = author.match(regex);
+        if (match) {
+            const [authorEmail, authorName] = match.slice(1);
+
+            if (authorName === 'Stackbit Code Editor') {
+                return 'stackbit';
+            }
+            return authorEmail ? authorEmail.toLowerCase() : author;
+        }
+        return author;
+    }
+
+    async diff(): Promise<string[]> {
+        this.logger.debug('[git] Diff check scheduled');
+        return this.worker.schedule(async () => {
+            this.logger.debug('[git] Diff check running');
+            const result = await this.run.command(
+                'git',
+                [
+                    'diff',
+                    '--name-only',
+                    '--no-renames', // this flag makes sure we get both old and new name of renamed file
+                    `origin/${this.repoPublishBranch}..${this.repoBranch}`
+                ],
+                this.repoDir
+            );
+            this.logger.debug('[git] Diff check done');
+            return result.stdout.split('\n').filter(Boolean);
+        });
+    }
+
+    async commitLog(): Promise<GitCommitLogEntry[]> {
+        this.logger.debug('[git] Changes check scheduled');
+        return this.worker.schedule(async () => {
+            this.logger.debug('[git] Changes check running');
+            if (!this.branchFetched) {
+                await this.run.command('git', ['fetch', 'origin', `${this.repoPublishBranch}:${this.repoPublishBranch}`], this.repoDir);
+                this.branchFetched = true;
+            }
+            const logResult = await this.run.command(
+                'git',
+                ['log', '--pretty=format:commit:%H%n%at%n%ae%x28%an%x29', '--name-status', `${this.repoPublishBranch}..${this.repoBranch}`],
+                this.repoDir
+            );
+            this.logger.debug('[git] Changes check done');
+            return logResult.stdout
+                .split('commit:')
+                .filter(Boolean)
+                .map((rawCommit) => {
+                    const split = rawCommit.trim().split('\n');
+                    return {
+                        author: this.parseGitCommitAuthor(split[2]),
+                        timestamp: split[1] ? new Date(parseInt(split[1]) * 1000) : undefined,
+                        commitHash: split[0],
+                        changes: split
+                            .slice(3)
+                            .map((line) => line.trim().split(/\t/))
+                            .filter(Boolean)
+                            .filter(([status, filename]) => status && filename)
+                            .map(([status, filename]) => [GIT_LOG_CHANGE_TYPES[status!] || 'modified', filename])
+                    };
+                })
+                .reverse();
+        });
+    }
+}

package/src/services/index.ts

@@ -0,0 +1,2 @@
+export * from './run';
+export * from './git';

package/src/services/run.ts

@@ -0,0 +1,59 @@
+import childProcess, { ChildProcessWithoutNullStreams } from 'child_process';
+import { RunServiceInterface } from '@stackbit/types';
+
+export class RunService implements RunServiceInterface {
+    private readonly env: NodeJS.ProcessEnv;
+    private readonly uid?: number;
+
+    constructor(options: { env: NodeJS.ProcessEnv; uid?: number }) {
+        this.env = options.env;
+        this.uid = options.uid;
+    }
+
+    command(command: string, args: string[], cwd?: string, shell?: boolean) {
+        return getProcessPromise(
+            childProcess.spawn(command, args, {
+                cwd: cwd,
+                shell: shell,
+                env: this.env,
+                ...(this.uid ? { uid: this.uid } : {})
+            })
+        );
+    }
+}
+
+function getProcessPromise(p: ChildProcessWithoutNullStreams): Promise<{
+    stdout: string;
+    stderr: string;
+    exitCode?: number;
+    err?: Error;
+}> {
+    return new Promise((resolve, reject) => {
+        let stdout = '';
+        let stderr = '';
+        p.stdout.on('data', (out) => (stdout += out));
+        p.stderr.on('data', (out) => (stderr += out));
+        p.on('exit', (exitCode) => {
+            if (exitCode !== 0) {
+                reject({
+                    err: new Error(`process exited with code: ${exitCode}, stderr: ${stderr}`),
+                    stdout,
+                    stderr,
+                    exitCode
+                });
+            } else {
+                resolve({
+                    stdout,
+                    stderr
+                });
+            }
+        });
+        p.on('error', (err) => {
+            reject({
+                err,
+                stdout,
+                stderr
+            });
+        });
+    });
+}

package/src/utils/create-update-csi-docs.ts

@@ -111,19 +111,32 @@ export async function createDocumentRecursively({
     if (!model || !csiModel) {
         throw new Error(`no model with name '${modelName}' was found`);
     }
+    const modelFields = model.fields ?? [];
+    const csiModelFields = csiModel.fields ?? [];
     if (model.type === 'page') {
         const tokens = extractTokensFromString(String(model.urlPath));
         const slugField = _.last(tokens);
-        if (object && slugField && slugField in object) {
-            const slugFieldValue = object[slugField];
+        //TODO legacy
+        if (object && slugField && (slugField in object || '_stackbit_slug' in object)) {
+            const slugFieldValue = object[slugField] || object['_stackbit_slug'];
             object[slugField] = sanitizeSlug(slugFieldValue);
+            if (!modelFields.find(field => field.name === slugField)) {
+                modelFields.push({
+                    type: 'slug',
+                    name: slugField
+                });
+                csiModelFields.push({
+                    type: 'slug',
+                    name: slugField
+                });
+            }
         }
     }
 
     const nestedResult = await createObjectRecursively({
         object,
-        modelFields: model.fields ?? [],
-        csiModelFields: csiModel.fields ?? [],
+        modelFields,
+        csiModelFields,
         fieldPath: [modelName],
         modelMap,
         csiModelMap,
@@ -324,7 +337,7 @@ async function createUpdateOperationFieldRecursively({
             return {
                 field: {
                     type: 'image',
-                    value: _.omit(value, ['$$type']) // backwards compatibility with legacy presets
+                    value: _.omit(value, ['$$type'])
                 },
                 newRefDocuments: []
             }

package/src/utils/duplicate-document.ts

@@ -110,16 +110,9 @@ function mergeObjectWithDocumentField({
             }
             break;
         }
-        case 'image': { 
-            if (typeof value !== 'undefined') {
-                return value;
-            }
+        case 'image': {
             const localizedField = getDocumentFieldForLocale(documentField, locale);
             if (localizedField && !localizedField.isUnset && isPlainObjectOrUndefined(value)) {
-                if (localizedField?.sourceData) {
-                    return localizedField?.sourceData;
-                }
-                //TODO needs testing, looks like we need to use the url field instead of this
                 return mergeObjectWithDocumentFields({
                     object: value,
                     documentFields: localizedField.fields,

package/src/utils/store-to-api-docs-converter.ts

@@ -1,25 +1,26 @@
 import _ from 'lodash';
+import path from 'path';
 import { omitByNil } from '@stackbit/utils';
 import * as ContentStoreTypes from '../content-store-types';
 
-export function mapDocumentsToLocalizedApiObjects(documents: ContentStoreTypes.Document[], locale?: string): ContentStoreTypes.APIDocumentObject[] {
-    return documents.map((document) => documentToLocalizedApiObject(document, locale));
+export function mapDocumentsToLocalizedApiObjects(documents: ContentStoreTypes.Document[], staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIDocumentObject[] {
+    return documents.map((document) => documentToLocalizedApiObject(document, staticAssetsPublicPath, locale));
 }
 
-function documentToLocalizedApiObject(document: ContentStoreTypes.Document, locale?: string): ContentStoreTypes.APIDocumentObject {
+function documentToLocalizedApiObject(document: ContentStoreTypes.Document, staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIDocumentObject {
     const { type, fields, ...rest } = document;
     return {
         type: 'object',
         ...rest,
-        fields: toLocalizedAPIFields(fields, locale)
+        fields: toLocalizedAPIFields(fields, staticAssetsPublicPath, locale)
     };
 }
 
-function toLocalizedAPIFields(docFields: Record<string, ContentStoreTypes.DocumentField>, locale?: string): Record<string, ContentStoreTypes.DocumentFieldAPI> {
-    return _.mapValues(docFields, (docField) => toLocalizedAPIField(docField, locale));
+function toLocalizedAPIFields(docFields: Record<string, ContentStoreTypes.DocumentField>, staticAssetsPublicPath: string, locale?: string): Record<string, ContentStoreTypes.DocumentFieldAPI> {
+    return _.mapValues(docFields, (docField) => toLocalizedAPIField(docField, staticAssetsPublicPath, locale));
 }
 
-function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?: string, isListItem = false): ContentStoreTypes.DocumentFieldAPI {
+function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, staticAssetsPublicPath: string, locale?: string, isListItem = false): ContentStoreTypes.DocumentFieldAPI {
     type ToBoolean<T extends boolean | undefined> = T extends true ? true : false;
     function localeFields<T extends boolean | undefined>(localized: T): null | { localized: false } | { localized: true; locale: string } {
         const isLocalized = !!localized as ToBoolean<T>;
@@ -71,33 +72,43 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
                 ...localeFields(docField.localized)
             };
         case 'image':
+            let result: ContentStoreTypes.DocumentFieldAPI;
             if (docField.localized) {
                 const { localized, locales, ...base } = docField;
                 const localeProps = locales && locale ? locales[locale] : undefined;
-                return {
-                    ...base,
-                    ...(localeProps
-                        ? {
-                              ...localeProps,
-                              fields: toLocalizedAPIFields(localeProps.fields, locale) as ContentStoreTypes.ImageFieldsAPI
-                          }
-                        : { isUnset: true }),
-                    ...localeFields(localized)
-                };
+                if (localeProps) {
+                    const fields = toLocalizedAPIFields(localeProps.fields, staticAssetsPublicPath, locale) as ContentStoreTypes.ImageFieldsAPI;
+                    result = {
+                        ...base,
+                        ...localeProps,
+                        fields,
+                        ...localeFields(localized)
+                    };
+                } else {
+                    result = {
+                        ...base,
+                        isUnset: true,
+                        ...localeFields(localized)
+                    }
+                }
+            } else {
+                if (docField.isUnset) {
+                    result = {
+                        ...docField,
+                        type: 'image',
+                        ...localeFields(docField.localized)
+                    }
+                } else {
+                    const fields = toLocalizedAPIFields(docField.fields, staticAssetsPublicPath, locale) as ContentStoreTypes.ImageFieldsAPI;
+                    result = {
+                        ...docField,
+                        type: 'image',
+                        fields,
+                        ...localeFields(docField.localized)
+                    }
+                }
             }
-            return {
-                ...(!docField.isUnset
-                    ? {
-                          ...docField,
-                          type: 'image',
-                          fields: toLocalizedAPIFields(docField.fields, locale) as ContentStoreTypes.ImageFieldsAPI
-                      }
-                    : {
-                          ...docField,
-                          type: 'image'
-                      }),
-                ...localeFields(docField.localized)
-            };
+            return result;
         case 'object':
         case 'model':
             if (docField.localized) {
@@ -109,7 +120,7 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
                         ...(localeProps
                             ? {
                                   ...localeProps,
-                                  fields: toLocalizedAPIFields(localeProps.fields, locale)
+                                  fields: toLocalizedAPIFields(localeProps.fields, staticAssetsPublicPath, locale)
                               }
                             : { isUnset: true }),
                         ...localeFields(localized)
@@ -123,7 +134,7 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
                         ...(localeProps
                             ? {
                                   ...localeProps,
-                                  fields: toLocalizedAPIFields(localeProps.fields, locale)
+                                  fields: toLocalizedAPIFields(localeProps.fields, staticAssetsPublicPath, locale)
                               }
                             : { isUnset: true }),
                         ...localeFields(localized)
@@ -135,7 +146,7 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
                     ? {
                           ...docField,
                           type: 'object',
-                          fields: toLocalizedAPIFields(docField.fields, locale)
+                          fields: toLocalizedAPIFields(docField.fields, staticAssetsPublicPath, locale)
                       }
                     : {
                           ...docField,
@@ -222,14 +233,14 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
                 return {
                     ...base,
                     ...localeProps,
-                    items: (localeProps?.items ?? []).map((field) => toLocalizedAPIField(field, locale, true) as ContentStoreTypes.DocumentListFieldItemsAPI),
+                    items: (localeProps?.items ?? []).map((field) => toLocalizedAPIField(field, staticAssetsPublicPath, locale, true) as ContentStoreTypes.DocumentListFieldItemsAPI),
                     ...localeFields(localized)
                 };
             }
             return {
                 ...docField,
                 ...localeFields(docField.localized),
-                items: docField.items.map((field) => toLocalizedAPIField(field, locale, true) as ContentStoreTypes.DocumentListFieldItemsAPI)
+                items: docField.items.map((field) => toLocalizedAPIField(field, staticAssetsPublicPath, locale, true) as ContentStoreTypes.DocumentListFieldItemsAPI)
             };
         default:
             const _exhaustiveCheck: never = docField;
@@ -238,20 +249,20 @@ function toLocalizedAPIField(docField: ContentStoreTypes.DocumentField, locale?:
     }
 }
 
-export function mapAssetsToLocalizedApiImages(assets: ContentStoreTypes.Asset[], locale?: string): ContentStoreTypes.APIImageObject[] {
-    return assets.map((asset) => assetToLocalizedApiImage(asset, locale));
+export function mapAssetsToLocalizedApiImages(assets: ContentStoreTypes.Asset[], staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIImageObject[] {
+    return assets.map((asset) => assetToLocalizedApiImage(asset, staticAssetsPublicPath, locale));
 }
 
-function assetToLocalizedApiImage(asset: ContentStoreTypes.Asset, locale?: string): ContentStoreTypes.APIImageObject {
+function assetToLocalizedApiImage(asset: ContentStoreTypes.Asset, staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIImageObject {
     const { type, fields, ...rest } = asset;
     return {
         type: 'image',
         ...rest,
-        fields: localizeAssetFields(fields, locale)
+        fields: localizeAssetFields(fields, staticAssetsPublicPath, locale)
     };
 }
 
-function localizeAssetFields(assetFields: ContentStoreTypes.AssetFields, locale?: string): ContentStoreTypes.AssetFieldsAPI {
+function localizeAssetFields(assetFields: ContentStoreTypes.AssetFields, staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.AssetFieldsAPI {
     const fields: ContentStoreTypes.AssetFieldsAPI = {
         title: {
             type: 'string' as const,
@@ -285,11 +296,11 @@ function localizeAssetFields(assetFields: ContentStoreTypes.AssetFields, locale?
     return fields;
 }
 
-export function mapStoreAssetsToAPIAssets(assets: ContentStoreTypes.Asset[], locale?: string): ContentStoreTypes.APIAsset[] {
-    return assets.map((asset) => storeAssetToAPIAsset(asset, locale)).filter((asset): asset is ContentStoreTypes.APIAsset => !!asset);
+export function mapStoreAssetsToAPIAssets(assets: ContentStoreTypes.Asset[], staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIAsset[] {
+    return assets.map((asset) => storeAssetToAPIAsset(asset, staticAssetsPublicPath, locale)).filter((asset): asset is ContentStoreTypes.APIAsset => !!asset);
 }
 
-function storeAssetToAPIAsset(asset: ContentStoreTypes.Asset, locale?: string): ContentStoreTypes.APIAsset | null {
+function storeAssetToAPIAsset(asset: ContentStoreTypes.Asset, staticAssetsPublicPath: string, locale?: string): ContentStoreTypes.APIAsset | null {
     const assetTitleField = asset.fields.title;
     const localizedTitleField = assetTitleField.localized ? assetTitleField.locales?.[locale!] : assetTitleField;
     const assetFileField = asset.fields.file;
@@ -300,7 +311,7 @@ function storeAssetToAPIAsset(asset: ContentStoreTypes.Asset, locale?: string):
     return {
         objectId: asset.srcObjectId,
         createdAt: asset.createdAt,
-        url: localizedFileField.url,
+        url: replaceAssetUrlIfNeeded(staticAssetsPublicPath, localizedFileField.url) ?? staticAssetsPublicPath,
         ...omitByNil({
             title: localizedTitleField?.value ?? null,
             fileName: localizedFileField.fileName,
@@ -311,3 +322,11 @@ function storeAssetToAPIAsset(asset: ContentStoreTypes.Asset, locale?: string):
         })
     };
 }
+
+function replaceAssetUrlIfNeeded(staticAssetsPublicPath: string, value: string | undefined) {
+    let url = value;
+    if (url && !url.startsWith('http:') && !url.startsWith('https:')) {
+        url = path.join(staticAssetsPublicPath, url);
+    }
+    return url;
+}

package/src/utils/store-to-csi-docs-converter.ts

@@ -95,8 +95,6 @@ function mapStoreFieldToCSIField(documentField: ContentStoreTypes.DocumentField)
                 return {
                     type: 'image',
                     localized: true,
-                    source: documentField.source,
-                    sourceData: documentField.sourceData,
                     locales: _.mapValues(documentField.locales, (locale) => ({
                         locale: locale.locale,
                         fields: mapStoreFieldsToCSIFields(locale.fields)
@@ -108,8 +106,6 @@ function mapStoreFieldToCSIField(documentField: ContentStoreTypes.DocumentField)
             }
             return {
                 type: 'image',
-                source: documentField.source,
-                sourceData: documentField.sourceData,
                 fields: mapStoreFieldsToCSIFields(documentField.fields)
             } as CSITypes.DocumentImageFieldNonLocalized;
         }