decap-server 1.4.0-beta.0

package/src/middlewares/localGit/index.ts

@@ -0,0 +1,463 @@
+import path from 'path';
+import { promises as fs } from 'fs';
+import {
+  branchFromContentKey,
+  generateContentKey,
+  contentKeyFromBranch,
+  CMS_BRANCH_PREFIX,
+  statusToLabel,
+  labelToStatus,
+  parseContentKey,
+} from 'decap-cms-lib-util/src/APIUtils';
+import { parse } from 'what-the-diff';
+// eslint-disable-next-line import/no-named-as-default
+import simpleGit from 'simple-git';
+import { Mutex, withTimeout } from 'async-mutex';
+
+import { defaultSchema, joi } from '../joi';
+import { pathTraversal } from '../joi/customValidators';
+import { listRepoFiles, writeFile, move, deleteFile, getUpdateDate } from '../utils/fs';
+import { entriesFromFiles, readMediaFile } from '../utils/entries';
+
+import type {
+  EntriesByFolderParams,
+  EntriesByFilesParams,
+  GetEntryParams,
+  DefaultParams,
+  UnpublishedEntryParams,
+  PersistEntryParams,
+  GetMediaParams,
+  Asset,
+  PublishUnpublishedEntryParams,
+  PersistMediaParams,
+  DeleteFileParams,
+  UpdateUnpublishedEntryStatusParams,
+  DataFile,
+  GetMediaFileParams,
+  DeleteEntryParams,
+  DeleteFilesParams,
+  UnpublishedEntryDataFileParams,
+  UnpublishedEntryMediaFileParams,
+} from '../types';
+import type express from 'express';
+import type winston from 'winston';
+import type { SimpleGit } from 'simple-git';
+
+async function commit(git: SimpleGit, commitMessage: string) {
+  await git.add('.');
+  await git.commit(commitMessage, undefined, {
+    // setting the value to a string passes name=value
+    // any other value passes just the key
+    '--no-verify': null,
+    '--no-gpg-sign': null,
+  });
+}
+
+async function getCurrentBranch(git: SimpleGit) {
+  const currentBranch = await git.branchLocal().then(summary => summary.current);
+  return currentBranch;
+}
+
+async function runOnBranch<T>(git: SimpleGit, branch: string, func: () => Promise<T>) {
+  const currentBranch = await getCurrentBranch(git);
+  try {
+    if (currentBranch !== branch) {
+      await git.checkout(branch);
+    }
+    const result = await func();
+    return result;
+  } finally {
+    await git.checkout(currentBranch);
+  }
+}
+
+function branchDescription(branch: string) {
+  return `branch.${branch}.description`;
+}
+
+type GitOptions = {
+  repoPath: string;
+  logger: winston.Logger;
+};
+
+async function commitEntry(
+  git: SimpleGit,
+  repoPath: string,
+  dataFiles: DataFile[],
+  assets: Asset[],
+  commitMessage: string,
+) {
+  // save entry content
+  await Promise.all(
+    dataFiles.map(dataFile => writeFile(path.join(repoPath, dataFile.path), dataFile.raw)),
+  );
+  // save assets
+  await Promise.all(
+    assets.map(a => writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding))),
+  );
+  if (dataFiles.every(dataFile => dataFile.newPath)) {
+    dataFiles.forEach(async dataFile => {
+      await move(path.join(repoPath, dataFile.path), path.join(repoPath, dataFile.newPath!));
+    });
+  }
+
+  // commits files
+  await commit(git, commitMessage);
+}
+
+async function rebase(git: SimpleGit, branch: string) {
+  const gpgSign = await git.raw(['config', 'commit.gpgsign']);
+  try {
+    if (gpgSign === 'true') {
+      await git.addConfig('commit.gpgsign', 'false');
+    }
+    await git.rebase([branch, '--no-verify']);
+  } finally {
+    if (gpgSign === 'true') {
+      await git.addConfig('commit.gpgsign', gpgSign);
+    }
+  }
+}
+
+async function merge(git: SimpleGit, from: string, to: string) {
+  const gpgSign = await git.raw(['config', 'commit.gpgsign']);
+  try {
+    if (gpgSign === 'true') {
+      await git.addConfig('commit.gpgsign', 'false');
+    }
+    await git.mergeFromTo(from, to);
+  } finally {
+    if (gpgSign === 'true') {
+      await git.addConfig('commit.gpgsign', gpgSign);
+    }
+  }
+}
+
+async function isBranchExists(git: SimpleGit, branch: string) {
+  const branchExists = await git.branchLocal().then(({ all }) => all.includes(branch));
+  return branchExists;
+}
+
+async function getDiffs(git: SimpleGit, source: string, dest: string) {
+  const rawDiff = await git.diff([source, dest]);
+  const diffs = parse(rawDiff).map(d => {
+    const oldPath = d.oldPath?.replace(/b\//, '') || '';
+    const newPath = d.newPath?.replace(/b\//, '') || '';
+    const path = newPath || (oldPath as string);
+    return {
+      oldPath,
+      newPath,
+      status: d.status,
+      newFile: d.status === 'added',
+      path,
+      id: path,
+      binary: d.binary || /.svg$/.test(path),
+    };
+  });
+  return diffs;
+}
+
+export async function validateRepo({ repoPath }: { repoPath: string }) {
+  const git = simpleGit(repoPath);
+  const isRepo = await git.checkIsRepo();
+  if (!isRepo) {
+    throw Error(`${repoPath} is not a valid git repository`);
+  }
+}
+
+export function getSchema({ repoPath }: { repoPath: string }) {
+  const schema = defaultSchema({ path: pathTraversal(repoPath) });
+  return schema;
+}
+
+export function localGitMiddleware({ repoPath, logger }: GitOptions) {
+  const git = simpleGit(repoPath);
+
+  // we can only perform a single git operation at any given time
+  const mutex = withTimeout(new Mutex(), 3000, new Error('Request timed out'));
+
+  return async function (req: express.Request, res: express.Response) {
+    let release;
+    try {
+      release = await mutex.acquire();
+      const { body } = req;
+      if (body.action === 'info') {
+        res.json({
+          repo: path.basename(repoPath),
+          publish_modes: ['simple', 'editorial_workflow'],
+          type: 'local_git',
+        });
+        return;
+      }
+      const { branch } = body.params as DefaultParams;
+
+      const branchExists = await isBranchExists(git, branch);
+      if (!branchExists) {
+        const message = `Default branch '${branch}' doesn't exist`;
+        res.status(422).json({ error: message });
+        return;
+      }
+
+      switch (body.action) {
+        case 'entriesByFolder': {
+          const payload = body.params as EntriesByFolderParams;
+          const { folder, extension, depth } = payload;
+          const entries = await runOnBranch(git, branch, () =>
+            listRepoFiles(repoPath, folder, extension, depth).then(files =>
+              entriesFromFiles(
+                repoPath,
+                files.map(file => ({ path: file })),
+              ),
+            ),
+          );
+          res.json(entries);
+          break;
+        }
+        case 'entriesByFiles': {
+          const payload = body.params as EntriesByFilesParams;
+          const entries = await runOnBranch(git, branch, () =>
+            entriesFromFiles(repoPath, payload.files),
+          );
+          res.json(entries);
+          break;
+        }
+        case 'getEntry': {
+          const payload = body.params as GetEntryParams;
+          const [entry] = await runOnBranch(git, branch, () =>
+            entriesFromFiles(repoPath, [{ path: payload.path }]),
+          );
+          res.json(entry);
+          break;
+        }
+        case 'unpublishedEntries': {
+          const cmsBranches = await git
+            .branchLocal()
+            .then(result => result.all.filter(b => b.startsWith(`${CMS_BRANCH_PREFIX}/`)));
+          res.json(cmsBranches.map(contentKeyFromBranch));
+          break;
+        }
+        case 'unpublishedEntry': {
+          let { id, collection, slug, cmsLabelPrefix } = body.params as UnpublishedEntryParams;
+          if (id) {
+            ({ collection, slug } = parseContentKey(id));
+          }
+          const contentKey = generateContentKey(collection as string, slug as string);
+          const cmsBranch = branchFromContentKey(contentKey);
+          const branchExists = await isBranchExists(git, cmsBranch);
+          if (branchExists) {
+            const diffs = await getDiffs(git, branch, cmsBranch);
+            const label = await git.raw(['config', branchDescription(cmsBranch)]);
+            const status = label && labelToStatus(label.trim(), cmsLabelPrefix || '');
+            const updatedAt =
+              diffs.length >= 0
+                ? await runOnBranch(git, cmsBranch, async () => {
+                    const dates = await Promise.all(
+                      diffs.map(({ newPath }) => getUpdateDate(repoPath, newPath)),
+                    );
+                    return dates.reduce((a, b) => {
+                      return a > b ? a : b;
+                    });
+                  })
+                : new Date();
+            const unpublishedEntry = {
+              collection,
+              slug,
+              status,
+              diffs,
+              updatedAt,
+            };
+            res.json(unpublishedEntry);
+          } else {
+            return res.status(404).json({ message: 'Not Found' });
+          }
+          break;
+        }
+        case 'unpublishedEntryDataFile': {
+          const { path, collection, slug } = body.params as UnpublishedEntryDataFileParams;
+          const contentKey = generateContentKey(collection as string, slug as string);
+          const cmsBranch = branchFromContentKey(contentKey);
+          const [entry] = await runOnBranch(git, cmsBranch, () =>
+            entriesFromFiles(repoPath, [{ path }]),
+          );
+          res.json({ data: entry.data });
+          break;
+        }
+        case 'unpublishedEntryMediaFile': {
+          const { path, collection, slug } = body.params as UnpublishedEntryMediaFileParams;
+          const contentKey = generateContentKey(collection as string, slug as string);
+          const cmsBranch = branchFromContentKey(contentKey);
+          const file = await runOnBranch(git, cmsBranch, () => readMediaFile(repoPath, path));
+          res.json(file);
+          break;
+        }
+        case 'deleteUnpublishedEntry': {
+          const { collection, slug } = body.params as DeleteEntryParams;
+          const contentKey = generateContentKey(collection, slug);
+          const cmsBranch = branchFromContentKey(contentKey);
+          const currentBranch = await getCurrentBranch(git);
+          if (currentBranch === cmsBranch) {
+            await git.checkoutLocalBranch(branch);
+          }
+          await git.branch(['-D', cmsBranch]);
+          res.json({ message: `deleted branch: ${cmsBranch}` });
+          break;
+        }
+        case 'persistEntry': {
+          const {
+            cmsLabelPrefix,
+            entry,
+            dataFiles = [entry as DataFile],
+            assets,
+            options,
+          } = body.params as PersistEntryParams;
+
+          if (!options.useWorkflow) {
+            await runOnBranch(git, branch, async () => {
+              await commitEntry(git, repoPath, dataFiles, assets, options.commitMessage);
+            });
+          } else {
+            const slug = dataFiles[0].slug;
+            const collection = options.collectionName as string;
+            const contentKey = generateContentKey(collection, slug);
+            const cmsBranch = branchFromContentKey(contentKey);
+            await runOnBranch(git, branch, async () => {
+              const branchExists = await isBranchExists(git, cmsBranch);
+              if (branchExists) {
+                await git.checkout(cmsBranch);
+              } else {
+                await git.checkoutLocalBranch(cmsBranch);
+              }
+              await rebase(git, branch);
+              const diffs = await getDiffs(git, branch, cmsBranch);
+              // delete media files that have been removed from the entry
+              const toDelete = diffs.filter(
+                d => d.binary && !assets.map(a => a.path).includes(d.path),
+              );
+              await Promise.all(toDelete.map(f => fs.unlink(path.join(repoPath, f.path))));
+              await commitEntry(git, repoPath, dataFiles, assets, options.commitMessage);
+
+              // add status for new entries
+              if (!branchExists) {
+                const description = statusToLabel(options.status, cmsLabelPrefix || '');
+                await git.addConfig(branchDescription(cmsBranch), description);
+              }
+            });
+          }
+          res.json({ message: 'entry persisted' });
+          break;
+        }
+        case 'updateUnpublishedEntryStatus': {
+          const { collection, slug, newStatus, cmsLabelPrefix } =
+            body.params as UpdateUnpublishedEntryStatusParams;
+          const contentKey = generateContentKey(collection, slug);
+          const cmsBranch = branchFromContentKey(contentKey);
+          const description = statusToLabel(newStatus, cmsLabelPrefix || '');
+          await git.addConfig(branchDescription(cmsBranch), description);
+          res.json({ message: `${branch} description was updated to ${description}` });
+          break;
+        }
+        case 'publishUnpublishedEntry': {
+          const { collection, slug } = body.params as PublishUnpublishedEntryParams;
+          const contentKey = generateContentKey(collection, slug);
+          const cmsBranch = branchFromContentKey(contentKey);
+          await merge(git, cmsBranch, branch);
+          await git.deleteLocalBranch(cmsBranch);
+          res.json({ message: `branch ${cmsBranch} merged to ${branch}` });
+          break;
+        }
+        case 'getMedia': {
+          const { mediaFolder } = body.params as GetMediaParams;
+          const mediaFiles = await runOnBranch(git, branch, async () => {
+            const files = await listRepoFiles(repoPath, mediaFolder, '', 1);
+            const serializedFiles = await Promise.all(
+              files.map(file => readMediaFile(repoPath, file)),
+            );
+            return serializedFiles;
+          });
+          res.json(mediaFiles);
+          break;
+        }
+        case 'getMediaFile': {
+          const { path } = body.params as GetMediaFileParams;
+          const mediaFile = await runOnBranch(git, branch, () => {
+            return readMediaFile(repoPath, path);
+          });
+          res.json(mediaFile);
+          break;
+        }
+        case 'persistMedia': {
+          const {
+            asset,
+            options: { commitMessage },
+          } = body.params as PersistMediaParams;
+
+          const file = await runOnBranch(git, branch, async () => {
+            await writeFile(
+              path.join(repoPath, asset.path),
+              Buffer.from(asset.content, asset.encoding),
+            );
+            await commit(git, commitMessage);
+            return readMediaFile(repoPath, asset.path);
+          });
+          res.json(file);
+          break;
+        }
+        case 'deleteFile': {
+          const {
+            path: filePath,
+            options: { commitMessage },
+          } = body.params as DeleteFileParams;
+          await runOnBranch(git, branch, async () => {
+            await deleteFile(repoPath, filePath);
+            await commit(git, commitMessage);
+          });
+          res.json({ message: `deleted file ${filePath}` });
+          break;
+        }
+        case 'deleteFiles': {
+          const {
+            paths,
+            options: { commitMessage },
+          } = body.params as DeleteFilesParams;
+          await runOnBranch(git, branch, async () => {
+            await Promise.all(paths.map(filePath => deleteFile(repoPath, filePath)));
+            await commit(git, commitMessage);
+          });
+          res.json({ message: `deleted files ${paths.join(', ')}` });
+          break;
+        }
+        case 'getDeployPreview': {
+          res.json(null);
+          break;
+        }
+        default: {
+          const message = `Unknown action ${body.action}`;
+          res.status(422).json({ error: message });
+          break;
+        }
+      }
+    } catch (e) {
+      logger.error(
+        `Error handling ${JSON.stringify(req.body)}: ${
+          e instanceof Error ? e.message : 'Unknown error'
+        }`,
+      );
+      res.status(500).json({ error: 'Unknown error' });
+    } finally {
+      release && release();
+    }
+  };
+}
+
+type Options = {
+  logger: winston.Logger;
+};
+
+export async function registerMiddleware(app: express.Express, options: Options) {
+  const { logger } = options;
+  const repoPath = path.resolve(process.env.GIT_REPO_DIRECTORY || process.cwd());
+  await validateRepo({ repoPath });
+  app.post('/api/v1', joi(getSchema({ repoPath })));
+  app.post('/api/v1', localGitMiddleware({ repoPath, logger }));
+  logger.info(`Decap CMS Git Proxy Server configured with ${repoPath}`);
+}

package/src/middlewares/types.ts

@@ -0,0 +1,101 @@
+export type DefaultParams = {
+  branch: string;
+};
+
+export type EntriesByFolderParams = {
+  folder: string;
+  extension: string;
+  depth: 1;
+};
+
+export type EntriesByFilesParams = {
+  files: { path: string }[];
+};
+
+export type GetEntryParams = {
+  path: string;
+};
+
+export type UnpublishedEntryParams = {
+  id?: string;
+  collection?: string;
+  slug?: string;
+  cmsLabelPrefix?: string;
+};
+
+export type UnpublishedEntryDataFileParams = {
+  collection: string;
+  slug: string;
+  id: string;
+  path: string;
+};
+
+export type UnpublishedEntryMediaFileParams = {
+  collection: string;
+  slug: string;
+  id: string;
+  path: string;
+};
+
+export type DeleteEntryParams = {
+  collection: string;
+  slug: string;
+};
+
+export type UpdateUnpublishedEntryStatusParams = {
+  collection: string;
+  slug: string;
+  newStatus: string;
+  cmsLabelPrefix?: string;
+};
+
+export type PublishUnpublishedEntryParams = {
+  collection: string;
+  slug: string;
+};
+
+export type DataFile = { slug: string; path: string; raw: string; newPath?: string };
+
+export type Asset = { path: string; content: string; encoding: 'base64' };
+
+export type PersistEntryParams = {
+  cmsLabelPrefix?: string;
+  entry?: DataFile;
+  dataFiles?: DataFile[];
+  assets: Asset[];
+  options: {
+    collectionName?: string;
+    commitMessage: string;
+    useWorkflow: boolean;
+    status: string;
+  };
+};
+
+export type GetMediaParams = {
+  mediaFolder: string;
+};
+
+export type GetMediaFileParams = {
+  path: string;
+};
+
+export type PersistMediaParams = {
+  asset: Asset;
+  options: {
+    commitMessage: string;
+  };
+};
+
+export type DeleteFileParams = {
+  path: string;
+  options: {
+    commitMessage: string;
+  };
+};
+
+export type DeleteFilesParams = {
+  paths: string[];
+  options: {
+    commitMessage: string;
+  };
+};

package/src/middlewares/utils/entries.ts

@@ -0,0 +1,48 @@
+import crypto from 'crypto';
+import path from 'path';
+import { promises as fs } from 'fs';
+
+function sha256(buffer: Buffer) {
+  return crypto.createHash('sha256').update(buffer).digest('hex');
+}
+
+// normalize windows os path format
+function normalizePath(path: string) {
+  return path.replace(/\\/g, '/');
+}
+
+export async function entriesFromFiles(
+  repoPath: string,
+  files: { path: string; label?: string }[],
+) {
+  return Promise.all(
+    files.map(async file => {
+      try {
+        const content = await fs.readFile(path.join(repoPath, file.path));
+        return {
+          data: content.toString(),
+          file: { path: normalizePath(file.path), label: file.label, id: sha256(content) },
+        };
+      } catch (e) {
+        return {
+          data: null,
+          file: { path: normalizePath(file.path), label: file.label, id: null },
+        };
+      }
+    }),
+  );
+}
+
+export async function readMediaFile(repoPath: string, file: string) {
+  const encoding = 'base64';
+  const buffer = await fs.readFile(path.join(repoPath, file));
+  const id = sha256(buffer);
+
+  return {
+    id,
+    content: buffer.toString(encoding),
+    encoding,
+    path: normalizePath(file),
+    name: path.basename(file),
+  };
+}

package/src/middlewares/utils/fs.ts

@@ -0,0 +1,65 @@
+import path from 'path';
+import { promises as fs } from 'fs';
+
+async function listFiles(dir: string, extension: string, depth: number): Promise<string[]> {
+  if (depth <= 0) {
+    return [];
+  }
+
+  try {
+    const dirents = await fs.readdir(dir, { withFileTypes: true });
+    const files = await Promise.all(
+      dirents.map(dirent => {
+        const res = path.join(dir, dirent.name);
+        return dirent.isDirectory()
+          ? listFiles(res, extension, depth - 1)
+          : [res].filter(f => f.endsWith(extension));
+      }),
+    );
+    return ([] as string[]).concat(...files);
+  } catch (e) {
+    return [];
+  }
+}
+
+export async function listRepoFiles(
+  repoPath: string,
+  folder: string,
+  extension: string,
+  depth: number,
+) {
+  const files = await listFiles(path.join(repoPath, folder), extension, depth);
+  return files.map(f => f.slice(repoPath.length + 1));
+}
+
+export async function writeFile(filePath: string, content: Buffer | string) {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, content);
+}
+
+export async function deleteFile(repoPath: string, filePath: string) {
+  await fs.unlink(path.join(repoPath, filePath)).catch(() => undefined);
+}
+
+async function moveFile(from: string, to: string) {
+  await fs.mkdir(path.dirname(to), { recursive: true });
+  await fs.rename(from, to);
+}
+
+export async function move(from: string, to: string) {
+  // move file
+  await moveFile(from, to);
+
+  // move children
+  const sourceDir = path.dirname(from);
+  const destDir = path.dirname(to);
+  const allFiles = await listFiles(sourceDir, '', 100);
+  await Promise.all(allFiles.map(file => moveFile(file, file.replace(sourceDir, destDir))));
+}
+
+export async function getUpdateDate(repoPath: string, filePath: string) {
+  return fs
+    .stat(path.join(repoPath, filePath))
+    .then(stat => stat.mtime)
+    .catch(() => new Date());
+}

package/src/middlewares.ts

@@ -0,0 +1,28 @@
+import { registerCommonMiddlewares } from './middlewares/common';
+import { registerMiddleware as localGit } from './middlewares/localGit';
+import { registerMiddleware as localFs } from './middlewares/localFs';
+import { createLogger } from './logger';
+
+import type express from 'express';
+
+type Options = {
+  logLevel?: string;
+};
+
+function createOptions(options: Options) {
+  return {
+    logger: createLogger({ level: options.logLevel || 'info' }),
+  };
+}
+
+export async function registerLocalGit(app: express.Express, options: Options = {}) {
+  const opts = createOptions(options);
+  registerCommonMiddlewares(app, opts);
+  await localGit(app, opts);
+}
+
+export async function registerLocalFs(app: express.Express, options: Options = {}) {
+  const opts = createOptions(options);
+  registerCommonMiddlewares(app, opts);
+  await localFs(app, opts);
+}