@stackbit/dev 0.0.3

package/src/index.ts

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+
+import yargs from 'yargs';
+
+yargs(process.argv.slice(2))
+    .command(
+        '$0',
+        'stackbit-dev',
+        (yargs) =>
+            yargs
+                .option('port', {
+                    alias: 'p',
+                    description: 'port',
+                    default: 3000
+                })
+                .options('host', {
+                    alias: 'h',
+                    description: 'hostname',
+                    default: 'localhost'
+                })
+                .options('dir', {
+                    alias: 'd',
+                    description: 'root dir',
+                    default: process.cwd()
+                })
+                .options('runnable-dir', {
+                    description: 'runnable dir',
+                    default: ''
+                })
+                .options('log-level', {
+                    description: 'log level',
+                    default: 'info'
+                })
+                .options('help', {}),
+        async (argv) => {
+            if (argv['log-level']) {
+                process.env.LOG_LEVEL = argv['log-level'];
+            }
+            return require('./dev').start({
+                ssgPort: argv.port,
+                ssgHost: argv.host,
+                rootDir: argv.dir,
+                runnableDir: argv['runnable-dir']
+            });
+        }
+    )
+    .demandCommand()
+    .parse();

package/src/runner/index.ts

@@ -0,0 +1,162 @@
+import _ from 'lodash';
+import path from 'path';
+import ssg from './ssg';
+import Editor from '@stackbit/dev-common/dist/runner/editor';
+import GitCMS from '@stackbit/dev-common/dist/cms/git';
+import Worker from '@stackbit/dev-common/dist/utils/worker';
+import { StackbitYaml } from '@stackbit/dev-common/dist/services/stackbit-yaml';
+import { watchDir } from '../services/file-watcher';
+
+import logger from '../services/logger';
+import consts from '../consts';
+
+export default class Runner {
+    private editor: any;
+    private cms: any;
+    private stackbitYaml: any;
+    private workers: any;
+    private watcher: any;
+
+    private rootDir: string;
+    private appDir?: string;
+
+    constructor({ rootDir, runnableDir }: { rootDir: string; runnableDir?: string }) {
+        this.rootDir = rootDir;
+        this.appDir = runnableDir ? path.join(this.rootDir, runnableDir) : this.rootDir;
+        this.workers = {
+            gitApp: new Worker()
+        };
+    }
+
+    async install() {
+        this.stackbitYaml = new StackbitYaml({
+            repoDir: this.rootDir,
+            appDir: this.appDir,
+            extendLogicFields: null,
+            onNodeVersionUpdated: null,
+            logger,
+            userLogger: logger,
+            envName: 'local' //TODO
+        });
+        await this.stackbitYaml.load();
+        this.cms = new GitCMS({
+            rootDir: this.rootDir,
+            repoDir: this.rootDir,
+            appDir: this.appDir,
+            stackbitYaml: this.stackbitYaml,
+            workers: this.workers,
+            assetIdPrefix: consts.ASSET_ID_PREFIX,
+            staticAssetsFilePath: consts.STATIC_ASSETS_FILE_PATH,
+            staticAssetsPublicPath: consts.STATIC_ASSETS_PUBLIC_PATH,
+            staticThemeAssetsFilePath: consts.STATIC_THEME_ASSETS_FILE_PATH,
+            staticThemeAssetsPublicPath: consts.STATIC_THEME_ASSETS_PUBLIC_PATH,
+            defaultContentTypeExtensions: consts.DEFAULT_CONTENT_TYPE_EXTENSIONS,
+            logger,
+            userLogger: logger
+        });
+        await this.cms.run();
+        this.editor = new Editor(null, this.cms, this.stackbitYaml, {
+            logger,
+            stackbitUrlPathField: consts.STACKBIT_URL_PATH_FIELD
+        });
+        this.watcher = watchDir(this.rootDir, (filePaths) => {
+            logger.debug(`File change detected: ${filePaths}`);
+            this.handleFileChanges(filePaths);
+        });
+    }
+
+    handleFileChanges(filePaths: string[]) {
+        this.cms.postPull(filePaths.map((filePath) => path.join(this.rootDir, filePath))).catch((err: any) => {
+            logger.debug('Error in postPull', { err });
+        });
+        for (const filePath of filePaths) {
+            if (!StackbitYaml.isStackbitYamlFile(filePath)) {
+                continue;
+            }
+            this.stackbitYaml.load().catch((err: any) => {
+                logger.debug('Error handling stackbit.yaml change', { err });
+                logger.error('Error reloading stackbit.yaml');
+            });
+            break;
+        }
+    }
+
+    getDirectPaths() {
+        return ssg.directPaths;
+    }
+
+    /**
+     * Function retuning a map or a function that rewrites direct proxy requests.
+     * https://github.com/chimurai/http-proxy-middleware#http-proxy-middleware-options
+     * @return {Function|Object}
+     */
+    getDirectRoutes(): { [hostOrPath: string]: string } {
+        return ssg.directRoutes;
+    }
+
+    getDirectChangeOrigin() {
+        return ssg.directChangeOrigin;
+    }
+
+    getDirectChangeSocketOrigin() {
+        return ssg.directChangeSocketOrigin;
+    }
+
+    async getObjects(objectIds = []) {
+        return this.editor.getObjects(objectIds);
+    }
+
+    async getObjectsWithAnnotations(annotationTree = null, clientAnnotationErrors = []) {
+        const result = this.editor.getObjectsWithAnnotations(annotationTree);
+        _.forEach([...(result.errors ?? []), ...clientAnnotationErrors], (error) => {
+            const errorProps = _.omit(error, ['type', 'message']);
+            logger.error(`${error.message} ${_.map(errorProps, (val, key) => `${key}:'${val && val.toString ? val.toString() : val}'`).join(', ')}`);
+        });
+        return result;
+    }
+
+    async getCollections() {
+        return this.editor.getCollections();
+    }
+
+    async getSiteMap() {
+        return this.editor.getSiteMap(this.cms.getEncodingResult());
+    }
+
+    async getSchema() {
+        return this.editor.getSchema();
+    }
+
+    async getUrl(objectId: string, projectId: string, locale: string): Promise<any> {
+        return this.editor.getUrl(objectId, projectId, locale);
+    }
+
+    async getObject(objectId: string, projectId: string): Promise<any> {
+        return this.cms.getObject(objectId, projectId);
+    }
+
+    createObject(modelName: string, filePath: any, object: string, user: any): Promise<any> {
+        return this.cms.createObject(modelName, filePath, object, user);
+    }
+
+    deleteObject(objectId: string, user: any): Promise<any> {
+        return this.cms.deleteObject(objectId, user);
+    }
+
+    updateObject(objectId: string, projectId: string, object: any, user: any): Promise<any> {
+        return this.getObject(objectId, projectId).then((existingObject) => {
+            if (!existingObject) {
+                throw new Error('Object not found');
+            }
+            return this.cms.updateObject(existingObject, object, user);
+        });
+    }
+
+    async listAssets(filterParams: any): Promise<any> {
+        return this.cms.listAssets(filterParams);
+    }
+
+    async uploadAsset(url: string, filename: string, user: any): Promise<any> {
+        return this.cms.uploadAsset(url, filename, user);
+    }
+}

package/src/runner/ssg.ts

@@ -0,0 +1,10 @@
+import consts from '../consts';
+
+export default {
+    directPaths: ['/_next/**', '/nextjs-live-updates/**'],
+    directRoutes: {
+        '/socket.io': `http://localhost:${consts.NEXTJS_SOURCEBIT_LIVE_UPDATE_PORT}`
+    },
+    directChangeSocketOrigin: false,
+    directChangeOrigin: true
+};

package/src/server/index.ts

@@ -0,0 +1,65 @@
+import _, { reject } from 'lodash';
+import http from 'http';
+import express from 'express';
+import path from 'path';
+
+import logger from '../services/logger';
+
+import routes from './routes';
+import proxy from './proxy';
+import Runner from '../runner';
+import consts from '../consts';
+import socketService from '@stackbit/dev-common/dist/services/socket-service';
+
+export async function start({
+    serverPort,
+    ssgPort,
+    ssgHost,
+    rootDir,
+    runnableDir
+}: {
+    serverPort: number;
+    ssgPort: number;
+    ssgHost: string;
+    rootDir: string;
+    runnableDir?: string;
+}) {
+    const server = express();
+    const httpServer = http.createServer(server);
+    socketService.use(httpServer, logger);
+
+    const runner = new Runner({
+        rootDir,
+        runnableDir
+    });
+    await runner.install();
+
+    server.use(require('cors')());
+    // if (config.setContentSecurityPolicy) {
+    //     server.use((req, res, next) => {
+    //         const hosts = getClientAllowedHosts(req).join(' ');
+    //         res.set('Content-Security-Policy', `frame-ancestors 'self' ${hosts}`);
+    //         next();
+    //     });
+    // }
+
+    // log requests
+    server.use((req, res, next) => {
+        logger.debug(`got request ${req.method} ${req.path}`, { method: req.method, originalUrl: req.originalUrl, url: req.url, path: req.path });
+        next();
+    });
+
+    server.use('/_static', express.static('static'));
+    server.use(consts.STATIC_ASSETS_PUBLIC_PATH, express.static(path.join(rootDir, consts.STATIC_ASSETS_FILE_PATH)));
+    server.use(consts.STATIC_THEME_ASSETS_PUBLIC_PATH, express.static(path.join(rootDir, consts.STATIC_THEME_ASSETS_FILE_PATH)));
+
+    routes(server, runner);
+    server.use(proxy(ssgHost, ssgPort, runner));
+
+    return new Promise<void>((resolve, reject) => {
+        httpServer.listen(serverPort, () => {
+            logger.debug('Server running on port ' + serverPort);
+            resolve();
+        });
+    });
+}

package/src/server/proxy.ts

@@ -0,0 +1,85 @@
+import _ from 'lodash';
+import path from 'path';
+import express, { Request } from 'express';
+import proxy from 'http-proxy-middleware';
+import logger from '../services/logger';
+import consts from '../consts';
+import { createProxyModifyResponseMiddleware, isHtml, injectScript } from '@stackbit/dev-common/dist/utils/proxy-utils';
+import { renderLoadingPage } from '@stackbit/dev-common/dist/utils/pages';
+import config from '../config';
+import Runner from '../runner';
+
+export default function proxyMiddleware(ssgHost: string, ssgPort: number, runner: Runner) {
+    const router = express.Router();
+
+    router.use((req, res, next) => {
+        _.unset(req, 'headers.x-forwarded-proto');
+        const rawHeaders = _.get(req, 'rawHeaders');
+        const headerIndex = _.findIndex(rawHeaders, (headerValue) => headerValue.toLowerCase() === 'x-forwarded-proto');
+        if (headerIndex > -1) {
+            // remove the header key and value
+            rawHeaders.splice(headerIndex, 2);
+        }
+        next();
+    });
+
+    const proxyTargetUrl = `http://${ssgHost}:${ssgPort}`;
+
+    const handlerProxyError = (err: any, req: any, res: any) => {
+        if (err?.code === 'ECONNREFUSED' && res?.status) {
+            renderLoadingPage(req, res, {
+                assetsDir: path.join(__dirname, '../assets'),
+                proxyUrl: proxyTargetUrl,
+                proxyPath: req?.path || '',
+                htmlMessage: '',
+                status: 504,
+                statusMessage: 'aa'
+            });
+        }
+    };
+
+    // visit https://github.com/chimurai/http-proxy-middleware#context-matching
+    // for docs on how path matching works, it is different from express matching
+    const directPaths = _.uniq(_.concat(consts.DEFAULT_DIRECT_PROXY_PATH, runner.getDirectPaths()));
+    router.use(
+        proxy(directPaths, {
+            target: proxyTargetUrl,
+            changeOrigin: runner.getDirectChangeOrigin(),
+            ws: true,
+            logProvider: () => logger,
+            logLevel: config.logLevel === 'debug' ? 'info' : 'silent',
+            router: runner.getDirectRoutes() ?? [],
+            onError: (err: any, req: any, res: any) => {
+                logger.debug(`onError in direct proxy, request: ${req.method} ${req.url}, error: ${err.message}`, { error: err });
+                handlerProxyError(err, req, res);
+            }
+        })
+    );
+
+    router.use(
+        proxy({
+            target: proxyTargetUrl,
+            changeOrigin: true,
+            followRedirects: true,
+            logProvider: () => logger,
+            logLevel: config.logLevel === 'debug' ? 'info' : 'silent',
+            onError: (err: any, req: any, res: any) => {
+                logger.debug(`onError in main proxy, request: ${req.method} ${req.url}, error: ${err.message}`, { error: err });
+                handlerProxyError(err, req, res);
+            },
+            selfHandleResponse: true,
+            onProxyRes: createProxyModifyResponseMiddleware({
+                logTagName: 'proxy',
+                shouldModifyBody: (proxyRes: Request, req: Request) => {
+                    return req.method === 'GET' && isHtml(proxyRes);
+                },
+                modifyBody: (body: string) => {
+                    body = injectScript(body, config.snippetUrl);
+                    return body;
+                }
+            })
+        })
+    );
+
+    return router;
+}

package/src/server/routes.ts

@@ -0,0 +1,186 @@
+import _ from 'lodash';
+import { Request, Response, NextFunction, Router } from 'express';
+import bodyParser from 'body-parser';
+import Runner from '../runner';
+import logger from '../services/logger';
+import { PageNotFoundError } from '@stackbit/dev-common/dist/services/response-errors';
+
+export default function routes(router: Router, runner: Runner) {
+    router.get('/_health', (req: Request, res: Response) => {
+        return res.status(200).json({ status: 'ok' });
+    });
+    router.post('/_objects', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { objectIds } = req.body;
+        return runner
+            .getObjects(objectIds)
+            .then((result) => {
+                res.json(result);
+            })
+            .catch((err) => {
+                logger.error('Error fetching data objects');
+                logger.debug('error getting fieldData from runner', { error: err.message || err });
+                if (_.has(err, 'stack')) {
+                    logger.debug('error getting fieldData from runner, error.stack: ' + err.stack);
+                }
+                res.status(500).send('Error');
+                next(err);
+            });
+    });
+    router.post('/_objectsWithAnnotations', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { annotationTree, clientAnnotationErrors } = req.body;
+        return runner
+            .getObjectsWithAnnotations(annotationTree, clientAnnotationErrors)
+            .then((result) => {
+                res.json(result);
+            })
+            .catch((err) => {
+                logger.error('Error fetching data objects with annotations');
+                logger.debug('error getting fieldData with annotations from runner', { error: err.message || err });
+                if (_.has(err, 'stack')) {
+                    logger.debug('error getting fieldData with annotations from runner, error.stack: ' + err.stack);
+                }
+                res.status(500).send('Error');
+                next(err);
+            });
+    });
+    router.get('/_collections', (req: Request, res: Response, next: NextFunction) => {
+        return runner
+            .getCollections()
+            .then((result) => {
+                res.json(result);
+            })
+            .catch((err) => {
+                logger.error('Error fetching collections');
+                logger.debug('error getting collections from runner', { error: err.message || err });
+                if (_.has(err, 'stack')) {
+                    logger.debug('error getting collections from runner, error.stack: ' + err.stack);
+                }
+                res.status(500).send('Error');
+                next(err);
+            });
+    });
+    router.get('/_sitemap', (req: Request, res: Response, next: NextFunction) => {
+        return runner
+            .getSiteMap()
+            .then((result) => {
+                res.json(result);
+            })
+            .catch((err) => {
+                logger.error('Error fetching sitemap');
+                logger.debug('error getting sitemap from runner', { error: err.message || err });
+                if (_.has(err, 'stack')) {
+                    logger.debug('error getting sitemap from runner, error.stack: ' + err.stack);
+                }
+                res.status(500).send('Error');
+                next(err);
+            });
+    });
+    router.get('/_schema', (req: Request, res: Response, next: NextFunction) => {
+        return runner
+            .getSchema()
+            .then((schema) => res.json(schema))
+            .catch((err) => {
+                logger.error('Error fetching schema');
+                logger.debug('error getting schema from runner', { error: err });
+                res.status(500).send('Error');
+                next(err);
+            });
+    });
+    router.get('/_pageUrl', (req: Request, res: Response, next: NextFunction) => {
+        const { entryId: objectId, spaceId: projectId, locale } = req.query;
+        runner
+            .getUrl(objectId as string, projectId as string, locale as string)
+            .then((url) => res.json({ url }))
+            .catch((err) => {
+                if (err !== PageNotFoundError) {
+                    logger.error('Error querying for page url');
+                }
+                const log = err === PageNotFoundError ? logger.warn : logger.debug;
+                log('error getting page path from runner', { params: req.query, error: err });
+                res.json({ error: err.toString() });
+                next(err);
+            });
+    });
+    router.get('/_object', (req: Request, res: Response, next: NextFunction) => {
+        const { objectId, projectId } = req.query;
+        return runner
+            .getObject(objectId as string, projectId as string)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error fetching object');
+                logger.debug('error getting object', { error: err });
+                next(err);
+            });
+    });
+    router.post('/_object', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { modelName, filePath, object, user } = req.body;
+        return runner
+            .createObject(modelName, filePath, object, user)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error creating object');
+                logger.debug('error creating object', { error: err });
+                next(err);
+            });
+    });
+    router.put('/_object', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { objectId, projectId } = req.query;
+        const { object, user } = req.body;
+        return runner
+            .updateObject(objectId as string, projectId as string, object, user)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error updating object');
+                logger.debug('error updating object', { error: err });
+                next(err);
+            });
+    });
+    router.delete('/_object', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { objectId } = req.query;
+        const { user } = req.body;
+        return runner
+            .deleteObject(objectId as string, user)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error deleting collections');
+                logger.debug('error deleting object', { error: err });
+                next(err);
+            });
+    });
+    router.get('/_assets', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const filterParams = _.pick(req.body, ['searchQuery', 'pageId', 'pageSize']);
+        return runner
+            .listAssets(filterParams)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error listing assets');
+                logger.debug('error listing assets', { error: err });
+                res.status(500).json({ error: err.toString() });
+                next(err);
+            });
+    });
+    router.post('/_assets', [bodyParser.json({ limit: '500kb' })], (req: Request, res: Response, next: NextFunction) => {
+        const { url, filename, user } = req.body;
+        return runner
+            .uploadAsset(url, filename, user)
+            .then((response) => {
+                res.json(response);
+            })
+            .catch((err) => {
+                logger.error('Error adding asset');
+                logger.debug('error uploading asset', { error: err });
+                res.status(500).json({ error: err.toString() });
+                next(err);
+            });
+    });
+}

package/src/services/api.ts

@@ -0,0 +1,10 @@
+import axios from 'axios';
+import config from '../config';
+
+export default {
+    updateLocalDev: (userId: string, localDevConfig: any) => {
+        return axios.post(`${config.apiUrl}/project/local/${userId}`, {
+            config: localDevConfig
+        });
+    }
+};

package/src/services/file-watcher.ts

@@ -0,0 +1,37 @@
+import _ from 'lodash';
+import chokidar from 'chokidar';
+
+export function watchDir(dir: string, onFileChange: (filePaths: string[]) => void, throttleDelay = 1000) {
+    const watcher = chokidar.watch('.', {
+        cwd: dir,
+        ignored: (filePath) =>
+            filePath.includes('/.git/') ||
+            filePath.includes('/.next/') ||
+            filePath.includes('/.cache/') ||
+            filePath.includes('/node_modules/') ||
+            (filePath.includes('/.') && !filePath.includes('/.stackbit/')),
+        persistent: true,
+        ignoreInitial: true
+    });
+    let changedFiles: string[] = [];
+    const throttledFileChange = _.throttle(() => {
+        if (changedFiles.length) {
+            onFileChange(changedFiles);
+            changedFiles = [];
+        }
+    }, throttleDelay);
+    const handleFileChange = (filePath: string) => {
+        if (!changedFiles.includes(filePath)) {
+            changedFiles.push(filePath);
+        }
+        throttledFileChange();
+    };
+    watcher
+        .on('add', handleFileChange)
+        .on('change', handleFileChange)
+        .on('unlink', handleFileChange)
+        .on('addDir', handleFileChange)
+        .on('unlinkDir', handleFileChange);
+
+    return watcher;
+}

package/src/services/logger.ts

@@ -0,0 +1,34 @@
+import winston from 'winston';
+import config from '../config';
+
+const defaultFormats = [winston.format.colorize(), winston.format.simple()];
+
+const transports = [
+    new winston.transports.Console({
+        format: winston.format.combine(...defaultFormats)
+    })
+];
+
+const logger = winston.createLogger({
+    level: config.logLevel,
+    transports: transports
+});
+
+export const createLogger = ({ label }: { label: string }) => {
+    const formats = defaultFormats.slice();
+    if (label) {
+        formats.push(
+            winston.format.label({
+                label: label,
+                message: true
+            })
+        );
+    }
+    return winston.createLogger({
+        level: config.logLevel,
+        format: winston.format.combine(...formats),
+        transports: transports
+    });
+};
+
+export default logger;

package/src/services/ngrok.ts

@@ -0,0 +1,11 @@
+import ngrok from 'ngrok';
+
+export async function createTunnel(port: number, logger: any) {
+    return ngrok.connect({
+        addr: port,
+        onStatusChange: (status) => {
+            logger?.debug('Tunnel status: ' + status);
+        },
+        onLogEvent: (data) => {}
+    });
+}

package/src/services/user-config.ts

@@ -0,0 +1,14 @@
+import Configstore from 'configstore';
+import uuid from 'uuid';
+
+const config = new Configstore(
+    'stackbit-dev',
+    {
+        userId: uuid.v4().toString().replace(/\-/g, '')
+    },
+    { globalConfigPath: true }
+);
+
+export function getUserId() {
+    return config.get('userId');
+}

package/src/types/http-proxy-middleware.d.ts

@@ -0,0 +1,3 @@
+declare module 'http-proxy-middleware' {
+    export default function proxy(options: any, options2?: any): any;
+}