@superblocksteam/vite-plugin-file-sync 2.0.6-next.59 → 2.0.6-next.6

package/dist/file-system-manager.js

@@ -12,9 +12,8 @@ import { generateJSXAttribute } from "./codegen.js";
 import { ComponentsManager } from "./components-manager.js";
 import { addLegacyCustomComponentVariables, modifyLegacyCustomComponentElements, modifyLegacyCustomComponentImports, } from "./custom-components.js";
 import { applyErrorHandling, } from "./errors/error-handler.js";
-import { getApiFilePath, getPageFolder, isPageFilePath, PAGES_DIRECTORY, ROUTES_FILE, SCOPE_FILE, } from "./file-system-helpers.js";
+import { getPageFolder, PAGES_DIRECTORY, ROUTES_FILE, SCOPE_FILE, } from "./file-system-helpers.js";
 import { generate } from "./generate.js";
-import { OperationProcessor } from "./operations/operation-processor.js";
 import { doesElementHaveBinding } from "./parsing/bindings.js";
 import { getSbElementId } from "./parsing/ids.js";
 import { makeJSXAttribute } from "./parsing/jsx.js";
@@ -48,34 +47,24 @@ const SUPPORTED_FILETYPES = [
     },
 ];
 const APP_THEME_FILE_NAME = "appTheme.ts";
-export class FileSystemManager extends TracedEventEmitter {
+export class FileSyncManager extends TracedEventEmitter {
     rootDir;
     tsFiles = {};
     apiFiles = {};
     sourceTracker;
     fsOperationQueue;
-    operationProcessor;
     generationNumberSequence;
     routes = {};
     watcher;
     registeredComponentPaths = {};
     renameManager = new RenameManager();
     _tracer;
-    transactionNonce = Date.now();
-    pendingTransactions = new Set();
-    processedTransactions = [];
     constructor(fsOperationQueue, generationNumberSequence, tracer) {
         super(tracer, { captureRejections: true });
         this.rootDir = "/";
         this.fsOperationQueue = fsOperationQueue;
         this.generationNumberSequence = generationNumberSequence;
         this._tracer = tracer;
-        // intentionally a new queue here, we don't want to share the queue with the fsOperationQueue
-        this.operationProcessor = new OperationProcessor({
-            batchWindowMs: 50,
-            maxBatchSize: 100,
-        });
-        this.operationProcessor.disable();
         applyErrorHandling(this, {
             watch: { operation: "editing from code" },
             handleCreatePage: { operation: "creating a page" },
@@ -121,7 +110,27 @@ export class FileSystemManager extends TracedEventEmitter {
         }
         return path.join(this.rootDir, "App.tsx");
     }
-    // MARK: core setup/init
+    updateApi = (content, path) => {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        const { api: apiContents, stepPathMap } = content;
+        const pageName = getPageName(path);
+        let scopeId = this.sourceTracker?.getScopeDefinitionForPage(pageName)?.id;
+        if (!scopeId) {
+            console.warn("Scope ID not found for API", apiContents.metadata.name);
+            scopeId = "";
+        }
+        const updatedApi = {
+            apiPb: yaml.parse(JSON.stringify(apiContents)),
+            pageName,
+            stepPathMap,
+            scopeId,
+        };
+        const isNewApi = !this.apiFiles[path];
+        this.apiFiles[path] = updatedApi;
+        return { updatedApi, pageName, isNewApi };
+    };
     async watch(watcher, rootPath) {
         const logger = getLogger();
         this.rootDir = rootPath;
@@ -207,10 +216,178 @@ export class FileSystemManager extends TracedEventEmitter {
                 return;
             const { type, path } = file;
             if (type === "api") {
-                this.updateInternalApiData(content, path);
+                this.updateApi(content, path);
+            }
+        });
+        const handleFileChange = async (event, filePath) => {
+            logger.info(`File changed: ${filePath}, event: ${event}`);
+            switch (event) {
+                case "add": {
+                    const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
+                    if (!fileType || !filePath.startsWith(rootPath)) {
+                        return;
+                    }
+                    const data = await readFile(filePath);
+                    if (typeof data !== "string")
+                        return;
+                    // if we match pages/*/index.tsx, we want to emit an addPage event
+                    if (/pages\/.*\/index\.tsx/.test(filePath)) {
+                        const file = await readFile(filePath);
+                        if (!file) {
+                            logger.error(`Failed to read file: ${filePath}`);
+                            return;
+                        }
+                        if (!(filePath in this.tsFiles)) {
+                            this.tsFiles[filePath] = file;
+                            this.handleNonVisualChangeByDeletingIds(filePath, file);
+                            this.emit("addPage", filePath);
+                        }
+                    }
+                    switch (fileType.type) {
+                        case "api":
+                        case "python-api-step":
+                        case "js-api-step": {
+                            await this.processApiUpdates(filePath, fileType);
+                            break;
+                        }
+                    }
+                    break;
+                }
+                case "change": {
+                    // Special routes file handling
+                    if (filePath === routePath) {
+                        const data = await readFile(filePath);
+                        try {
+                            this.routes = JSON.parse(data);
+                            this.emit("routesChanged", this.routes);
+                        }
+                        catch (e) {
+                            logger.error("Error parsing routes file", getErrorMeta(e));
+                        }
+                        return;
+                    }
+                    const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
+                    if (!fileType || !filePath.startsWith(rootPath)) {
+                        return;
+                    }
+                    const data = await readFile(filePath);
+                    if (typeof data !== "string")
+                        return;
+                    switch (fileType.type) {
+                        case "tsx":
+                        case "scope":
+                            {
+                                if (!(filePath in this.tsFiles && this.tsFiles[filePath] === data)) {
+                                    logger.info(`File changed: ${filePath} updating AST tracker`);
+                                    this.tsFiles[filePath] = data;
+                                    // only update the source tracker if the file is different
+                                    this.handleNonVisualChangeByDeletingIds(filePath, data);
+                                    this.emit("fileChanged", filePath, data, true);
+                                }
+                                else {
+                                    logger.info(`File unchanged from last tracked state: ${filePath}`);
+                                    this.emit("fileChanged", filePath, data, false);
+                                }
+                            }
+                            break;
+                        case "api":
+                        case "python-api-step":
+                        case "js-api-step":
+                            {
+                                await this.processApiUpdates(filePath, fileType);
+                            }
+                            break;
+                    }
+                    break;
+                }
+                case "unlink": {
+                    if (filePath in this.apiFiles) {
+                        const api = this.apiFiles[filePath];
+                        delete this.apiFiles[filePath];
+                        if (!api || !this.sourceTracker)
+                            break;
+                        const scopeId = await this.sourceTracker.deleteApi({
+                            pageName: api.pageName,
+                            apiName: api.apiPb.metadata.name,
+                        });
+                        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+                        await this.writeChanges(changes);
+                        this.emit("apiManualDelete", {
+                            api: {
+                                id: getClientApiId(api.apiPb.metadata.name, api.pageName),
+                                apiName: api.apiPb.metadata.name,
+                                // TODO(saksham): get pagename more defensively
+                                pageName: getPageName(filePath),
+                                scopeId,
+                            },
+                        });
+                    }
+                    if (filePath in this.tsFiles) {
+                        delete this.tsFiles[filePath];
+                        this.sourceTracker?.removeFile(filePath);
+                        this.emit("deletePage", filePath);
+                    }
+                    break;
+                }
+            }
+        };
+        watcher.on("all", async (event, filePath) => {
+            const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
+            switch (fileType?.type) {
+                case "tsx":
+                case "scope":
+                    // these can be awaited to ensure sequential execution
+                    this.fsOperationQueue.enqueue(async () => {
+                        return await handleFileChange(event, filePath);
+                    });
+                    break;
+                default:
+                    // some files including APIs cannot be awaited currently
+                    this.fsOperationQueue.enqueue(async () => {
+                        void handleFileChange(event, filePath);
+                    });
             }
         });
-        watcher.on("all", this.handleFileChange);
+    }
+    getApiFiles() {
+        return this.formatApisToClientApis(this.apiFiles);
+    }
+    getTsFilePaths() {
+        return Object.keys(this.tsFiles);
+    }
+    getSourceTracker() {
+        return this.sourceTracker;
+    }
+    async writeFile(path, content, kind) {
+        if (kind === "ts") {
+            // happens eagerly regardless of error - possible desync
+            this.tsFiles[path] = content;
+            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
+            await this.fsOperationQueue.enqueue(async () => {
+                await fs.writeFile(path, content);
+            });
+        }
+        else if (kind === "api") {
+            const currentApiFile = this.apiFiles[path];
+            const apiPb = yaml.parse(content);
+            // Client APIs have id, but server APIs don't
+            if (apiPb.metadata.id) {
+                delete apiPb.metadata.id;
+            }
+            const stepPathMap = currentApiFile?.stepPathMap ?? {};
+            this.updateApi({ api: apiPb, stepPathMap }, path);
+            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
+            await this.fsOperationQueue.enqueue(async () => {
+                await writeApiFiles({ apiPb }, "api", path.split("/").slice(0, -1).join("/"), false, [], [], { extractLargeSourceFiles: true, minLinesForExtraction: 1 }, new Set(Object.keys(this.apiFiles)), stepPathMap);
+                // stepPathMap will be generated after the write when the file doesn't exist
+                if (this.apiFiles[path]) {
+                    this.apiFiles[path].stepPathMap = stepPathMap;
+                }
+            });
+        }
+    }
+    readFile(path) {
+        return this.tsFiles[path];
     }
     initializeSourceTracker() {
         this.sourceTracker = new SourceTracker(this._tracer);
@@ -319,235 +496,6 @@ export class FileSystemManager extends TracedEventEmitter {
             return null;
         }
     }
-    enableOperationsQueue() {
-        this.operationProcessor.enable();
-    }
-    disableOperationsQueue() {
-        this.operationProcessor.disable();
-    }
-    async flushOperations() {
-        await this.operationProcessor.flush();
-    }
-    // MARK: file change handling
-    handleFileChange = async (event, filePath) => {
-        const logger = getLogger();
-        logger.info(`File changed: ${filePath}, event: ${event}`);
-        const rootPath = this.rootDir;
-        if (!rootPath) {
-            throw new Error("Root directory not set");
-        }
-        // Skip directory events
-        if (event === "addDir" || event === "unlinkDir") {
-            return;
-        }
-        const routePath = path.join(rootPath, ROUTES_FILE);
-        const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
-        // Only handle files we care about and that are in our root path
-        if (!fileType || !filePath.startsWith(rootPath)) {
-            return;
-        }
-        // Queue the operation based on the event type
-        switch (event) {
-            case "add": {
-                const data = await readFile(filePath);
-                if (typeof data !== "string")
-                    return;
-                const isPage = isPageFilePath(filePath);
-                if (isPage) {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        execute: async () => {
-                            const file = await readFile(filePath);
-                            if (!file) {
-                                logger.error(`Failed to read file: ${filePath}`);
-                                return;
-                            }
-                            if (!(filePath in this.tsFiles)) {
-                                this.tsFiles[filePath] = file;
-                                this.handleNonVisualChangeByDeletingIds(filePath, file);
-                                this.emit("addPage", filePath);
-                            }
-                        },
-                    });
-                }
-                else {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        execute: async () => {
-                            switch (fileType.type) {
-                                case "api":
-                                case "python-api-step":
-                                case "js-api-step": {
-                                    await this.processApiFileUpdates(filePath, fileType);
-                                    break;
-                                }
-                            }
-                        },
-                    });
-                }
-                break;
-            }
-            case "change": {
-                if (filePath === routePath) {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        priority: true,
-                        execute: async () => {
-                            try {
-                                const data = JSON.parse((await readFile(filePath)) ?? "{}");
-                                if (!isEqual(this.routes, data)) {
-                                    this.routes = data;
-                                    this.emit("routesChanged", this.routes);
-                                }
-                            }
-                            catch (e) {
-                                logger.error("Error parsing routes file", getErrorMeta(e));
-                            }
-                        },
-                    });
-                    return;
-                }
-                else {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        execute: async () => {
-                            const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
-                            if (!fileType || !filePath.startsWith(rootPath)) {
-                                return;
-                            }
-                            const data = await readFile(filePath);
-                            if (typeof data !== "string")
-                                return;
-                            switch (fileType.type) {
-                                case "tsx":
-                                case "scope":
-                                    {
-                                        if (!(filePath in this.tsFiles &&
-                                            this.tsFiles[filePath] === data)) {
-                                            logger.info(`File changed: ${filePath} updating AST tracker`);
-                                            this.tsFiles[filePath] = data;
-                                            // only update the source tracker if the file is different
-                                            this.handleNonVisualChangeByDeletingIds(filePath, data);
-                                            this.emit("fileChanged", filePath, data, true);
-                                        }
-                                        else {
-                                            logger.info(`File unchanged from last tracked state: ${filePath}`);
-                                            this.emit("fileChanged", filePath, data, false);
-                                        }
-                                    }
-                                    break;
-                                case "api":
-                                case "python-api-step":
-                                case "js-api-step":
-                                    {
-                                        await this.processApiFileUpdates(filePath, fileType);
-                                    }
-                                    break;
-                            }
-                        },
-                    });
-                }
-                break;
-            }
-            case "unlink": {
-                if (filePath in this.tsFiles) {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        execute: async () => {
-                            await this.deleteTsFile(filePath);
-                        },
-                    });
-                }
-                else if (filePath in this.apiFiles) {
-                    void this.operationProcessor.addOperation({
-                        metadata: {
-                            filePath,
-                        },
-                        execute: async () => {
-                            await this.removeApiData(filePath);
-                        },
-                    });
-                }
-                break;
-            }
-        }
-    };
-    async deleteTsFile(filePath) {
-        delete this.tsFiles[filePath];
-        this.sourceTracker?.removeFile(filePath);
-        if (isPageFilePath(filePath)) {
-            this.emit("deletePage", filePath);
-        }
-    }
-    getTsFilePaths() {
-        return Object.keys(this.tsFiles);
-    }
-    getSourceTracker() {
-        return this.sourceTracker;
-    }
-    // MARK: fs read/write
-    async writeFile(path, content, kind) {
-        if (kind === "ts") {
-            // happens eagerly regardless of error - possible desync
-            this.tsFiles[path] = content;
-            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
-            await this.fsOperationQueue.enqueue(async () => {
-                await fs.writeFile(path, content);
-            });
-        }
-        else if (kind === "api") {
-            const currentApiFile = this.apiFiles[path];
-            const apiPb = yaml.parse(content);
-            // Client APIs have id, but server APIs don't
-            if (apiPb.metadata.id) {
-                delete apiPb.metadata.id;
-            }
-            const stepPathMap = currentApiFile?.stepPathMap ?? {};
-            this.updateInternalApiData({ api: apiPb, stepPathMap }, path);
-            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
-            await this.fsOperationQueue.enqueue(async () => {
-                await writeApiFiles({ apiPb }, "api", path.split("/").slice(0, -1).join("/"), false, [], [], { extractLargeSourceFiles: true, minLinesForExtraction: 1 }, new Set(Object.keys(this.apiFiles)), stepPathMap);
-                // stepPathMap will be generated after the write when the file doesn't exist
-                if (this.apiFiles[path]) {
-                    this.apiFiles[path].stepPathMap = stepPathMap;
-                }
-            });
-        }
-    }
-    readFile(path) {
-        return this.tsFiles[path];
-    }
-    async addRoute(route, filePath) {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        this.routes[route] = { file: this.getRelativeRoutePath(filePath) };
-        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
-    }
-    async removeRoute(filePath) {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const relativeFilePath = this.getRelativeRoutePath(filePath);
-        this.routes = Object.fromEntries(Object.entries(this.routes).filter(([_, value]) => value.file !== relativeFilePath));
-        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
-    }
-    async writeChanges(changes, callback) {
-        return Promise.all(changes.map(async ({ fileName, source, kind }) => {
-            await this.writeFile(fileName, source, kind);
-            return callback?.(fileName, source);
-        }));
-    }
     getLocalBindingEntities(path) {
         const logger = getLogger();
         const files = this.sourceTracker?.getCurrentFiles();
@@ -575,36 +523,8 @@ export class FileSystemManager extends TracedEventEmitter {
         });
         return Array.from(localBindingEntities);
     }
-    getPageRoots(filePath) {
-        const scopeFilePath = path.join(path.dirname(filePath), "index.tsx");
-        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
-        if (!currentFile) {
-            return null;
-        }
-        return getPageRoots(filePath, currentFile);
-    }
-    getScope(filePath) {
-        // get the scope.ts file in the same directory as path
-        const scopeFilePath = path.join(path.dirname(filePath), SCOPE_FILE);
-        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
-        if (!currentFile) {
-            console.log("File not found", scopeFilePath);
-            return null;
-        }
-        const scope = getScope(scopeFilePath, currentFile);
-        return scope;
-    }
-    getRoutes() {
-        const routes = [];
-        for (const [path, { file }] of Object.entries(this.routes)) {
-            routes.push({
-                path,
-                component: file,
-            });
-        }
-        return routes;
-    }
-    // MARK: transaction handling
+    pendingTransactions = new Set();
+    processedTransactions = [];
     flushTransactions = () => {
         // TODO do something more sophisticated than this.
         this.processedTransactions = this.processedTransactions.slice(-20);
@@ -617,13 +537,13 @@ export class FileSystemManager extends TracedEventEmitter {
         }
         this.pendingTransactions.add(transactionId);
     };
+    transactionNonce = Date.now();
     // Until we have a sophisticated way to track ALL operations, including non-UI initiated ops, we will use a nonce
     // TODO https://github.com/superblocksteam/superblocks/pull/11788
     getProcessedTransactionsWithNonce() {
         const nonce = `t-${this.transactionNonce++}`;
         return this.processedTransactions.concat(nonce);
     }
-    // MARK: editor operations
     handleCreatePage = async (payload) => {
         const { name } = payload;
         if (!this.rootDir) {
@@ -640,9 +560,7 @@ function Page() {
 
 export default registerPage(Page, { name: "${name}" });
 `;
-        await this.fsOperationQueue.enqueue(async () => {
-            await fs.mkdir(pagePath, { recursive: true });
-        });
+        await fs.mkdir(pagePath, { recursive: true });
         await this.writeFile(pageIndexPath, pageContent, "ts");
         await this.handleNonVisualChangeByDeletingIds(pageIndexPath, pageContent);
         this.emit("addPage", pageIndexPath);
@@ -650,11 +568,11 @@ export default registerPage(Page, { name: "${name}" });
     handleReparent = async (payload, writeFile = true) => {
         const { from, to, changedProps, transaction } = payload;
         this.trackTransaction(transaction?.id);
-        this.sourceTracker?.setProperties({
+        await this.sourceTracker?.setProperties({
             source: from.source,
             changes: changedProps ?? {},
         });
-        this.sourceTracker?.moveElement({
+        await this.sourceTracker?.moveElement({
             from,
             to,
         });
@@ -668,7 +586,7 @@ export default registerPage(Page, { name: "${name}" });
     };
     handleCreateComponent = async (payload, writeFile = true) => {
         this.trackTransaction(payload.transaction?.id);
-        const sourceId = this.sourceTracker?.addElement(payload);
+        const sourceId = await this.sourceTracker?.addElement(payload);
         if (writeFile) {
             const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
             await this.writeChanges(changes ?? [], (fileName) => {
@@ -682,7 +600,7 @@ export default registerPage(Page, { name: "${name}" });
         const { elements, transaction } = payload;
         this.trackTransaction(transaction?.id);
         for (const element of elements) {
-            this.sourceTracker?.deleteElement({
+            await this.sourceTracker?.deleteElement({
                 source: element.source,
                 scopeName: element.scopeName,
             });
@@ -698,7 +616,7 @@ export default registerPage(Page, { name: "${name}" });
     handleSetProperty = async (payload, writeFile = true) => {
         const { element: { source }, property, value, transaction, } = payload;
         this.trackTransaction(transaction?.id);
-        this.sourceTracker?.setProperty({
+        await this.sourceTracker?.setProperty({
             source,
             property,
             info: value,
@@ -713,7 +631,7 @@ export default registerPage(Page, { name: "${name}" });
     handleSetProperties = async (payload, writeFile = true) => {
         const { element: { source }, properties, transaction, } = payload;
         this.trackTransaction(transaction?.id);
-        this.sourceTracker?.setProperties({
+        await this.sourceTracker?.setProperties({
             source,
             changes: properties,
         });
@@ -769,147 +687,6 @@ export default registerPage(Page, { name: "${name}" });
         this.flushTransactions();
         return returnValues;
     };
-    // MARK: entity operations
-    handleAddEntity = async (payload) => {
-        this.sourceTracker?.addEntity({
-            scopeId: payload.scopeId,
-            entity: {
-                type: payload.type,
-                name: payload.name,
-                attributes: payload.attributes,
-            },
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes, (fileName) => {
-            this.emit("addEntity", fileName, payload);
-        });
-    };
-    handleUpdateEntity = async (payload) => {
-        this.sourceTracker?.updateEntity({
-            entityId: payload.entityId,
-            updates: payload.updates,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes, (fileName) => {
-            this.emit("updateEntity", fileName, payload);
-        });
-    };
-    handleDeleteEntity = async (payload) => {
-        const deletedEntityName = this.sourceTracker?.deleteEntity({
-            entityId: payload.entityId,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes, (fileName) => {
-            if (deletedEntityName) {
-                this.emit("deleteEntity", fileName, deletedEntityName);
-            }
-        });
-    };
-    handleUpdateTheme = async (payload) => {
-        const { theme } = payload;
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const filePath = path.join(this.rootDir, APP_THEME_FILE_NAME);
-        this.sourceTracker?.updateTheme({
-            themeFilePath: filePath,
-            theme,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-    };
-    // MARK: rename operations
-    handleRenameElement = async (payload) => {
-        if (payload.kind === "component") {
-            return this.handleRenameComponent(payload);
-        }
-        else if (payload.kind === "entity") {
-            return this.handleRenameEntity(payload);
-        }
-        else if (payload.kind === "page") {
-            return this.handleRenamePage(payload);
-        }
-    };
-    handleRenameComponent = async (payload) => {
-        const { elementId, newName, oldName, scopeName } = payload;
-        await this.sourceTracker?.renameComponent({
-            widgetSourceId: elementId,
-            oldName,
-            newName,
-            scopeName,
-        });
-        await this.renameIdentifierInApis({
-            elementId,
-            oldName,
-            newName,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes, (fileName) => {
-            this.emit("renameComponent", fileName);
-        });
-    };
-    handleRenameEntity = async (payload) => {
-        const { elementId, newName, oldName, scopeName } = payload;
-        this.sourceTracker?.renameEntity({
-            entityId: elementId,
-            oldName,
-            newName,
-            scopeName,
-        });
-        await this.renameIdentifierInApis(payload);
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes, (fileName) => {
-            this.emit("renameEntity", fileName);
-        });
-    };
-    handleRenamePage = async (payload) => {
-        const { newName, oldName } = payload;
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const newPageFolder = getPageFolder(this.rootDir, newName);
-        const newIndexFilePath = path.join(newPageFolder, "index.tsx");
-        const oldIndexFilePath = path.join(this.rootDir, "pages", oldName, "index.tsx");
-        const oldPageFolder = getPageFolder(this.rootDir, oldName);
-        this.watcher?.unwatch(newPageFolder);
-        this.watcher?.unwatch(oldPageFolder);
-        const existingRoute = Object.keys(this.routes).find((route) => this.routes[route]?.file ===
-            this.getRelativeRoutePath(oldIndexFilePath));
-        if (!existingRoute) {
-            throw new Error(`Route for ${oldName} not found`);
-        }
-        // Write the name attribute to the page file
-        await this.sourceTracker?.renamePage({
-            newName,
-            filePath: oldIndexFilePath,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-        // A rename of a folder is an unlink followed by an add, so the file watcher will take over initializing this
-        // "new" page
-        await fs.rename(oldPageFolder, newPageFolder);
-        // Now we just clean up the routes
-        await this.removeRoute(oldIndexFilePath);
-        await this.addRoute(existingRoute, newIndexFilePath);
-        const newIndexFile = await readFile(newIndexFilePath);
-        if (!newIndexFile) {
-            throw new Error(`New index file ${newIndexFilePath} not found`);
-        }
-        this.tsFiles[newIndexFilePath] = newIndexFile;
-        await this.handleNonVisualChangeByDeletingIds(newIndexFilePath, newIndexFile);
-        this.emit("renamePage", newIndexFilePath);
-        // Re-add the watcher
-        this.watcher?.add(newPageFolder);
-        this.watcher?.add(oldPageFolder);
-    };
-    getRelativeRoutePath(filePath) {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        // no leading slash
-        return path.relative(path.join(this.rootDir, PAGES_DIRECTORY), filePath);
-    }
-    // MARK: API operations
     handleUpdateApi = async (payload) => {
         const { api } = payload;
         if (!this.sourceTracker) {
@@ -925,24 +702,24 @@ export default registerPage(Page, { name: "${name}" });
         if (!apiName) {
             throw new Error("API name is not set");
         }
-        const apiFilePath = getApiFilePath(this.rootDir, api.pageName, apiName);
-        const apiDir = path.dirname(apiFilePath);
-        const isNewApi = !this.getApiFiles()[apiFilePath];
+        const apiDir = path.join(this.rootDir, "pages", api.pageName, "apis", apiName);
+        const apiPath = path.join(apiDir, "api.yaml");
+        const isNewApi = !this.getApiFiles()[apiPath];
         try {
             const stats = await fs.stat(apiDir);
             if (!stats.isDirectory()) {
-                await this.fsOperationQueue.enqueue(async () => await fs.mkdir(apiDir, { recursive: true }));
+                await fs.mkdir(apiDir, { recursive: true });
             }
         }
         catch {
-            await this.fsOperationQueue.enqueue(async () => await fs.mkdir(apiDir, { recursive: true }));
+            await fs.mkdir(apiDir, { recursive: true });
         }
-        await this.writeFile(apiFilePath, yaml.stringify(api.apiPb), "api");
+        await this.writeFile(apiPath, yaml.stringify(api.apiPb), "api");
         const generationNumber = this.generationNumberSequence.next();
         const apiDef = this.createClientApi(api);
         let scopeId = "";
         if (isNewApi) {
-            scopeId = await this.addApiToScope(api);
+            scopeId = await this.createScopedApi(api);
         }
         else {
             const scopeDef = this.sourceTracker.getScopeDefinitionForPage(api.pageName);
@@ -951,6 +728,20 @@ export default registerPage(Page, { name: "${name}" });
         this.emit("apiUpdate", { api: apiDef, scopeId });
         return { api: apiDef, scopeId, generationNumber };
     };
+    createScopedApi = async (api) => {
+        if (!this.sourceTracker) {
+            throw new Error("Source tracker not initialized");
+        }
+        // We want to add the API entity to our scope, but we do not want to emit entity events, because
+        // the API update event handles this particular side effect.
+        const scopeId = await this.sourceTracker.addApi({
+            pageName: api.pageName,
+            apiName: api.apiPb.metadata.name,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        return scopeId;
+    };
     handleDeleteApi = async (payload) => {
         const logger = getLogger();
         const { apis } = payload;
@@ -958,34 +749,40 @@ export default registerPage(Page, { name: "${name}" });
             throw new Error("Root directory not set");
         }
         const rootDir = this.rootDir;
-        const deletedApis = [];
-        for (const { apiName, pageName } of apis) {
-            const apiFilePath = getApiFilePath(rootDir, pageName, apiName);
-            const api = this.apiFiles[apiFilePath];
-            if (!api || !this.sourceTracker) {
-                continue;
-            }
-            const apiDir = path.dirname(apiFilePath);
-            try {
-                const stats = await fs.stat(apiDir);
-                if (stats.isDirectory()) {
-                    await fs.rmdir(apiDir, { recursive: true });
+        const executeDeleteApis = Promise.all(apis.map(({ apiName, pageName }) => {
+            return new Promise(
+            // eslint-disable-next-line no-async-promise-executor
+            async (resolve) => {
+                const apiFilePath = path.join(rootDir, "pages", pageName, "apis", apiName, "api.yaml");
+                const api = this.apiFiles[apiFilePath];
+                if (!api || !this.sourceTracker) {
+                    return resolve(undefined);
                 }
-                delete this.apiFiles[apiFilePath];
-                const scopeId = this.sourceTracker.deleteApi({
-                    pageName,
-                    apiName,
-                });
-                deletedApis.push({
-                    apiName,
-                    pageName,
-                    scopeId,
-                });
-            }
-            catch (e) {
-                logger.warn(`Could not delete api ${apiFilePath} ${JSON.stringify(e)}`);
-            }
-        }
+                const apiDir = path.join(rootDir, "pages", pageName, "apis", apiName);
+                try {
+                    const stats = await fs.stat(apiDir);
+                    if (stats.isDirectory()) {
+                        await fs.rmdir(apiDir, { recursive: true });
+                    }
+                    delete this.apiFiles[apiFilePath];
+                    const scopeId = await this.sourceTracker.deleteApi({
+                        pageName,
+                        apiName,
+                    });
+                    resolve({
+                        apiName,
+                        pageName,
+                        scopeId,
+                    });
+                }
+                catch (e) {
+                    logger.warn(`Could not delete api ${apiFilePath} ${JSON.stringify(e)}`);
+                    resolve(undefined);
+                }
+                resolve(undefined);
+            });
+        }));
+        const deletedApis = (await executeDeleteApis).filter((api) => api !== undefined);
         const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
         await this.writeChanges(changes);
         this.emit("apiDelete", {
@@ -1066,40 +863,229 @@ export default registerPage(Page, { name: "${name}" });
             // TODO: Should I delete here?
         }));
     };
-    async removeApiData(filePath) {
-        const api = this.apiFiles[filePath];
-        if (!api) {
-            return;
+    handleAddEntity = async (payload) => {
+        await this.sourceTracker?.addEntity({
+            scopeId: payload.scopeId,
+            entity: {
+                type: payload.type,
+                name: payload.name,
+                attributes: payload.attributes,
+            },
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes, (fileName) => {
+            this.emit("addEntity", fileName, payload);
+        });
+    };
+    handleUpdateEntity = async (payload) => {
+        await this.sourceTracker?.updateEntity({
+            entityId: payload.entityId,
+            updates: payload.updates,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes, (fileName) => {
+            this.emit("updateEntity", fileName, payload);
+        });
+    };
+    handleDeleteEntity = async (payload) => {
+        const deletedEntityName = await this.sourceTracker?.deleteEntity({
+            entityId: payload.entityId,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes, (fileName) => {
+            if (deletedEntityName) {
+                this.emit("deleteEntity", fileName, deletedEntityName);
+            }
+        });
+    };
+    handleUpdateTheme = async (payload) => {
+        const { theme } = payload;
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
         }
-        delete this.apiFiles[filePath];
-        this.sourceTracker?.deleteApi({
-            pageName: api.pageName,
-            apiName: api.apiPb.metadata.name,
+        const filePath = path.join(this.rootDir, APP_THEME_FILE_NAME);
+        await this.sourceTracker?.updateTheme({
+            themeFilePath: filePath,
+            theme,
         });
         const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
         await this.writeChanges(changes);
-        const scopeId = this.sourceTracker?.getScopeDefinitionForPage(api.pageName)?.id;
-        this.emit("apiManualDelete", {
-            api: {
-                id: getClientApiId(api.apiPb.metadata.name, api.pageName),
-                apiName: api.apiPb.metadata.name,
-                // TODO(saksham): get pagename more defensively
-                pageName: getPageName(filePath),
-                scopeId,
-            },
+    };
+    handleRenameElement = async (payload) => {
+        if (payload.kind === "component") {
+            return this.handleRenameComponent(payload);
+        }
+        else if (payload.kind === "entity") {
+            return this.handleRenameEntity(payload);
+        }
+        else if (payload.kind === "page") {
+            return this.handleRenamePage(payload);
+        }
+    };
+    handleRenameComponent = async (payload) => {
+        const { elementId, newName, oldName, scopeName } = payload;
+        await this.sourceTracker?.renameComponent({
+            widgetSourceId: elementId,
+            oldName,
+            newName,
+            scopeName,
         });
+        await this.renameIdentifierInApis({
+            elementId,
+            oldName,
+            newName,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes, (fileName) => {
+            this.emit("renameComponent", fileName);
+        });
+    };
+    handleRenameEntity = async (payload) => {
+        const { elementId, newName, oldName, scopeName } = payload;
+        await this.sourceTracker?.renameEntity({
+            entityId: elementId,
+            oldName,
+            newName,
+            scopeName,
+        });
+        await this.renameIdentifierInApis(payload);
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes, (fileName) => {
+            this.emit("renameEntity", fileName);
+        });
+    };
+    handleRenamePage = async (payload) => {
+        const { newName, oldName } = payload;
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        const newPageFolder = getPageFolder(this.rootDir, newName);
+        const newIndexFilePath = path.join(newPageFolder, "index.tsx");
+        const oldIndexFilePath = path.join(this.rootDir, "pages", oldName, "index.tsx");
+        const oldPageFolder = getPageFolder(this.rootDir, oldName);
+        this.watcher?.unwatch(newPageFolder);
+        this.watcher?.unwatch(oldPageFolder);
+        const existingRoute = Object.keys(this.routes).find((route) => this.routes[route]?.file ===
+            this.getRelativeRoutePath(oldIndexFilePath));
+        if (!existingRoute) {
+            throw new Error(`Route for ${oldName} not found`);
+        }
+        // Write the name attribute to the page file
+        await this.sourceTracker?.renamePage({
+            newName,
+            filePath: oldIndexFilePath,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        // A rename of a folder is an unlink followed by an add, so the file watcher will take over initializing this
+        // "new" page
+        await fs.rename(oldPageFolder, newPageFolder);
+        // Now we just clean up the routes
+        await this.removeRoute(oldIndexFilePath);
+        await this.addRoute(existingRoute, newIndexFilePath);
+        const newIndexFile = await readFile(newIndexFilePath);
+        if (!newIndexFile) {
+            throw new Error(`New index file ${newIndexFilePath} not found`);
+        }
+        this.tsFiles[newIndexFilePath] = newIndexFile;
+        await this.handleNonVisualChangeByDeletingIds(newIndexFilePath, newIndexFile);
+        this.emit("renamePage", newIndexFilePath);
+        // Re-add the watcher
+        this.watcher?.add(newPageFolder);
+        this.watcher?.add(oldPageFolder);
+    };
+    getPageRoots(filePath) {
+        const scopeFilePath = path.join(path.dirname(filePath), "index.tsx");
+        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
+        if (!currentFile) {
+            return null;
+        }
+        return getPageRoots(filePath, currentFile);
     }
-    getApiFiles() {
-        return Object.keys(this.apiFiles).reduce((acc, key) => {
-            if (!this.apiFiles[key]) {
-                return acc;
+    getScope(filePath) {
+        // get the scope.ts file in the same directory as path
+        const scopeFilePath = path.join(path.dirname(filePath), SCOPE_FILE);
+        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
+        if (!currentFile) {
+            console.log("File not found", scopeFilePath);
+            return null;
+        }
+        const scope = getScope(scopeFilePath, currentFile);
+        return scope;
+    }
+    getRoutes() {
+        const routes = [];
+        for (const [path, { file }] of Object.entries(this.routes)) {
+            routes.push({
+                path,
+                component: file,
+            });
+        }
+        return routes;
+    }
+    async addRoute(route, filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        this.routes[route] = { file: this.getRelativeRoutePath(filePath) };
+        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
+    }
+    async removeRoute(filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        const relativeFilePath = this.getRelativeRoutePath(filePath);
+        this.routes = Object.fromEntries(Object.entries(this.routes).filter(([_, value]) => value.file !== relativeFilePath));
+        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
+    }
+    async writeChanges(changes, callback) {
+        return Promise.all(changes.map(async ({ fileName, source, kind }) => {
+            await this.writeFile(fileName, source, kind);
+            return callback?.(fileName, source);
+        }));
+    }
+    getNodeForWidgetSourceId(id) {
+        return this.sourceTracker?.getElementToLocation(id);
+    }
+    getAstForWidgetSourceId(id) {
+        const filePath = this.sourceTracker?.getElementToFilePath(id);
+        if (!filePath) {
+            return null;
+        }
+        return this.sourceTracker?.getCurrentFiles()[filePath]?.ast;
+    }
+    renameIdentifierInApis = async ({ elementId, oldName, newName, parentBinding, }) => {
+        const apisInScope = structuredClone(this.getApisInScope(elementId));
+        await this.renameManager.renameEntityInApis({
+            oldName,
+            newName,
+            parentBinding,
+            apis: apisInScope,
+        });
+        // only save the APIs that have changed
+        await Promise.all(apisInScope.map(({ api, filePath }) => {
+            if (isEqual(api?.apiPb, this.apiFiles[filePath]?.apiPb)) {
+                return Promise.resolve();
             }
-            acc[key] = {
-                api: this.createClientApi(this.apiFiles[key]),
-                scopeId: this.apiFiles[key].scopeId,
-            };
-            return acc;
-        }, {});
+            return this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
+        }));
+    };
+    getApisInScope = (elementId) => {
+        const filePath = this.sourceTracker?.getElementToFilePath(elementId);
+        if (!filePath) {
+            return [];
+        }
+        const pagePath = path.dirname(filePath);
+        return Object.entries(this.apiFiles)
+            .filter(([filePath]) => filePath.includes(pagePath))
+            .map(([filePath, api]) => ({ api, filePath }));
+    };
+    getRelativeRoutePath(filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        // no leading slash
+        return path.relative(path.join(this.rootDir, PAGES_DIRECTORY), filePath);
     }
     // Utilities for converting server API format to Client API format
     // We internally save the API as the server does, but we return should always return it
@@ -1116,21 +1102,19 @@ export default registerPage(Page, { name: "${name}" });
             },
         };
     }
-    addApiToScope = async (api) => {
-        if (!this.sourceTracker) {
-            throw new Error("Source tracker not initialized");
-        }
-        // We want to add the API entity to our scope, but we do not want to emit entity events, because
-        // the API update event handles this particular side effect.
-        const scopeId = await this.sourceTracker.addApi({
-            pageName: api.pageName,
-            apiName: api.apiPb.metadata.name,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-        return scopeId;
-    };
-    async processApiFileUpdates(filePath, fileType) {
+    formatApisToClientApis(serverApis) {
+        return Object.keys(serverApis).reduce((acc, key) => {
+            if (!serverApis[key]) {
+                return acc;
+            }
+            acc[key] = {
+                api: this.createClientApi(serverApis[key]),
+                scopeId: serverApis[key].scopeId,
+            };
+            return acc;
+        }, {});
+    }
+    async processApiUpdates(filePath, fileType) {
         let yamlPath = filePath;
         if (fileType.type === "python-api-step" ||
             fileType.type === "js-api-step") {
@@ -1148,7 +1132,7 @@ export default registerPage(Page, { name: "${name}" });
             const parsedData = { apiPb: apiContent?.api };
             if (!(yamlPath in this.apiFiles &&
                 isEqual(this.apiFiles[yamlPath]?.apiPb, parsedData.apiPb))) {
-                const { updatedApi, pageName, isNewApi } = this.updateInternalApiData({
+                const { updatedApi, pageName, isNewApi } = this.updateApi({
                     api: parsedData.apiPb,
                     stepPathMap: apiContent.stepPathMap,
                 }, yamlPath);
@@ -1159,7 +1143,7 @@ export default registerPage(Page, { name: "${name}" });
                     return;
                 }
                 if (isNewApi) {
-                    await this.addApiToScope(updatedApi);
+                    await this.createScopedApi(updatedApi);
                 }
                 this.emit("apiManualUpdate", {
                     api: this.createClientApi(updatedApi),
@@ -1175,56 +1159,9 @@ export default registerPage(Page, { name: "${name}" });
             logger.error(`Error updating API: ${yamlPath}, error: ${error}`);
         }
     }
-    updateInternalApiData = (content, path) => {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const { api: apiContents, stepPathMap } = content;
-        const pageName = getPageName(path);
-        let scopeId = this.sourceTracker?.getScopeDefinitionForPage(pageName)?.id;
-        if (!scopeId) {
-            console.warn("Scope ID not found for API", apiContents.metadata.name);
-            scopeId = "";
-        }
-        const updatedApi = {
-            apiPb: yaml.parse(JSON.stringify(apiContents)),
-            pageName,
-            stepPathMap,
-            scopeId,
-        };
-        const isNewApi = !this.apiFiles[path];
-        this.apiFiles[path] = updatedApi;
-        return { updatedApi, pageName, isNewApi };
-    };
-    renameIdentifierInApis = async ({ elementId, oldName, newName, parentBinding, }) => {
-        const apisInScope = structuredClone(this.getApisInScope(elementId));
-        await this.renameManager.renameEntityInApis({
-            oldName,
-            newName,
-            parentBinding,
-            apis: apisInScope,
-        });
-        // only save the APIs that have changed
-        await Promise.all(apisInScope.map(({ api, filePath }) => {
-            if (isEqual(api?.apiPb, this.apiFiles[filePath]?.apiPb)) {
-                return Promise.resolve();
-            }
-            return this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
-        }));
-    };
-    getApisInScope = (elementId) => {
-        const filePath = this.sourceTracker?.getElementToFilePath(elementId);
-        if (!filePath) {
-            return [];
-        }
-        const pagePath = path.dirname(filePath);
-        return Object.entries(this.apiFiles)
-            .filter(([filePath]) => filePath.includes(pagePath))
-            .map(([filePath, api]) => ({ api, filePath }));
-    };
 }
 // Add new mock implementation
-export class MockFileSyncManager extends FileSystemManager {
+export class MockFileSyncManager extends FileSyncManager {
     tsFiles = {};
     apiFiles = {};
     async watch(_watcher, _path) {
@@ -1300,7 +1237,7 @@ async function readFiles(dir) {
 const getMergedApiContent = async (path) => {
     return readAppApiYamlFile(path.split("/").slice(0, -1).join("/"));
 };
-export const getPageName = (path) => {
+const getPageName = (path) => {
     const parts = path.split("/");
     const pagesIndex = parts.findIndex((part) => part === "pages");
     if (pagesIndex !== -1 && parts[pagesIndex + 1]) {