@superblocksteam/vite-plugin-file-sync 2.0.6-next.4 → 2.0.6-next.40

package/dist/file-system-manager.js

@@ -12,8 +12,9 @@ import { generateJSXAttribute } from "./codegen.js";
 import { ComponentsManager } from "./components-manager.js";
 import { addLegacyCustomComponentVariables, modifyLegacyCustomComponentElements, modifyLegacyCustomComponentImports, } from "./custom-components.js";
 import { applyErrorHandling, } from "./errors/error-handler.js";
-import { getPageFolder, PAGES_DIRECTORY, ROUTES_FILE, SCOPE_FILE, } from "./file-system-helpers.js";
+import { getApiFilePath, getPageFolder, isPageFilePath, PAGES_DIRECTORY, ROUTES_FILE, SCOPE_FILE, } from "./file-system-helpers.js";
 import { generate } from "./generate.js";
+import { OperationProcessor } from "./operations/operation-processor.js";
 import { doesElementHaveBinding } from "./parsing/bindings.js";
 import { getSbElementId } from "./parsing/ids.js";
 import { makeJSXAttribute } from "./parsing/jsx.js";
@@ -47,24 +48,34 @@ const SUPPORTED_FILETYPES = [
     },
 ];
 const APP_THEME_FILE_NAME = "appTheme.ts";
-export class FileSyncManager extends TracedEventEmitter {
+export class FileSystemManager extends TracedEventEmitter {
     rootDir;
     tsFiles = {};
     apiFiles = {};
     sourceTracker;
     fsOperationQueue;
+    operationProcessor;
     generationNumberSequence;
     routes = {};
     watcher;
     registeredComponentPaths = {};
     renameManager = new RenameManager();
     _tracer;
+    transactionNonce = Date.now();
+    pendingTransactions = new Set();
+    processedTransactions = [];
     constructor(fsOperationQueue, generationNumberSequence, tracer) {
         super(tracer, { captureRejections: true });
         this.rootDir = "/";
         this.fsOperationQueue = fsOperationQueue;
         this.generationNumberSequence = generationNumberSequence;
         this._tracer = tracer;
+        // intentionally a new queue here, we don't want to share the queue with the fsOperationQueue
+        this.operationProcessor = new OperationProcessor({
+            batchWindowMs: 50,
+            maxBatchSize: 100,
+        });
+        this.operationProcessor.disable();
         applyErrorHandling(this, {
             watch: { operation: "editing from code" },
             handleCreatePage: { operation: "creating a page" },
@@ -110,27 +121,7 @@ export class FileSyncManager extends TracedEventEmitter {
         }
         return path.join(this.rootDir, "App.tsx");
     }
-    updateApi = (content, path) => {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const { api: apiContents, stepPathMap } = content;
-        const pageName = getPageName(path);
-        let scopeId = this.sourceTracker?.getScopeDefinitionForPage(pageName)?.id;
-        if (!scopeId) {
-            console.warn("Scope ID not found for API", apiContents.metadata.name);
-            scopeId = "";
-        }
-        const updatedApi = {
-            apiPb: yaml.parse(JSON.stringify(apiContents)),
-            pageName,
-            stepPathMap,
-            scopeId,
-        };
-        const isNewApi = !this.apiFiles[path];
-        this.apiFiles[path] = updatedApi;
-        return { updatedApi, pageName, isNewApi };
-    };
+    // MARK: core setup/init
     async watch(watcher, rootPath) {
         const logger = getLogger();
         this.rootDir = rootPath;
@@ -216,178 +207,10 @@ export class FileSyncManager extends TracedEventEmitter {
                 return;
             const { type, path } = file;
             if (type === "api") {
-                this.updateApi(content, path);
-            }
-        });
-        const handleFileChange = async (event, filePath) => {
-            logger.info(`File changed: ${filePath}, event: ${event}`);
-            switch (event) {
-                case "add": {
-                    const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
-                    if (!fileType || !filePath.startsWith(rootPath)) {
-                        return;
-                    }
-                    const data = await readFile(filePath);
-                    if (typeof data !== "string")
-                        return;
-                    // if we match pages/*/index.tsx, we want to emit an addPage event
-                    if (/pages\/.*\/index\.tsx/.test(filePath)) {
-                        const file = await readFile(filePath);
-                        if (!file) {
-                            logger.error(`Failed to read file: ${filePath}`);
-                            return;
-                        }
-                        if (!(filePath in this.tsFiles)) {
-                            this.tsFiles[filePath] = file;
-                            this.handleNonVisualChangeByDeletingIds(filePath, file);
-                            this.emit("addPage", filePath);
-                        }
-                    }
-                    switch (fileType.type) {
-                        case "api":
-                        case "python-api-step":
-                        case "js-api-step": {
-                            await this.processApiUpdates(filePath, fileType);
-                            break;
-                        }
-                    }
-                    break;
-                }
-                case "change": {
-                    // Special routes file handling
-                    if (filePath === routePath) {
-                        const data = await readFile(filePath);
-                        try {
-                            this.routes = JSON.parse(data);
-                            this.emit("routesChanged", this.routes);
-                        }
-                        catch (e) {
-                            logger.error("Error parsing routes file", getErrorMeta(e));
-                        }
-                        return;
-                    }
-                    const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
-                    if (!fileType || !filePath.startsWith(rootPath)) {
-                        return;
-                    }
-                    const data = await readFile(filePath);
-                    if (typeof data !== "string")
-                        return;
-                    switch (fileType.type) {
-                        case "tsx":
-                        case "scope":
-                            {
-                                if (!(filePath in this.tsFiles && this.tsFiles[filePath] === data)) {
-                                    logger.info(`File changed: ${filePath} updating AST tracker`);
-                                    this.tsFiles[filePath] = data;
-                                    // only update the source tracker if the file is different
-                                    this.handleNonVisualChangeByDeletingIds(filePath, data);
-                                    this.emit("fileChanged", filePath, data, true);
-                                }
-                                else {
-                                    logger.info(`File unchanged from last tracked state: ${filePath}`);
-                                    this.emit("fileChanged", filePath, data, false);
-                                }
-                            }
-                            break;
-                        case "api":
-                        case "python-api-step":
-                        case "js-api-step":
-                            {
-                                await this.processApiUpdates(filePath, fileType);
-                            }
-                            break;
-                    }
-                    break;
-                }
-                case "unlink": {
-                    if (filePath in this.apiFiles) {
-                        const api = this.apiFiles[filePath];
-                        delete this.apiFiles[filePath];
-                        if (!api || !this.sourceTracker)
-                            break;
-                        const scopeId = await this.sourceTracker.deleteApi({
-                            pageName: api.pageName,
-                            apiName: api.apiPb.metadata.name,
-                        });
-                        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-                        await this.writeChanges(changes);
-                        this.emit("apiManualDelete", {
-                            api: {
-                                id: getClientApiId(api.apiPb.metadata.name, api.pageName),
-                                apiName: api.apiPb.metadata.name,
-                                // TODO(saksham): get pagename more defensively
-                                pageName: getPageName(filePath),
-                                scopeId,
-                            },
-                        });
-                    }
-                    if (filePath in this.tsFiles) {
-                        delete this.tsFiles[filePath];
-                        this.sourceTracker?.removeFile(filePath);
-                        this.emit("deletePage", filePath);
-                    }
-                    break;
-                }
-            }
-        };
-        watcher.on("all", async (event, filePath) => {
-            const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
-            switch (fileType?.type) {
-                case "tsx":
-                case "scope":
-                    // these can be awaited to ensure sequential execution
-                    this.fsOperationQueue.enqueue(async () => {
-                        return await handleFileChange(event, filePath);
-                    });
-                    break;
-                default:
-                    // some files including APIs cannot be awaited currently
-                    this.fsOperationQueue.enqueue(async () => {
-                        void handleFileChange(event, filePath);
-                    });
+                this.updateInternalApiData(content, path);
             }
         });
-    }
-    getApiFiles() {
-        return this.formatApisToClientApis(this.apiFiles);
-    }
-    getTsFilePaths() {
-        return Object.keys(this.tsFiles);
-    }
-    getSourceTracker() {
-        return this.sourceTracker;
-    }
-    async writeFile(path, content, kind) {
-        if (kind === "ts") {
-            // happens eagerly regardless of error - possible desync
-            this.tsFiles[path] = content;
-            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
-            await this.fsOperationQueue.enqueue(async () => {
-                await fs.writeFile(path, content);
-            });
-        }
-        else if (kind === "api") {
-            const currentApiFile = this.apiFiles[path];
-            const apiPb = yaml.parse(content);
-            // Client APIs have id, but server APIs don't
-            if (apiPb.metadata.id) {
-                delete apiPb.metadata.id;
-            }
-            const stepPathMap = currentApiFile?.stepPathMap ?? {};
-            this.updateApi({ api: apiPb, stepPathMap }, path);
-            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
-            await this.fsOperationQueue.enqueue(async () => {
-                await writeApiFiles({ apiPb }, "api", path.split("/").slice(0, -1).join("/"), false, [], [], { extractLargeSourceFiles: true, minLinesForExtraction: 1 }, new Set(Object.keys(this.apiFiles)), stepPathMap);
-                // stepPathMap will be generated after the write when the file doesn't exist
-                if (this.apiFiles[path]) {
-                    this.apiFiles[path].stepPathMap = stepPathMap;
-                }
-            });
-        }
-    }
-    readFile(path) {
-        return this.tsFiles[path];
+        watcher.on("all", this.handleFileChange);
     }
     initializeSourceTracker() {
         this.sourceTracker = new SourceTracker(this._tracer);
@@ -496,6 +319,235 @@ export class FileSyncManager extends TracedEventEmitter {
             return null;
         }
     }
+    enableOperationsQueue() {
+        this.operationProcessor.enable();
+    }
+    disableOperationsQueue() {
+        this.operationProcessor.disable();
+    }
+    async flushOperations() {
+        await this.operationProcessor.flush();
+    }
+    // MARK: file change handling
+    handleFileChange = async (event, filePath) => {
+        const logger = getLogger();
+        logger.info(`File changed: ${filePath}, event: ${event}`);
+        const rootPath = this.rootDir;
+        if (!rootPath) {
+            throw new Error("Root directory not set");
+        }
+        // Skip directory events
+        if (event === "addDir" || event === "unlinkDir") {
+            return;
+        }
+        const routePath = path.join(rootPath, ROUTES_FILE);
+        const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
+        // Only handle files we care about and that are in our root path
+        if (!fileType || !filePath.startsWith(rootPath)) {
+            return;
+        }
+        // Queue the operation based on the event type
+        switch (event) {
+            case "add": {
+                const data = await readFile(filePath);
+                if (typeof data !== "string")
+                    return;
+                const isPage = isPageFilePath(filePath);
+                if (isPage) {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        execute: async () => {
+                            const file = await readFile(filePath);
+                            if (!file) {
+                                logger.error(`Failed to read file: ${filePath}`);
+                                return;
+                            }
+                            if (!(filePath in this.tsFiles)) {
+                                this.tsFiles[filePath] = file;
+                                this.handleNonVisualChangeByDeletingIds(filePath, file);
+                                this.emit("addPage", filePath);
+                            }
+                        },
+                    });
+                }
+                else {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        execute: async () => {
+                            switch (fileType.type) {
+                                case "api":
+                                case "python-api-step":
+                                case "js-api-step": {
+                                    await this.processApiFileUpdates(filePath, fileType);
+                                    break;
+                                }
+                            }
+                        },
+                    });
+                }
+                break;
+            }
+            case "change": {
+                if (filePath === routePath) {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        priority: true,
+                        execute: async () => {
+                            try {
+                                const data = JSON.parse((await readFile(filePath)) ?? "{}");
+                                if (!isEqual(this.routes, data)) {
+                                    this.routes = data;
+                                    this.emit("routesChanged", this.routes);
+                                }
+                            }
+                            catch (e) {
+                                logger.error("Error parsing routes file", getErrorMeta(e));
+                            }
+                        },
+                    });
+                    return;
+                }
+                else {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        execute: async () => {
+                            const fileType = SUPPORTED_FILETYPES.find((f) => filePath.endsWith(f.extension));
+                            if (!fileType || !filePath.startsWith(rootPath)) {
+                                return;
+                            }
+                            const data = await readFile(filePath);
+                            if (typeof data !== "string")
+                                return;
+                            switch (fileType.type) {
+                                case "tsx":
+                                case "scope":
+                                    {
+                                        if (!(filePath in this.tsFiles &&
+                                            this.tsFiles[filePath] === data)) {
+                                            logger.info(`File changed: ${filePath} updating AST tracker`);
+                                            this.tsFiles[filePath] = data;
+                                            // only update the source tracker if the file is different
+                                            this.handleNonVisualChangeByDeletingIds(filePath, data);
+                                            this.emit("fileChanged", filePath, data, true);
+                                        }
+                                        else {
+                                            logger.info(`File unchanged from last tracked state: ${filePath}`);
+                                            this.emit("fileChanged", filePath, data, false);
+                                        }
+                                    }
+                                    break;
+                                case "api":
+                                case "python-api-step":
+                                case "js-api-step":
+                                    {
+                                        await this.processApiFileUpdates(filePath, fileType);
+                                    }
+                                    break;
+                            }
+                        },
+                    });
+                }
+                break;
+            }
+            case "unlink": {
+                if (filePath in this.tsFiles) {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        execute: async () => {
+                            await this.deleteTsFile(filePath);
+                        },
+                    });
+                }
+                else if (filePath in this.apiFiles) {
+                    void this.operationProcessor.addOperation({
+                        metadata: {
+                            filePath,
+                        },
+                        execute: async () => {
+                            await this.removeApiData(filePath);
+                        },
+                    });
+                }
+                break;
+            }
+        }
+    };
+    async deleteTsFile(filePath) {
+        delete this.tsFiles[filePath];
+        this.sourceTracker?.removeFile(filePath);
+        if (isPageFilePath(filePath)) {
+            this.emit("deletePage", filePath);
+        }
+    }
+    getTsFilePaths() {
+        return Object.keys(this.tsFiles);
+    }
+    getSourceTracker() {
+        return this.sourceTracker;
+    }
+    // MARK: fs read/write
+    async writeFile(path, content, kind) {
+        if (kind === "ts") {
+            // happens eagerly regardless of error - possible desync
+            this.tsFiles[path] = content;
+            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
+            await this.fsOperationQueue.enqueue(async () => {
+                await fs.writeFile(path, content);
+            });
+        }
+        else if (kind === "api") {
+            const currentApiFile = this.apiFiles[path];
+            const apiPb = yaml.parse(content);
+            // Client APIs have id, but server APIs don't
+            if (apiPb.metadata.id) {
+                delete apiPb.metadata.id;
+            }
+            const stepPathMap = currentApiFile?.stepPathMap ?? {};
+            this.updateInternalApiData({ api: apiPb, stepPathMap }, path);
+            // TODO(george): as an optimization, we could update the memory snapshot of the file that SyncService is holding
+            await this.fsOperationQueue.enqueue(async () => {
+                await writeApiFiles({ apiPb }, "api", path.split("/").slice(0, -1).join("/"), false, [], [], { extractLargeSourceFiles: true, minLinesForExtraction: 1 }, new Set(Object.keys(this.apiFiles)), stepPathMap);
+                // stepPathMap will be generated after the write when the file doesn't exist
+                if (this.apiFiles[path]) {
+                    this.apiFiles[path].stepPathMap = stepPathMap;
+                }
+            });
+        }
+    }
+    readFile(path) {
+        return this.tsFiles[path];
+    }
+    async addRoute(route, filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        this.routes[route] = { file: this.getRelativeRoutePath(filePath) };
+        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
+    }
+    async removeRoute(filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        const relativeFilePath = this.getRelativeRoutePath(filePath);
+        this.routes = Object.fromEntries(Object.entries(this.routes).filter(([_, value]) => value.file !== relativeFilePath));
+        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
+    }
+    async writeChanges(changes, callback) {
+        return Promise.all(changes.map(async ({ fileName, source, kind }) => {
+            await this.writeFile(fileName, source, kind);
+            return callback?.(fileName, source);
+        }));
+    }
     getLocalBindingEntities(path) {
         const logger = getLogger();
         const files = this.sourceTracker?.getCurrentFiles();
@@ -523,8 +575,36 @@ export class FileSyncManager extends TracedEventEmitter {
         });
         return Array.from(localBindingEntities);
     }
-    pendingTransactions = new Set();
-    processedTransactions = [];
+    getPageRoots(filePath) {
+        const scopeFilePath = path.join(path.dirname(filePath), "index.tsx");
+        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
+        if (!currentFile) {
+            return null;
+        }
+        return getPageRoots(filePath, currentFile);
+    }
+    getScope(filePath) {
+        // get the scope.ts file in the same directory as path
+        const scopeFilePath = path.join(path.dirname(filePath), SCOPE_FILE);
+        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
+        if (!currentFile) {
+            console.log("File not found", scopeFilePath);
+            return null;
+        }
+        const scope = getScope(scopeFilePath, currentFile);
+        return scope;
+    }
+    getRoutes() {
+        const routes = [];
+        for (const [path, { file }] of Object.entries(this.routes)) {
+            routes.push({
+                path,
+                component: file,
+            });
+        }
+        return routes;
+    }
+    // MARK: transaction handling
     flushTransactions = () => {
         // TODO do something more sophisticated than this.
         this.processedTransactions = this.processedTransactions.slice(-20);
@@ -537,13 +617,13 @@ export class FileSyncManager extends TracedEventEmitter {
         }
         this.pendingTransactions.add(transactionId);
     };
-    transactionNonce = Date.now();
     // Until we have a sophisticated way to track ALL operations, including non-UI initiated ops, we will use a nonce
     // TODO https://github.com/superblocksteam/superblocks/pull/11788
     getProcessedTransactionsWithNonce() {
         const nonce = `t-${this.transactionNonce++}`;
         return this.processedTransactions.concat(nonce);
     }
+    // MARK: editor operations
     handleCreatePage = async (payload) => {
         const { name } = payload;
         if (!this.rootDir) {
@@ -560,7 +640,9 @@ function Page() {
 
 export default registerPage(Page, { name: "${name}" });
 `;
-        await fs.mkdir(pagePath, { recursive: true });
+        await this.fsOperationQueue.enqueue(async () => {
+            await fs.mkdir(pagePath, { recursive: true });
+        });
         await this.writeFile(pageIndexPath, pageContent, "ts");
         await this.handleNonVisualChangeByDeletingIds(pageIndexPath, pageContent);
         this.emit("addPage", pageIndexPath);
@@ -568,11 +650,11 @@ export default registerPage(Page, { name: "${name}" });
     handleReparent = async (payload, writeFile = true) => {
         const { from, to, changedProps, transaction } = payload;
         this.trackTransaction(transaction?.id);
-        await this.sourceTracker?.setProperties({
+        this.sourceTracker?.setProperties({
             source: from.source,
             changes: changedProps ?? {},
         });
-        await this.sourceTracker?.moveElement({
+        this.sourceTracker?.moveElement({
             from,
             to,
         });
@@ -586,7 +668,7 @@ export default registerPage(Page, { name: "${name}" });
     };
     handleCreateComponent = async (payload, writeFile = true) => {
         this.trackTransaction(payload.transaction?.id);
-        const sourceId = await this.sourceTracker?.addElement(payload);
+        const sourceId = this.sourceTracker?.addElement(payload);
         if (writeFile) {
             const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
             await this.writeChanges(changes ?? [], (fileName) => {
@@ -600,7 +682,7 @@ export default registerPage(Page, { name: "${name}" });
         const { elements, transaction } = payload;
         this.trackTransaction(transaction?.id);
         for (const element of elements) {
-            await this.sourceTracker?.deleteElement({
+            this.sourceTracker?.deleteElement({
                 source: element.source,
                 scopeName: element.scopeName,
             });
@@ -616,7 +698,7 @@ export default registerPage(Page, { name: "${name}" });
     handleSetProperty = async (payload, writeFile = true) => {
         const { element: { source }, property, value, transaction, } = payload;
         this.trackTransaction(transaction?.id);
-        await this.sourceTracker?.setProperty({
+        this.sourceTracker?.setProperty({
             source,
             property,
             info: value,
@@ -631,7 +713,7 @@ export default registerPage(Page, { name: "${name}" });
     handleSetProperties = async (payload, writeFile = true) => {
         const { element: { source }, properties, transaction, } = payload;
         this.trackTransaction(transaction?.id);
-        await this.sourceTracker?.setProperties({
+        this.sourceTracker?.setProperties({
             source,
             changes: properties,
         });
@@ -687,184 +769,9 @@ export default registerPage(Page, { name: "${name}" });
         this.flushTransactions();
         return returnValues;
     };
-    handleUpdateApi = async (payload) => {
-        const { api } = payload;
-        if (!this.sourceTracker) {
-            throw new Error("Source tracker not initialized");
-        }
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        if (!api.pageName) {
-            throw new Error("API page name is not set");
-        }
-        const apiName = api.apiPb.metadata.name;
-        if (!apiName) {
-            throw new Error("API name is not set");
-        }
-        const apiDir = path.join(this.rootDir, "pages", api.pageName, "apis", apiName);
-        const apiPath = path.join(apiDir, "api.yaml");
-        const isNewApi = !this.getApiFiles()[apiPath];
-        try {
-            const stats = await fs.stat(apiDir);
-            if (!stats.isDirectory()) {
-                await fs.mkdir(apiDir, { recursive: true });
-            }
-        }
-        catch {
-            await fs.mkdir(apiDir, { recursive: true });
-        }
-        await this.writeFile(apiPath, yaml.stringify(api.apiPb), "api");
-        const generationNumber = this.generationNumberSequence.next();
-        const apiDef = this.createClientApi(api);
-        let scopeId = "";
-        if (isNewApi) {
-            scopeId = await this.createScopedApi(api);
-        }
-        else {
-            const scopeDef = this.sourceTracker.getScopeDefinitionForPage(api.pageName);
-            scopeId = scopeDef?.id ?? "";
-        }
-        this.emit("apiUpdate", { api: apiDef, scopeId });
-        return { api: apiDef, scopeId, generationNumber };
-    };
-    createScopedApi = async (api) => {
-        if (!this.sourceTracker) {
-            throw new Error("Source tracker not initialized");
-        }
-        // We want to add the API entity to our scope, but we do not want to emit entity events, because
-        // the API update event handles this particular side effect.
-        const scopeId = await this.sourceTracker.addApi({
-            pageName: api.pageName,
-            apiName: api.apiPb.metadata.name,
-        });
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-        return scopeId;
-    };
-    handleDeleteApi = async (payload) => {
-        const logger = getLogger();
-        const { apis } = payload;
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const rootDir = this.rootDir;
-        const executeDeleteApis = Promise.all(apis.map(({ apiName, pageName }) => {
-            return new Promise(
-            // eslint-disable-next-line no-async-promise-executor
-            async (resolve) => {
-                const apiFilePath = path.join(rootDir, "pages", pageName, "apis", apiName, "api.yaml");
-                const api = this.apiFiles[apiFilePath];
-                if (!api || !this.sourceTracker) {
-                    return resolve(undefined);
-                }
-                const apiDir = path.join(rootDir, "pages", pageName, "apis", apiName);
-                try {
-                    const stats = await fs.stat(apiDir);
-                    if (stats.isDirectory()) {
-                        await fs.rmdir(apiDir, { recursive: true });
-                    }
-                    delete this.apiFiles[apiFilePath];
-                    const scopeId = await this.sourceTracker.deleteApi({
-                        pageName,
-                        apiName,
-                    });
-                    resolve({
-                        apiName,
-                        pageName,
-                        scopeId,
-                    });
-                }
-                catch (e) {
-                    logger.warn(`Could not delete api ${apiFilePath} ${JSON.stringify(e)}`);
-                    resolve(undefined);
-                }
-                resolve(undefined);
-            });
-        }));
-        const deletedApis = (await executeDeleteApis).filter((api) => api !== undefined);
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-        this.emit("apiDelete", {
-            apis: deletedApis.filter((api) => api !== undefined),
-        });
-        return { deletedApis };
-    };
-    handleRenameApi = async (payload) => {
-        const { oldName, newName, pageName } = payload;
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
-        }
-        const pagePath = getPageFolder(this.rootDir, pageName);
-        const existingApiFolder = path.join(this.rootDir, "pages", pageName, "apis", oldName);
-        const newApiFolder = path.join(this.rootDir, "pages", pageName, "apis", newName);
-        const files = this.sourceTracker?.getCurrentFiles();
-        const file = files?.[path.join(pagePath, "index.tsx")];
-        if (!file || !file.ast) {
-            throw new Error(`Page ${pageName} not found`);
-        }
-        const apiFilePath = path.join(existingApiFolder, "api.yaml");
-        const apiDef = this.apiFiles[apiFilePath];
-        if (!apiDef) {
-            throw new Error(`API ${oldName} not found`);
-        }
-        const otherPageAPIs = structuredClone(Object.keys(this.apiFiles)
-            .filter((path) => path.startsWith(pagePath) && path !== apiFilePath)
-            .map((path) => ({
-            api: { apiPb: this.apiFiles[path]?.apiPb },
-            filePath: path,
-        }))
-            .filter((api) => !!api.api));
-        const newApiFolderExists = await fs.stat(newApiFolder).catch(() => false);
-        if (newApiFolderExists) {
-            throw new Error(`API ${newName} already exists`);
-        }
-        this.watcher?.unwatch(existingApiFolder);
-        this.watcher?.unwatch(newApiFolder);
-        apiDef.apiPb.metadata.name = newName;
-        await fs.rename(existingApiFolder, newApiFolder);
-        const scopeDef = this.sourceTracker?.getScopeDefinitionForPage(pageName);
-        if (!scopeDef) {
-            throw new Error(`Scope definition not found for API`);
-        }
-        this.sourceTracker?.renameEntity({
-            oldName,
-            newName,
-            entityId: scopeDef.scopeNameToEntityId[oldName],
-        });
-        delete this.apiFiles[apiFilePath];
-        this.writeFile(path.join(newApiFolder, "api.yaml"), yaml.stringify(apiDef.apiPb), "api");
-        this.watcher?.add(existingApiFolder);
-        this.watcher?.add(newApiFolder);
-        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
-        await this.writeChanges(changes);
-        this.emit("apiManualUpdate", {
-            api: this.createClientApi(apiDef),
-            oldName,
-            pageName,
-            scopeId: scopeDef?.id ?? "",
-        }, true);
-        await this.renameManager.renameEntityInApis({
-            oldName,
-            newName,
-            apis: otherPageAPIs,
-        });
-        // only save the APIs that have changed
-        await Promise.all(otherPageAPIs.map(async ({ api, filePath }) => {
-            if (isEqual(api?.apiPb, this.apiFiles[filePath]?.apiPb)) {
-                return Promise.resolve();
-            }
-            await this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
-            this.emit("apiManualUpdate", {
-                api: this.createClientApi(api),
-                pageName,
-                scopeId: scopeDef?.id ?? "",
-            }, true);
-            // TODO: Should I delete here?
-        }));
-    };
+    // MARK: entity operations
     handleAddEntity = async (payload) => {
-        await this.sourceTracker?.addEntity({
+        this.sourceTracker?.addEntity({
             scopeId: payload.scopeId,
             entity: {
                 type: payload.type,
@@ -878,7 +785,7 @@ export default registerPage(Page, { name: "${name}" });
         });
     };
     handleUpdateEntity = async (payload) => {
-        await this.sourceTracker?.updateEntity({
+        this.sourceTracker?.updateEntity({
             entityId: payload.entityId,
             updates: payload.updates,
         });
@@ -888,7 +795,7 @@ export default registerPage(Page, { name: "${name}" });
         });
     };
     handleDeleteEntity = async (payload) => {
-        const deletedEntityName = await this.sourceTracker?.deleteEntity({
+        const deletedEntityName = this.sourceTracker?.deleteEntity({
             entityId: payload.entityId,
         });
         const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
@@ -904,13 +811,14 @@ export default registerPage(Page, { name: "${name}" });
             throw new Error("Root directory not set");
         }
         const filePath = path.join(this.rootDir, APP_THEME_FILE_NAME);
-        await this.sourceTracker?.updateTheme({
+        this.sourceTracker?.updateTheme({
             themeFilePath: filePath,
             theme,
         });
         const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
         await this.writeChanges(changes);
     };
+    // MARK: rename operations
     handleRenameElement = async (payload) => {
         if (payload.kind === "component") {
             return this.handleRenameComponent(payload);
@@ -942,7 +850,7 @@ export default registerPage(Page, { name: "${name}" });
     };
     handleRenameEntity = async (payload) => {
         const { elementId, newName, oldName, scopeName } = payload;
-        await this.sourceTracker?.renameEntity({
+        this.sourceTracker?.renameEntity({
             entityId: elementId,
             oldName,
             newName,
@@ -994,98 +902,204 @@ export default registerPage(Page, { name: "${name}" });
         this.watcher?.add(newPageFolder);
         this.watcher?.add(oldPageFolder);
     };
-    getPageRoots(filePath) {
-        const scopeFilePath = path.join(path.dirname(filePath), "index.tsx");
-        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
-        if (!currentFile) {
-            return null;
+    getRelativeRoutePath(filePath) {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
         }
-        return getPageRoots(filePath, currentFile);
+        // no leading slash
+        return path.relative(path.join(this.rootDir, PAGES_DIRECTORY), filePath);
     }
-    getScope(filePath) {
-        // get the scope.ts file in the same directory as path
-        const scopeFilePath = path.join(path.dirname(filePath), SCOPE_FILE);
-        const currentFile = this.sourceTracker?.getCurrentFiles()[scopeFilePath];
-        if (!currentFile) {
-            console.log("File not found", scopeFilePath);
-            return null;
+    // MARK: API operations
+    handleUpdateApi = async (payload) => {
+        const { api } = payload;
+        if (!this.sourceTracker) {
+            throw new Error("Source tracker not initialized");
         }
-        const scope = getScope(scopeFilePath, currentFile);
-        return scope;
-    }
-    getRoutes() {
-        const routes = [];
-        for (const [path, { file }] of Object.entries(this.routes)) {
-            routes.push({
-                path,
-                component: file,
-            });
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
         }
-        return routes;
-    }
-    async addRoute(route, filePath) {
+        if (!api.pageName) {
+            throw new Error("API page name is not set");
+        }
+        const apiName = api.apiPb.metadata.name;
+        if (!apiName) {
+            throw new Error("API name is not set");
+        }
+        const apiFilePath = getApiFilePath(this.rootDir, api.pageName, apiName);
+        const apiDir = path.dirname(apiFilePath);
+        const isNewApi = !this.getApiFiles()[apiFilePath];
+        try {
+            const stats = await fs.stat(apiDir);
+            if (!stats.isDirectory()) {
+                await this.fsOperationQueue.enqueue(async () => await fs.mkdir(apiDir, { recursive: true }));
+            }
+        }
+        catch {
+            await this.fsOperationQueue.enqueue(async () => await fs.mkdir(apiDir, { recursive: true }));
+        }
+        await this.writeFile(apiFilePath, yaml.stringify(api.apiPb), "api");
+        const generationNumber = this.generationNumberSequence.next();
+        const apiDef = this.createClientApi(api);
+        let scopeId = "";
+        if (isNewApi) {
+            scopeId = await this.addApiToScope(api);
+        }
+        else {
+            const scopeDef = this.sourceTracker.getScopeDefinitionForPage(api.pageName);
+            scopeId = scopeDef?.id ?? "";
+        }
+        this.emit("apiUpdate", { api: apiDef, scopeId });
+        return { api: apiDef, scopeId, generationNumber };
+    };
+    handleDeleteApi = async (payload) => {
+        const logger = getLogger();
+        const { apis } = payload;
         if (!this.rootDir) {
             throw new Error("Root directory not set");
         }
-        this.routes[route] = { file: this.getRelativeRoutePath(filePath) };
-        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
-    }
-    async removeRoute(filePath) {
+        const rootDir = this.rootDir;
+        const deletedApis = [];
+        for (const { apiName, pageName } of apis) {
+            const apiFilePath = getApiFilePath(rootDir, pageName, apiName);
+            const api = this.apiFiles[apiFilePath];
+            if (!api || !this.sourceTracker) {
+                continue;
+            }
+            const apiDir = path.dirname(apiFilePath);
+            try {
+                const stats = await fs.stat(apiDir);
+                if (stats.isDirectory()) {
+                    await fs.rmdir(apiDir, { recursive: true });
+                }
+                delete this.apiFiles[apiFilePath];
+                const scopeId = this.sourceTracker.deleteApi({
+                    pageName,
+                    apiName,
+                });
+                deletedApis.push({
+                    apiName,
+                    pageName,
+                    scopeId,
+                });
+            }
+            catch (e) {
+                logger.warn(`Could not delete api ${apiFilePath} ${JSON.stringify(e)}`);
+            }
+        }
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        this.emit("apiDelete", {
+            apis: deletedApis.filter((api) => api !== undefined),
+        });
+        return { deletedApis };
+    };
+    handleRenameApi = async (payload) => {
+        const { oldName, newName, pageName } = payload;
         if (!this.rootDir) {
             throw new Error("Root directory not set");
         }
-        const relativeFilePath = this.getRelativeRoutePath(filePath);
-        this.routes = Object.fromEntries(Object.entries(this.routes).filter(([_, value]) => value.file !== relativeFilePath));
-        await this.writeFile(path.join(this.rootDir, ROUTES_FILE), JSON.stringify(this.routes, null, 2));
-    }
-    async writeChanges(changes, callback) {
-        return Promise.all(changes.map(async ({ fileName, source, kind }) => {
-            await this.writeFile(fileName, source, kind);
-            return callback?.(fileName, source);
-        }));
-    }
-    getNodeForWidgetSourceId(id) {
-        return this.sourceTracker?.getElementToLocation(id);
-    }
-    getAstForWidgetSourceId(id) {
-        const filePath = this.sourceTracker?.getElementToFilePath(id);
-        if (!filePath) {
-            return null;
+        const pagePath = getPageFolder(this.rootDir, pageName);
+        const existingApiFolder = path.join(this.rootDir, "pages", pageName, "apis", oldName);
+        const newApiFolder = path.join(this.rootDir, "pages", pageName, "apis", newName);
+        const files = this.sourceTracker?.getCurrentFiles();
+        const file = files?.[path.join(pagePath, "index.tsx")];
+        if (!file || !file.ast) {
+            throw new Error(`Page ${pageName} not found`);
         }
-        return this.sourceTracker?.getCurrentFiles()[filePath]?.ast;
-    }
-    renameIdentifierInApis = async ({ elementId, oldName, newName, parentBinding, }) => {
-        const apisInScope = structuredClone(this.getApisInScope(elementId));
+        const apiFilePath = path.join(existingApiFolder, "api.yaml");
+        const apiDef = this.apiFiles[apiFilePath];
+        if (!apiDef) {
+            throw new Error(`API ${oldName} not found`);
+        }
+        const otherPageAPIs = structuredClone(Object.keys(this.apiFiles)
+            .filter((path) => path.startsWith(pagePath) && path !== apiFilePath)
+            .map((path) => ({
+            api: { apiPb: this.apiFiles[path]?.apiPb },
+            filePath: path,
+        }))
+            .filter((api) => !!api.api));
+        const newApiFolderExists = await fs.stat(newApiFolder).catch(() => false);
+        if (newApiFolderExists) {
+            throw new Error(`API ${newName} already exists`);
+        }
+        this.watcher?.unwatch(existingApiFolder);
+        this.watcher?.unwatch(newApiFolder);
+        apiDef.apiPb.metadata.name = newName;
+        await fs.rename(existingApiFolder, newApiFolder);
+        const scopeDef = this.sourceTracker?.getScopeDefinitionForPage(pageName);
+        if (!scopeDef) {
+            throw new Error(`Scope definition not found for API`);
+        }
+        this.sourceTracker?.renameEntity({
+            oldName,
+            newName,
+            entityId: scopeDef.scopeNameToEntityId[oldName],
+        });
+        delete this.apiFiles[apiFilePath];
+        this.writeFile(path.join(newApiFolder, "api.yaml"), yaml.stringify(apiDef.apiPb), "api");
+        this.watcher?.add(existingApiFolder);
+        this.watcher?.add(newApiFolder);
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        this.emit("apiManualUpdate", {
+            api: this.createClientApi(apiDef),
+            oldName,
+            pageName,
+            scopeId: scopeDef?.id ?? "",
+        }, true);
         await this.renameManager.renameEntityInApis({
             oldName,
             newName,
-            parentBinding,
-            apis: apisInScope,
+            apis: otherPageAPIs,
         });
         // only save the APIs that have changed
-        await Promise.all(apisInScope.map(({ api, filePath }) => {
+        await Promise.all(otherPageAPIs.map(async ({ api, filePath }) => {
             if (isEqual(api?.apiPb, this.apiFiles[filePath]?.apiPb)) {
                 return Promise.resolve();
             }
-            return this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
+            await this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
+            this.emit("apiManualUpdate", {
+                api: this.createClientApi(api),
+                pageName,
+                scopeId: scopeDef?.id ?? "",
+            }, true);
+            // TODO: Should I delete here?
         }));
     };
-    getApisInScope = (elementId) => {
-        const filePath = this.sourceTracker?.getElementToFilePath(elementId);
-        if (!filePath) {
-            return [];
-        }
-        const pagePath = path.dirname(filePath);
-        return Object.entries(this.apiFiles)
-            .filter(([filePath]) => filePath.includes(pagePath))
-            .map(([filePath, api]) => ({ api, filePath }));
-    };
-    getRelativeRoutePath(filePath) {
-        if (!this.rootDir) {
-            throw new Error("Root directory not set");
+    async removeApiData(filePath) {
+        const api = this.apiFiles[filePath];
+        if (!api) {
+            return;
         }
-        // no leading slash
-        return path.relative(path.join(this.rootDir, PAGES_DIRECTORY), filePath);
+        delete this.apiFiles[filePath];
+        this.sourceTracker?.deleteApi({
+            pageName: api.pageName,
+            apiName: api.apiPb.metadata.name,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        const scopeId = this.sourceTracker?.getScopeDefinitionForPage(api.pageName)?.id;
+        this.emit("apiManualDelete", {
+            api: {
+                id: getClientApiId(api.apiPb.metadata.name, api.pageName),
+                apiName: api.apiPb.metadata.name,
+                // TODO(saksham): get pagename more defensively
+                pageName: getPageName(filePath),
+                scopeId,
+            },
+        });
+    }
+    getApiFiles() {
+        return Object.keys(this.apiFiles).reduce((acc, key) => {
+            if (!this.apiFiles[key]) {
+                return acc;
+            }
+            acc[key] = {
+                api: this.createClientApi(this.apiFiles[key]),
+                scopeId: this.apiFiles[key].scopeId,
+            };
+            return acc;
+        }, {});
     }
     // Utilities for converting server API format to Client API format
     // We internally save the API as the server does, but we return should always return it
@@ -1102,19 +1116,21 @@ export default registerPage(Page, { name: "${name}" });
             },
         };
     }
-    formatApisToClientApis(serverApis) {
-        return Object.keys(serverApis).reduce((acc, key) => {
-            if (!serverApis[key]) {
-                return acc;
-            }
-            acc[key] = {
-                api: this.createClientApi(serverApis[key]),
-                scopeId: serverApis[key].scopeId,
-            };
-            return acc;
-        }, {});
-    }
-    async processApiUpdates(filePath, fileType) {
+    addApiToScope = async (api) => {
+        if (!this.sourceTracker) {
+            throw new Error("Source tracker not initialized");
+        }
+        // We want to add the API entity to our scope, but we do not want to emit entity events, because
+        // the API update event handles this particular side effect.
+        const scopeId = await this.sourceTracker.addApi({
+            pageName: api.pageName,
+            apiName: api.apiPb.metadata.name,
+        });
+        const changes = (await this.sourceTracker?.getAndFlushChanges()) ?? [];
+        await this.writeChanges(changes);
+        return scopeId;
+    };
+    async processApiFileUpdates(filePath, fileType) {
         let yamlPath = filePath;
         if (fileType.type === "python-api-step" ||
             fileType.type === "js-api-step") {
@@ -1132,7 +1148,7 @@ export default registerPage(Page, { name: "${name}" });
             const parsedData = { apiPb: apiContent?.api };
             if (!(yamlPath in this.apiFiles &&
                 isEqual(this.apiFiles[yamlPath]?.apiPb, parsedData.apiPb))) {
-                const { updatedApi, pageName, isNewApi } = this.updateApi({
+                const { updatedApi, pageName, isNewApi } = this.updateInternalApiData({
                     api: parsedData.apiPb,
                     stepPathMap: apiContent.stepPathMap,
                 }, yamlPath);
@@ -1143,7 +1159,7 @@ export default registerPage(Page, { name: "${name}" });
                     return;
                 }
                 if (isNewApi) {
-                    await this.createScopedApi(updatedApi);
+                    await this.addApiToScope(updatedApi);
                 }
                 this.emit("apiManualUpdate", {
                     api: this.createClientApi(updatedApi),
@@ -1159,9 +1175,56 @@ export default registerPage(Page, { name: "${name}" });
             logger.error(`Error updating API: ${yamlPath}, error: ${error}`);
         }
     }
+    updateInternalApiData = (content, path) => {
+        if (!this.rootDir) {
+            throw new Error("Root directory not set");
+        }
+        const { api: apiContents, stepPathMap } = content;
+        const pageName = getPageName(path);
+        let scopeId = this.sourceTracker?.getScopeDefinitionForPage(pageName)?.id;
+        if (!scopeId) {
+            console.warn("Scope ID not found for API", apiContents.metadata.name);
+            scopeId = "";
+        }
+        const updatedApi = {
+            apiPb: yaml.parse(JSON.stringify(apiContents)),
+            pageName,
+            stepPathMap,
+            scopeId,
+        };
+        const isNewApi = !this.apiFiles[path];
+        this.apiFiles[path] = updatedApi;
+        return { updatedApi, pageName, isNewApi };
+    };
+    renameIdentifierInApis = async ({ elementId, oldName, newName, parentBinding, }) => {
+        const apisInScope = structuredClone(this.getApisInScope(elementId));
+        await this.renameManager.renameEntityInApis({
+            oldName,
+            newName,
+            parentBinding,
+            apis: apisInScope,
+        });
+        // only save the APIs that have changed
+        await Promise.all(apisInScope.map(({ api, filePath }) => {
+            if (isEqual(api?.apiPb, this.apiFiles[filePath]?.apiPb)) {
+                return Promise.resolve();
+            }
+            return this.writeFile(filePath, yaml.stringify(api?.apiPb), "api");
+        }));
+    };
+    getApisInScope = (elementId) => {
+        const filePath = this.sourceTracker?.getElementToFilePath(elementId);
+        if (!filePath) {
+            return [];
+        }
+        const pagePath = path.dirname(filePath);
+        return Object.entries(this.apiFiles)
+            .filter(([filePath]) => filePath.includes(pagePath))
+            .map(([filePath, api]) => ({ api, filePath }));
+    };
 }
 // Add new mock implementation
-export class MockFileSyncManager extends FileSyncManager {
+export class MockFileSyncManager extends FileSystemManager {
     tsFiles = {};
     apiFiles = {};
     async watch(_watcher, _path) {
@@ -1237,7 +1300,7 @@ async function readFiles(dir) {
 const getMergedApiContent = async (path) => {
     return readAppApiYamlFile(path.split("/").slice(0, -1).join("/"));
 };
-const getPageName = (path) => {
+export const getPageName = (path) => {
     const parts = path.split("/");
     const pagesIndex = parts.findIndex((part) => part === "pages");
     if (pagesIndex !== -1 && parts[pagesIndex + 1]) {