@superblocksteam/cli 1.9.2 → 1.10.0

package/dist/common/version-control.js

@@ -1,16 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteResourcesAndUpdateRootConfig = exports.validateLocalResource = exports.extractApiName = exports.isGitRepoDirty = exports.isCI = exports.getHeadCommit = exports.getCurrentGitBranch = exports.getCurrentGitBranchIfGit = exports.getLocalGitRepoState = exports.sortByKey = exports.getMode = exports.removeResourceFromDisk = exports.writeMultiPageApplicationToDisk = exports.writeResourceToDisk = exports.readApiFromDisk = exports.readMultiPageApplicationFromDisk = exports.readApplicationFromDisk = exports.getFileStructureType = exports.atLeastOneSelection = exports.MULTI_SELECT_PROMPT_HELP = exports.SELECT_PROMPT_HELP = exports.modeFlagToViewMode = exports.FileStructureType = exports.modeFlagValuesMap = exports.DEFAULT_BRANCH = exports.DEPLOYED_MODE = exports.MOST_RECENT_COMMIT_MODE = exports.LATEST_EDITS_MODE = void 0;
+exports.deleteResourcesAndUpdateRootConfig = exports.validateLocalResource = exports.writeAppApi = exports.extractApiName = exports.isGitRepoDirty = exports.isCI = exports.getHeadCommit = exports.getCurrentGitBranch = exports.getCurrentGitBranchIfGit = exports.getLocalGitRepoState = exports.sortByKey = exports.getMode = exports.removeResourceFromDisk = exports.addExistingFilePathsForApi = exports.writeMultiPageApplicationToDisk = exports.writeResourceToDisk = exports.readApiFromDisk = exports.readMultiPageApplicationFromDisk = exports.readApplicationFromDisk = exports.getFileStructureType = exports.readAppApiYamlFile = exports.getApiRepresentation = exports.atLeastOneSelection = exports.MULTI_SELECT_PROMPT_HELP = exports.SELECT_PROMPT_HELP = exports.modeFlagToViewMode = exports.FileStructureType = exports.modeFlagValuesMap = exports.DEFAULT_BRANCH = exports.DEPLOYED_MODE = exports.MOST_RECENT_COMMIT_MODE = exports.LATEST_EDITS_MODE = void 0;
 const tslib_1 = require("tslib");
 const https = tslib_1.__importStar(require("https"));
 const node_path_1 = tslib_1.__importDefault(require("node:path"));
 const path_1 = require("path");
+const sdk_1 = require("@superblocksteam/sdk");
 const util_1 = require("@superblocksteam/util");
-const util_2 = require("@superblocksteam/util");
 const colorette_1 = require("colorette");
 const fs = tslib_1.__importStar(require("fs-extra"));
 const lodash_1 = require("lodash");
 const lodash_2 = tslib_1.__importDefault(require("lodash"));
+const semver = tslib_1.__importStar(require("semver"));
 const simple_git_1 = require("simple-git");
 const slugify_1 = tslib_1.__importDefault(require("slugify"));
 const yaml_1 = require("yaml");
@@ -18,6 +19,10 @@ exports.LATEST_EDITS_MODE = "latest-edits";
 exports.MOST_RECENT_COMMIT_MODE = "most-recent-commit";
 exports.DEPLOYED_MODE = "deployed";
 exports.DEFAULT_BRANCH = "main";
+const LANGUAGE_STEP_EXTENSIONS = {
+    javascript: "js",
+    python: "py",
+};
 exports.modeFlagValuesMap = {
     [exports.LATEST_EDITS_MODE]: "Latest edits",
     [exports.MOST_RECENT_COMMIT_MODE]: "Most recent commit",
@@ -50,6 +55,34 @@ const atLeastOneSelection = (value) => {
     return true;
 };
 exports.atLeastOneSelection = atLeastOneSelection;
+const DEFAULT_FILE_VERSION = "0.1.0";
+const SPLIT_LARGE_API_STEPS_VERSION = "0.2.0";
+const LATEST_FILE_VERSION = SPLIT_LARGE_API_STEPS_VERSION;
+function getApiRepresentation(featureFlags, resourceConfig) {
+    var _a;
+    const linesForLargeSteps = (_a = featureFlags.linesForLargeSteps()) !== null && _a !== void 0 ? _a : sdk_1.DEFAULT_LINES_FOR_LARGE_STEPS;
+    if (featureFlags.splitLargeApiStepsEnabled() &&
+        isPostSplitLargeApiSteps(resourceConfig)) {
+        return {
+            extractLargeSourceFiles: true,
+            minLinesForExtraction: linesForLargeSteps,
+        };
+    }
+    // Return the default
+    return {
+        extractLargeSourceFiles: false,
+        minLinesForExtraction: linesForLargeSteps,
+    };
+}
+exports.getApiRepresentation = getApiRepresentation;
+function isPostSplitLargeApiSteps(resourceConfig) {
+    var _a, _b;
+    if (!resourceConfig) {
+        return false;
+    }
+    const version = (_b = (_a = resourceConfig.metadata) === null || _a === void 0 ? void 0 : _a.fileVersion) !== null && _b !== void 0 ? _b : DEFAULT_FILE_VERSION;
+    return semver.compare(version, SPLIT_LARGE_API_STEPS_VERSION) >= 0;
+}
 function slugifyName(originalName) {
     return (0, slugify_1.default)(originalName, {
         replacement: "_",
@@ -95,6 +128,95 @@ async function downloadFile(rootDirectory, filepath, url) {
     }
     return Promise.resolve("");
 }
+async function readAppApiYamlFile(parentDirectory, apiName) {
+    var _a;
+    // The API is either stored in its entirety in a single YAML file, or
+    // or in a subdirectory containing the YAML file and zero or more language-specific files.
+    const apiNameSlug = slugifyName(apiName !== null && apiName !== void 0 ? apiName : "api");
+    const singularApiYamlPath = `${parentDirectory}/${apiNameSlug}.yaml`;
+    const apiDirPath = `${parentDirectory}/${apiNameSlug}`;
+    const nestedApiYamlPath = `${apiDirPath}/${apiNameSlug}.yaml`;
+    // Read the YAML file
+    let yamlPath;
+    let yamlParentPath;
+    if (await fs.pathExists(singularApiYamlPath)) {
+        // The API YAML file is in the parent directory
+        yamlPath = singularApiYamlPath;
+        yamlParentPath = parentDirectory;
+    }
+    else if (await fs.pathExists(nestedApiYamlPath)) {
+        // The API YAML file is nested in an API-specific directory
+        yamlPath = nestedApiYamlPath;
+        yamlParentPath = apiDirPath;
+    }
+    else {
+        throw new Error(`API ${apiName !== null && apiName !== void 0 ? apiName : ""} not found at ${parentDirectory}`);
+    }
+    // That is nested in an API-specific directory
+    const apiDefn = (await readYamlFile(yamlPath));
+    // The API YAML file may or may not have language-specific content in separate files.
+    // Replace any file references with the actual content
+    await resolveLanguageSpecificStepContentFromBlocks(yamlParentPath, (_a = apiDefn.blocks) !== null && _a !== void 0 ? _a : []);
+    return apiDefn;
+}
+exports.readAppApiYamlFile = readAppApiYamlFile;
+async function resolveLanguageSpecificStepContentFromBlocks(parentPath, blocks) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x;
+    for (const block of blocks) {
+        // Handle language-specific step content
+        if (block.step) {
+            const step = block.step;
+            await resolveLanguageSpecificStepContentFromStep(parentPath, step);
+        }
+        // Handle conditional blocks
+        if (block.conditional) {
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_b = (_a = block.conditional.if) === null || _a === void 0 ? void 0 : _a.blocks) !== null && _b !== void 0 ? _b : []);
+            for (const elseIfBlock of block.conditional.elseIf) {
+                await resolveLanguageSpecificStepContentFromBlocks(parentPath, elseIfBlock.blocks);
+            }
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_d = (_c = block.conditional.else) === null || _c === void 0 ? void 0 : _c.blocks) !== null && _d !== void 0 ? _d : []);
+        }
+        // Handle loop blocks
+        await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_f = (_e = block.loop) === null || _e === void 0 ? void 0 : _e.blocks) !== null && _f !== void 0 ? _f : []);
+        // Handle try-catch blocks
+        if (block.tryCatch) {
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_h = (_g = block.tryCatch.try) === null || _g === void 0 ? void 0 : _g.blocks) !== null && _h !== void 0 ? _h : []);
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_k = (_j = block.tryCatch.catch) === null || _j === void 0 ? void 0 : _j.blocks) !== null && _k !== void 0 ? _k : []);
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_m = (_l = block.tryCatch.finally) === null || _l === void 0 ? void 0 : _l.blocks) !== null && _m !== void 0 ? _m : []);
+        }
+        // Handle parallel blocks
+        if (block.parallel) {
+            for (const path of Object.values((_q = (_p = (_o = block.parallel) === null || _o === void 0 ? void 0 : _o.static) === null || _p === void 0 ? void 0 : _p.paths) !== null && _q !== void 0 ? _q : {})) {
+                await resolveLanguageSpecificStepContentFromBlocks(parentPath, path.blocks);
+            }
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_t = (_s = (_r = block.parallel) === null || _r === void 0 ? void 0 : _r.dynamic) === null || _s === void 0 ? void 0 : _s.blocks) !== null && _t !== void 0 ? _t : []);
+        }
+        // Handle stream blocks
+        if (block.stream) {
+            await resolveLanguageSpecificStepContentFromBlocks(parentPath, (_v = (_u = block.stream.process) === null || _u === void 0 ? void 0 : _u.blocks) !== null && _v !== void 0 ? _v : []);
+            await resolveLanguageSpecificStepContentFromStep(parentPath, (_x = (_w = block.stream.trigger) === null || _w === void 0 ? void 0 : _w.step) !== null && _x !== void 0 ? _x : {});
+        }
+    }
+}
+async function resolveLanguageSpecificStepContentFromStep(parentPath, step) {
+    // Handle language-specific step content
+    if (!(step === null || step === void 0 ? void 0 : step.integration) || !LANGUAGE_STEP_EXTENSIONS[step.integration]) {
+        return;
+    }
+    const languageContent = step[step.integration];
+    if (languageContent.body &&
+        typeof languageContent.body === "object" &&
+        "path" in languageContent.body) {
+        const languageFilePath = `${parentPath}/${languageContent.body.path}`;
+        if (await fs.pathExists(languageFilePath)) {
+            // Read the file content
+            const stepBodyContent = await fs.readFile(languageFilePath, "utf8");
+            if (stepBodyContent && languageContent) {
+                languageContent.body = stepBodyContent;
+            }
+        }
+    }
+}
 async function readYamlFile(path) {
     return (0, yaml_1.parse)(await fs.readFile(path, "utf8"));
 }
@@ -120,7 +242,7 @@ async function readApplicationFromDisk(rootPath, existingRelativeLocation) {
     if (await fs.pathExists(apisDirName)) {
         const apiFiles = await fs.readdir(apisDirName);
         for (const apiFile of apiFiles) {
-            const apiContent = await readYamlFile(`${apisDirName}/${apiFile}`);
+            const apiContent = await readAppApiYamlFile(apisDirName, apiFile);
             // This mimics the shape of the ApiV3Dto object
             apis.push({
                 id: apiContent.metadata.id,
@@ -145,12 +267,16 @@ async function readMultiPageApplicationFromDisk(rootPath, existingRelativeLocati
     const pages = [];
     if (await fs.pathExists(pagesDirName)) {
         for (const page of Object.values((_a = superblocksApplicationConfig.pages) !== null && _a !== void 0 ? _a : {})) {
+            // Read in the page definition
             const pageContent = await readYamlFile(`${pagesDirName}/${slugifyName(page.name)}/page.yaml`);
+            // Read in the API definitions for this page
             const pageApisDirName = `${pagesDirName}/${slugifyName(page.name)}/apis`;
             const apis = [];
             if (await fs.pathExists(pageApisDirName)) {
-                for (const apiName of Object.values(page.apis)) {
-                    const apiContent = await readYamlFile(`${pageApisDirName}/${slugifyName(apiName)}.yaml`);
+                // Get the API names from pageApis if it exists, or from apis if it doesn't
+                const pageApiNames = getApiNamesFromPageConfig(page);
+                for (const apiName of pageApiNames) {
+                    const apiContent = await readAppApiYamlFile(pageApisDirName, apiName);
                     // This mimics the shape of the ApiV3Dto object
                     apis.push({
                         id: apiContent.metadata.id,
@@ -172,8 +298,11 @@ async function readMultiPageApplicationFromDisk(rootPath, existingRelativeLocati
     const appApisDirName = `${rootPath}/${existingRelativeLocation}/apis`;
     const apis = [];
     if (await fs.pathExists(appApisDirName)) {
-        for (const apiName of Object.values(superblocksApplicationConfig.apis)) {
-            const apiContent = await readYamlFile(`${appApisDirName}/${slugifyName(apiName)}.yaml`);
+        // Get the API names from appApis if it exists, or from apis if it doesn't
+        const appConfig = superblocksApplicationConfig;
+        const apiNames = getApiNamesFromApplicationConfig(appConfig);
+        for (const apiName of apiNames) {
+            const apiContent = await readAppApiYamlFile(appApisDirName, apiName);
             // This mimics the shape of the ApiV3Dto object
             apis.push({
                 id: apiContent.metadata.id,
@@ -188,15 +317,29 @@ async function readMultiPageApplicationFromDisk(rootPath, existingRelativeLocati
     };
 }
 exports.readMultiPageApplicationFromDisk = readMultiPageApplicationFromDisk;
+function getApiNamesFromApplicationConfig(applicationConfig) {
+    var _a;
+    return applicationConfig.appApis
+        ? Object.values(applicationConfig.appApis).map((api) => api.name)
+        : Object.values((_a = applicationConfig.apis) !== null && _a !== void 0 ? _a : {});
+}
+function getApiNamesFromPageConfig(pageConfig) {
+    var _a;
+    return pageConfig.pageApis
+        ? Object.values(pageConfig.pageApis).map((api) => api.name)
+        : Object.values((_a = pageConfig.apis) !== null && _a !== void 0 ? _a : {});
+}
 async function readApiFromDisk(rootPath, existingRelativeLocation) {
+    const path = `${rootPath}/${existingRelativeLocation}`;
+    const apiContent = await readAppApiYamlFile(path);
     return {
-        apiPb: await readYamlFile(`${rootPath}/${existingRelativeLocation}/api.yaml`),
+        apiPb: apiContent,
     };
 }
 exports.readApiFromDisk = readApiFromDisk;
 // NOTE: If a change is made to how applications are written to disk, please update
 // logic to read applications from disk in the "readApplicationFromDisk" function accordingly.
-async function writeResourceToDisk(resourceType, resourceId, resource, rootPath, existingRelativeLocation) {
+async function writeResourceToDisk(resourceType, resourceId, resource, rootPath, featureFlags, existingRelativeLocation, preferredApiRepresentation) {
     var _a, _b, _c;
     switch (resourceType) {
         case "APPLICATION": {
@@ -221,43 +364,39 @@ async function writeResourceToDisk(resourceType, resourceId, resource, rootPath,
             const apiPromises = [];
             const apisDirName = `${appDirName}/apis`;
             await fs.ensureDir(apisDirName);
+            const existingApplicationConfig = await (0, util_1.getSuperblocksApplicationConfigIfExists)(appDirName);
+            const existingFilePaths = getExistingFilePathsForApplicationApi(existingApplicationConfig, appDirName);
             const newApplicationConfig = {
                 configType: "APPLICATION",
-                apis: {},
                 defaultPageId: (_a = resource.page) === null || _a === void 0 ? void 0 : _a.id,
                 id: resource.application.id,
+                metadata: getResourceConfigMetadata(featureFlags, existingApplicationConfig),
             };
+            const apiRepresentation = preferredApiRepresentation !== null && preferredApiRepresentation !== void 0 ? preferredApiRepresentation : getApiRepresentation(featureFlags, newApplicationConfig);
             if (resource.apis) {
                 for (const api of resource.apis) {
-                    const originalApiName = extractApiName(api);
-                    const apiName = slugifyName(originalApiName);
-                    const apiContent = (0, yaml_1.stringify)(api.apiPb, {
-                        sortMapEntries: true,
-                        blockQuote: "literal",
-                    });
-                    const handleApi = async () => {
-                        await fs.outputFile(`${apisDirName}/${apiName}.yaml`, apiContent);
-                    };
-                    apiPromises.push(handleApi());
-                    newApplicationConfig.apis[api.id] = originalApiName;
+                    const apiInfo = await writeAppApi(api, appDirName, existingFilePaths, apiPromises, apiRepresentation);
+                    if (apiRepresentation.extractLargeSourceFiles) {
+                        if (!newApplicationConfig.appApis) {
+                            newApplicationConfig.appApis = {};
+                        }
+                        newApplicationConfig.appApis[api.apiPb.metadata.id] = apiInfo;
+                    }
+                    else {
+                        if (!newApplicationConfig.apis) {
+                            newApplicationConfig.apis = {};
+                        }
+                        newApplicationConfig.apis[api.apiPb.metadata.id] = apiInfo.name;
+                    }
                 }
                 await Promise.all(apiPromises);
             }
-            const existingApplicationConfig = await (0, util_1.getSuperblocksApplicationConfigIfExists)(appDirName);
-            // iterate through existing apis and remove from disk any that are no longer present
-            if (existingApplicationConfig === null || existingApplicationConfig === void 0 ? void 0 : existingApplicationConfig.apis) {
-                for (const existingApiId of Object.keys(existingApplicationConfig.apis)) {
-                    const existingApiName = existingApplicationConfig.apis[existingApiId];
-                    const existingApiPath = `${apisDirName}/${slugifyName(existingApiName)}.yaml`;
-                    const newApiName = newApplicationConfig.apis[existingApiId];
-                    if (!(existingApiId in newApplicationConfig.apis) ||
-                        slugifyName(newApiName) !== slugifyName(existingApiName)) {
-                        await fs.remove(existingApiPath);
-                    }
-                }
+            // Delete any existing files that were not overwritten
+            for (const filePath of existingFilePaths) {
+                await fs.remove(filePath);
             }
-            await fs.ensureDir(`${appDirName}/${util_2.SUPERBLOCKS_HOME_FOLDER_NAME}`);
-            await fs.writeFile(`${appDirName}/${util_2.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(newApplicationConfig), null, 2));
+            await fs.ensureDir(`${appDirName}/${util_1.SUPERBLOCKS_HOME_FOLDER_NAME}`);
+            await fs.writeFile(`${appDirName}/${util_1.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(newApplicationConfig), null, 2));
             const createdFiles = await Promise.resolve(
             // Defensive check for when application settings are missing componentFiles
             (_c = (_b = resource.componentFiles) === null || _b === void 0 ? void 0 : _b.map((file) => downloadFile(appDirName, file.filename, file.url))) !== null && _c !== void 0 ? _c : []);
@@ -281,17 +420,26 @@ async function writeResourceToDisk(resourceType, resourceId, resource, rootPath,
             if (!(await fs.pathExists(backendDirName))) {
                 await fs.mkdir(backendDirName, { recursive: true });
             }
-            const backendContent = (0, yaml_1.stringify)(resource.apiPb, {
-                sortMapEntries: true,
-                blockQuote: "literal",
-            });
-            await fs.outputFile(`${backendDirName}/api.yaml`, backendContent);
+            const existingBackendConfig = await (0, util_1.getSuperblocksBackendConfigIfExists)(backendDirName);
+            const existingFilePaths = getExistingFilePathsForBackendApi(existingBackendConfig, backendDirName);
             const backendConfig = {
                 id: resourceId,
                 configType: "BACKEND",
+                metadata: getResourceConfigMetadata(featureFlags, existingBackendConfig),
             };
-            await fs.ensureDir(`${backendDirName}/${util_2.SUPERBLOCKS_HOME_FOLDER_NAME}`);
-            await fs.writeFile(`${backendDirName}/${util_2.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(backendConfig), null, 2));
+            const apiRepresentation = preferredApiRepresentation !== null && preferredApiRepresentation !== void 0 ? preferredApiRepresentation : getApiRepresentation(featureFlags, backendConfig);
+            // Write the API file(s)
+            const apiInfo = await writeBackendApi(resource, backendDirName, [], apiRepresentation, existingFilePaths);
+            if (apiRepresentation.extractLargeSourceFiles) {
+                backendConfig.sourceFiles = apiInfo.sourceFiles;
+            }
+            // Write the backend config file
+            await fs.ensureDir(`${backendDirName}/${util_1.SUPERBLOCKS_HOME_FOLDER_NAME}`);
+            await fs.writeFile(`${backendDirName}/${util_1.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(backendConfig), null, 2));
+            // Delete any existing files that were not overwritten
+            for (const filePath of existingFilePaths) {
+                await fs.remove(filePath);
+            }
             return {
                 location: relativeLocation,
                 resourceType: "BACKEND",
@@ -305,8 +453,8 @@ async function writeResourceToDisk(resourceType, resourceId, resource, rootPath,
 exports.writeResourceToDisk = writeResourceToDisk;
 // NOTE: If a change is made to how applications are written to disk, please update
 // logic to read applications from disk in the "readMultiPageApplicationFromDisk" function accordingly.
-async function writeMultiPageApplicationToDisk(resource, rootPath, existingRelativeLocation, migrateFromSinglePage) {
-    var _a, _b, _c, _d;
+async function writeMultiPageApplicationToDisk(resource, rootPath, featureFlags, existingRelativeLocation, migrateFromSinglePage, preferredApiRepresentation) {
+    var _a, _b, _c;
     const parentDirName = "apps";
     const newRelativeLocation = `${parentDirName}/${slugifyName(resource.application.name)}`;
     const relativeLocation = existingRelativeLocation !== null && existingRelativeLocation !== void 0 ? existingRelativeLocation : newRelativeLocation;
@@ -318,11 +466,15 @@ async function writeMultiPageApplicationToDisk(resource, rootPath, existingRelat
         sortMapEntries: true,
     });
     await fs.outputFile(`${appDirName}/application.yaml`, applicationContent);
+    // Find the existing application config and existing file paths
+    const existingApplicationConfig = await (0, util_1.getSuperblocksApplicationConfigIfExists)(appDirName);
+    const existingFilePaths = getExistingFilePathsForApplicationApi(existingApplicationConfig, appDirName);
     const newApplicationConfig = {
         configType: "APPLICATION",
-        apis: {},
         id: resource.application.id,
+        metadata: getResourceConfigMetadata(featureFlags, existingApplicationConfig),
     };
+    const apiRepresentation = preferredApiRepresentation !== null && preferredApiRepresentation !== void 0 ? preferredApiRepresentation : getApiRepresentation(featureFlags, newApplicationConfig);
     newApplicationConfig.pages = {};
     const apiPromises = [];
     for (const page of Object.values(resource.pages)) {
@@ -334,8 +486,8 @@ async function writeMultiPageApplicationToDisk(resource, rootPath, existingRelat
         newApplicationConfig.pages[pageId] = {
             id: page.id,
             name: page.name,
-            apis: {},
         };
+        const pageConfig = newApplicationConfig.pages[pageId];
         const pageContent = (0, yaml_1.stringify)({
             id: page.id,
             name: page.name,
@@ -346,79 +498,63 @@ async function writeMultiPageApplicationToDisk(resource, rootPath, existingRelat
             sortMapEntries: true,
             blockQuote: "literal",
         });
-        await fs.outputFile(`${pageDirName}/page.yaml`, pageContent);
-        const pageApisDirName = `${pageDirName}/apis`;
+        const pageFilePath = `${pageDirName}/page.yaml`;
+        await fs.outputFile(pageFilePath, pageContent);
+        existingFilePaths.delete(pageFilePath);
         for (const api of page.apis) {
-            const originalApiName = extractApiName(api);
-            apiPromises.push(writeApi(api, originalApiName, pageApisDirName));
-            newApplicationConfig.pages[pageId].apis[api.id] = originalApiName;
+            const apiInfo = await writeAppApi(api, pageDirName, existingFilePaths, apiPromises, apiRepresentation);
+            // Update the page config with the new api info
+            const apiId = api.apiPb.metadata.id;
+            if (apiRepresentation.extractLargeSourceFiles) {
+                // Add the new pageApis field whenever large source files might be extracted
+                if (!pageConfig.pageApis) {
+                    pageConfig.pageApis = {};
+                }
+                pageConfig.pageApis[apiId] = {
+                    name: apiInfo.name,
+                    sourceFiles: (_a = apiInfo.sourceFiles) === null || _a === void 0 ? void 0 : _a.sort(),
+                };
+            }
+            else {
+                if (!pageConfig.apis) {
+                    pageConfig.apis = {};
+                }
+                pageConfig.apis[apiId] = apiInfo.name;
+            }
         }
     }
-    const apisDirName = `${appDirName}/apis`;
+    const appApis = {};
     if (resource.apis && resource.apis.length) {
-        await fs.ensureDir(apisDirName);
         for (const api of resource.apis) {
-            const originalApiName = extractApiName(api);
-            apiPromises.push(writeApi(api, originalApiName, apisDirName));
-            newApplicationConfig.apis[api.id] = originalApiName;
-        }
-        await Promise.all(apiPromises);
-    }
-    const existingApplicationConfig = await (0, util_1.getSuperblocksApplicationConfigIfExists)(appDirName);
-    // iterate through existing app-level apis and remove from disk any that are no longer present
-    for (const existingApiId of Object.keys((_a = existingApplicationConfig === null || existingApplicationConfig === void 0 ? void 0 : existingApplicationConfig.apis) !== null && _a !== void 0 ? _a : {})) {
-        const existingApiName = existingApplicationConfig === null || existingApplicationConfig === void 0 ? void 0 : existingApplicationConfig.apis[existingApiId];
-        const existingApiPath = `${apisDirName}/${slugifyName(existingApiName)}.yaml`;
-        const newApiName = newApplicationConfig.apis[existingApiId];
-        if (!(existingApiId in newApplicationConfig.apis) ||
-            slugifyName(newApiName) !== slugifyName(existingApiName)) {
-            await fs.remove(existingApiPath);
-        }
-    }
-    // iterate through existing pages and remove from disk any that are no longer present
-    for (const existingPage of Object.values((_b = existingApplicationConfig === null || existingApplicationConfig === void 0 ? void 0 : existingApplicationConfig.pages) !== null && _b !== void 0 ? _b : {})) {
-        const existingPageId = existingPage.id;
-        const existingPageDir = `${appDirName}/pages/${slugifyName(existingPage.name)}`;
-        if (existingPageId in newApplicationConfig.pages) {
-            // page still exists
-            const currentPage = newApplicationConfig.pages[existingPageId];
-            const newPageName = currentPage.name;
-            if (slugifyName(newPageName) !== slugifyName(existingPage.name)) {
-                // page was renamed
-                await fs.remove(existingPageDir);
-            }
-            // iterate through existing page-level apis and remove from disk any that are no longer present
-            for (const [existingApiId, existingApiName] of Object.entries(existingPage.apis)) {
-                const existingApiPath = `${existingPageDir}/apis/${slugifyName(existingApiName)}.yaml`;
-                if (existingApiId in currentPage.apis) {
-                    // api still exists
-                    const newApiName = newApplicationConfig.pages[existingPageId].apis[existingApiId];
-                    if (slugifyName(newApiName) !== slugifyName(existingApiName)) {
-                        // api was renamed
-                        await fs.remove(existingApiPath);
-                    }
-                }
-                else {
-                    // api was removed
-                    await fs.remove(existingApiPath);
+            const apiInfo = await writeAppApi(api, appDirName, existingFilePaths, apiPromises, apiRepresentation);
+            const apiId = api.apiPb.metadata.id;
+            if (apiRepresentation.extractLargeSourceFiles) {
+                if (!newApplicationConfig.appApis) {
+                    newApplicationConfig.appApis = {};
                 }
+                newApplicationConfig.appApis[apiId] = apiInfo;
             }
-        }
-        else {
-            // page was removed
-            await fs.remove(existingPageDir);
-            // remove all page-level apis
-            for (const existingApiName of Object.values(existingPage.apis)) {
-                const existingApiPath = `${existingPageDir}/apis/${slugifyName(existingApiName)}.yaml`;
-                await fs.remove(existingApiPath);
+            else {
+                if (!newApplicationConfig.apis) {
+                    newApplicationConfig.apis = {};
+                }
+                newApplicationConfig.apis[apiId] = apiInfo.name;
             }
         }
+        await Promise.all(apiPromises);
+    }
+    if (apiRepresentation.extractLargeSourceFiles) {
+        newApplicationConfig.appApis = appApis;
     }
-    await fs.ensureDir(`${appDirName}/${util_2.SUPERBLOCKS_HOME_FOLDER_NAME}`);
-    await fs.writeFile(`${appDirName}/${util_2.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(newApplicationConfig), null, 2));
+    // Delete any existing files that were not overwritten
+    for (const filePath of existingFilePaths) {
+        await fs.remove(filePath);
+    }
+    await fs.ensureDir(`${appDirName}/${util_1.SUPERBLOCKS_HOME_FOLDER_NAME}`);
+    await fs.writeFile(`${appDirName}/${util_1.RESOURCE_CONFIG_PATH}`, JSON.stringify(sortByKey(newApplicationConfig), null, 2));
     const createdFiles = await Promise.resolve(
     // Defensive check for when application settings are missing componentFiles
-    (_d = (_c = resource.componentFiles) === null || _c === void 0 ? void 0 : _c.map((file) => downloadFile(appDirName, file.filename, file.url))) !== null && _d !== void 0 ? _d : []);
+    (_c = (_b = resource.componentFiles) === null || _b === void 0 ? void 0 : _b.map((file) => downloadFile(appDirName, file.filename, file.url))) !== null && _c !== void 0 ? _c : []);
     // print out failed downloads synchronously here
     createdFiles
         .filter((createdFiles) => createdFiles.length)
@@ -439,6 +575,95 @@ async function writeMultiPageApplicationToDisk(resource, rootPath, existingRelat
     };
 }
 exports.writeMultiPageApplicationToDisk = writeMultiPageApplicationToDisk;
+function getExistingFilePathsForApplicationApi(superblocksApplicationConfig, location) {
+    const paths = new Set();
+    if (!superblocksApplicationConfig) {
+        return paths;
+    }
+    // Pages
+    if (superblocksApplicationConfig.pages) {
+        for (const page of Object.values(superblocksApplicationConfig.pages)) {
+            // Page YAML
+            const pageNameSlug = slugifyName(page.name);
+            const pagePath = `${location}/pages/${pageNameSlug}`;
+            paths.add(`${pagePath}/page.yaml`);
+            // Page APIs
+            if (page.pageApis) {
+                for (const api of Object.values(page.pageApis)) {
+                    addExistingFilePathsForApi(api, `${pagePath}/apis`, paths, true);
+                }
+            }
+            else if (page.apis) {
+                for (const apiName of Object.values(page.apis)) {
+                    addExistingFilePathsForApi({ name: apiName }, `${pagePath}/apis`, paths, true);
+                }
+            }
+        }
+    }
+    // APIs
+    if (superblocksApplicationConfig.appApis) {
+        // For file version >= 0.2.0, there are API files either the 'apis' folder or in an API-specific folder
+        for (const api of Object.values(superblocksApplicationConfig.appApis)) {
+            addExistingFilePathsForApi(api, `${location}/apis`, paths, true);
+        }
+    }
+    else if (superblocksApplicationConfig.apis) {
+        // For file version < 0.2.0, there are only API file
+        for (const apiName of Object.values(superblocksApplicationConfig.apis)) {
+            addExistingFilePathsForApi({ name: apiName }, `${location}/apis`, paths, true);
+        }
+    }
+    return paths;
+}
+function getExistingFilePathsForBackendApi(superblocksBackendConfig, location) {
+    const paths = new Set();
+    if (!superblocksBackendConfig) {
+        return paths;
+    }
+    // Pages
+    if (superblocksBackendConfig.sourceFiles) {
+        addExistingFilePathsForApi({ name: "api", sourceFiles: superblocksBackendConfig.sourceFiles }, location, paths, false);
+    }
+    else {
+        addExistingFilePathsForApi({ name: "api" }, location, paths, false);
+    }
+    return paths;
+}
+function addExistingFilePathsForApi(api, location, paths, useNestedFolder) {
+    var _a, _b;
+    const apiNameSlug = slugifyName(api.name);
+    if ((_a = api.sourceFiles) === null || _a === void 0 ? void 0 : _a.length) {
+        // API files are in an API-specific folder
+        const apiDirPath = useNestedFolder
+            ? `${location}/${apiNameSlug}`
+            : location;
+        paths.add(`${apiDirPath}/${apiNameSlug}.yaml`);
+        // And there are source files
+        for (const sourceFile of (_b = api.sourceFiles) !== null && _b !== void 0 ? _b : []) {
+            paths.add(`${apiDirPath}/${sourceFile}`);
+        }
+    }
+    else {
+        // API file is in the 'apis' folder with no separate source files
+        paths.add(`${location}/${apiNameSlug}.yaml`);
+    }
+    return paths;
+}
+exports.addExistingFilePathsForApi = addExistingFilePathsForApi;
+function getResourceConfigMetadata(featureFlags, existingResourceConfig) {
+    if (!existingResourceConfig) {
+        // This is a new application, and we may need to add a metadata field to the application config
+        if (featureFlags.splitLargeApiStepsInNewEnabled()) {
+            return {
+                fileVersion: LATEST_FILE_VERSION,
+            };
+        }
+    }
+    else if (existingResourceConfig.metadata) {
+        return existingResourceConfig.metadata;
+    }
+    return undefined;
+}
 async function removeResourceFromDisk(rootPath, resourceRelativeLocation) {
     const resourceLocation = node_path_1.default.resolve(rootPath, resourceRelativeLocation);
     await fs.remove(resourceLocation);
@@ -593,21 +818,161 @@ function extractApiName(api) {
     return (_d = (_c = (_b = (_a = api.apiPb) === null || _a === void 0 ? void 0 : _a.metadata) === null || _b === void 0 ? void 0 : _b.name) !== null && _c !== void 0 ? _c : api === null || api === void 0 ? void 0 : api.name) !== null && _d !== void 0 ? _d : "Unknown";
 }
 exports.extractApiName = extractApiName;
-function writeApi(api, originalApiName, appDirName) {
-    const apiName = slugifyName(originalApiName);
-    const apiContent = (0, yaml_1.stringify)(api.apiPb, {
+async function writeAppApi(api, directoryPath, existingFilePaths, apiPromises, apiRepresentation) {
+    const originalApiName = extractApiName(api);
+    const additionalStepFiles = [];
+    await writeApiFiles(api, slugifyName(originalApiName), `${directoryPath}/apis`, true, apiPromises, additionalStepFiles, apiRepresentation, existingFilePaths);
+    return {
+        name: originalApiName,
+        sourceFiles: additionalStepFiles.map((file) => file.relativePath).sort(),
+    };
+}
+exports.writeAppApi = writeAppApi;
+async function writeBackendApi(api, directoryPath, apiPromises, apiRepresentation, existingFilePaths) {
+    const originalApiName = extractApiName(api);
+    const additionalStepFiles = [];
+    await writeApiFiles(api, "api", directoryPath, false, apiPromises, additionalStepFiles, apiRepresentation, existingFilePaths);
+    return {
+        name: originalApiName,
+        sourceFiles: additionalStepFiles.map((file) => file.relativePath).sort(),
+    };
+}
+async function writeApiFiles(api, apiNameSlug, directoryPath, nestedApiFiles, apiPromises, additionalStepFiles, apiRepresentation, existingFilePaths) {
+    let pathForApiFile = directoryPath;
+    // Determine whether to extract into separate files  the source code from the steps
+    const updatedApi = extractAdditionalStepFiles(api.apiPb, additionalStepFiles, apiRepresentation);
+    if (additionalStepFiles.length > 0) {
+        if (nestedApiFiles) {
+            // This API has at least one large step code file, so we need a nested directory
+            pathForApiFile = `${pathForApiFile}/${apiNameSlug}`;
+        }
+        // Make sure the directory exists
+        await fs.ensureDir(pathForApiFile);
+        // Write any additional step source code files to the disk
+        for (const stepFile of additionalStepFiles) {
+            const sourceFilePath = `${pathForApiFile}/${stepFile.relativePath}`;
+            const handleApi = async () => {
+                await fs.outputFile(sourceFilePath, stepFile.contents);
+            };
+            apiPromises.push(handleApi());
+            // Mark the existing source file as being reused
+            existingFilePaths === null || existingFilePaths === void 0 ? void 0 : existingFilePaths.delete(sourceFilePath);
+        }
+    }
+    else {
+        // No additional step files to write, so mark the API directory as existing so that it will be removed later.
+        // Do this even if the current representation does not extract large source files, in case they were previously
+        existingFilePaths === null || existingFilePaths === void 0 ? void 0 : existingFilePaths.add(`${directoryPath}/${apiNameSlug}`);
+    }
+    // Convert the updated API to YAML and write it to disk
+    const apiContent = (0, yaml_1.stringify)(updatedApi, {
         sortMapEntries: true,
         blockQuote: "literal",
     });
+    const yamlFilePath = `${pathForApiFile}/${apiNameSlug}.yaml`;
     const handleApi = async () => {
-        await fs.outputFile(`${appDirName}/${apiName}.yaml`, apiContent);
+        await fs.outputFile(yamlFilePath, apiContent);
     };
-    return handleApi();
+    apiPromises.push(handleApi());
+    // Mark the existing source file as being reused
+    existingFilePaths === null || existingFilePaths === void 0 ? void 0 : existingFilePaths.delete(yamlFilePath);
+}
+function extractAdditionalStepFiles(api, additionalStepFiles, apiRepresentation) {
+    var _a, _b;
+    if (!api.blocks || !(apiRepresentation === null || apiRepresentation === void 0 ? void 0 : apiRepresentation.extractLargeSourceFiles)) {
+        return api;
+    }
+    const apiClone = lodash_2.default.cloneDeep(api);
+    const minLinesForExtraction = (_a = apiRepresentation === null || apiRepresentation === void 0 ? void 0 : apiRepresentation.minLinesForExtraction) !== null && _a !== void 0 ? _a : sdk_1.DEFAULT_LINES_FOR_LARGE_STEPS;
+    const existingFilePaths = new Set();
+    extractAdditionalStepFilesFromBlocks((_b = apiClone.blocks) !== null && _b !== void 0 ? _b : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+    return apiClone;
+}
+function extractAdditionalStepFilesFromBlocks(blocks, additionalStepFiles, existingFilePaths, minLinesForExtraction) {
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w;
+    for (const block of blocks) {
+        // Handle language step files
+        if (block.step) {
+            extractAdditionalStepFilesFromStep(block.step, block.name, additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        }
+        // Handle loops
+        extractAdditionalStepFilesFromBlocks((_b = (_a = block.loop) === null || _a === void 0 ? void 0 : _a.blocks) !== null && _b !== void 0 ? _b : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        // Handle try/catch
+        if (block.tryCatch) {
+            extractAdditionalStepFilesFromBlocks((_d = (_c = block.tryCatch.try) === null || _c === void 0 ? void 0 : _c.blocks) !== null && _d !== void 0 ? _d : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            extractAdditionalStepFilesFromBlocks((_f = (_e = block.tryCatch.catch) === null || _e === void 0 ? void 0 : _e.blocks) !== null && _f !== void 0 ? _f : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            extractAdditionalStepFilesFromBlocks((_h = (_g = block.tryCatch.finally) === null || _g === void 0 ? void 0 : _g.blocks) !== null && _h !== void 0 ? _h : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        }
+        // Handle conditional
+        if (block.conditional) {
+            extractAdditionalStepFilesFromBlocks((_k = (_j = block.conditional.if) === null || _j === void 0 ? void 0 : _j.blocks) !== null && _k !== void 0 ? _k : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            for (const elseIfBlock of (_l = block.conditional.elseIf) !== null && _l !== void 0 ? _l : []) {
+                extractAdditionalStepFilesFromBlocks(elseIfBlock.blocks, additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            }
+            extractAdditionalStepFilesFromBlocks((_o = (_m = block.conditional.else) === null || _m === void 0 ? void 0 : _m.blocks) !== null && _o !== void 0 ? _o : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        }
+        // Handle Parallel API
+        if (block.parallel) {
+            for (const path of Object.values((_r = (_q = (_p = block.parallel) === null || _p === void 0 ? void 0 : _p.static) === null || _q === void 0 ? void 0 : _q.paths) !== null && _r !== void 0 ? _r : {})) {
+                extractAdditionalStepFilesFromBlocks(path.blocks, additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            }
+            extractAdditionalStepFilesFromBlocks((_u = (_t = (_s = block.parallel) === null || _s === void 0 ? void 0 : _s.dynamic) === null || _t === void 0 ? void 0 : _t.blocks) !== null && _u !== void 0 ? _u : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        }
+        // Handle stream
+        if (block.stream) {
+            const trigger = block.stream.trigger;
+            if (trigger && trigger.step) {
+                extractAdditionalStepFilesFromStep(trigger.step, trigger.name, additionalStepFiles, existingFilePaths, minLinesForExtraction);
+            }
+            extractAdditionalStepFilesFromBlocks((_w = (_v = block.stream.process) === null || _v === void 0 ? void 0 : _v.blocks) !== null && _w !== void 0 ? _w : [], additionalStepFiles, existingFilePaths, minLinesForExtraction);
+        }
+    }
+}
+function extractAdditionalStepFilesFromStep(step, name, additionalStepFiles, existingFilePaths, minLinesForExtraction) {
+    // Handle language step files
+    if (step && "integration" in step) {
+        const integration = step.integration;
+        const extension = LANGUAGE_STEP_EXTENSIONS[integration];
+        if (extension) {
+            const languageContent = step[integration];
+            if (languageContent) {
+                const blockBody = languageContent.body;
+                if (blockBody && typeof blockBody === "string") {
+                    const lines = blockBody === null || blockBody === void 0 ? void 0 : blockBody.split("\n");
+                    const linesNum = lines === null || lines === void 0 ? void 0 : lines.length;
+                    if (linesNum && linesNum > minLinesForExtraction) {
+                        // Record the separate file is needed, with the language contents
+                        const relativePath = findUnusedFilePath(slugifyName(name), extension, existingFilePaths);
+                        additionalStepFiles.push({
+                            relativePath: relativePath,
+                            contents: blockBody,
+                            language: integration,
+                        });
+                        // Replace the body with a reference to the separate file
+                        languageContent.body = {
+                            path: `./${relativePath}`,
+                        };
+                    }
+                }
+            }
+        }
+    }
+}
+function findUnusedFilePath(filename, extension, existingFilePaths) {
+    let proposedFilename = `${filename}.${extension}`;
+    let i = 1;
+    while (existingFilePaths.has(proposedFilename)) {
+        proposedFilename = `${filename}_${i}.${extension}`;
+        i++;
+    }
+    existingFilePaths.add(proposedFilename);
+    return proposedFilename;
 }
 async function validateMultiPageApplication(applicationConfig, superblocksRootPath, location) {
     var _a;
     // validate app level APIs
-    for (const apiName of Object.values(applicationConfig.apis)) {
+    const apiNames = getApiNamesFromApplicationConfig(applicationConfig);
+    for (const apiName of apiNames) {
         const apiPath = node_path_1.default.resolve(superblocksRootPath, location, "apis", `${slugifyName(apiName)}.yaml`);
         const validateApiError = await validateYamlFile(apiPath);
         if (validateApiError) {
@@ -622,11 +987,26 @@ async function validateMultiPageApplication(applicationConfig, superblocksRootPa
             return validatePageError;
         }
         // validate page level APIs
-        for (const apiName of Object.values(page.apis)) {
-            const apiPath = node_path_1.default.resolve(superblocksRootPath, location, "pages", slugifyName(page.name), "apis", `${slugifyName(apiName)}.yaml`);
-            const validateApiError = await validateYamlFile(apiPath);
-            if (validateApiError) {
-                return validateApiError;
+        const pageApiNames = getApiNamesFromPageConfig(page);
+        for (const apiName of pageApiNames) {
+            const apisPath = node_path_1.default.resolve(superblocksRootPath, location, "pages", slugifyName(page.name), "apis");
+            // Try the API path not in a nested directory
+            const apiPath = node_path_1.default.resolve(apisPath, `${slugifyName(apiName)}.yaml`);
+            if (fs.pathExistsSync(apiPath)) {
+                const validateApiError = await validateYamlFile(apiPath);
+                if (validateApiError) {
+                    return validateApiError;
+                }
+            }
+            else {
+                // Try the API path in a nested directory
+                const nestedApiPath = node_path_1.default.resolve(apisPath, slugifyName(apiName), "api.yaml");
+                if (fs.pathExistsSync(nestedApiPath)) {
+                    const validateApiError = await validateYamlFile(nestedApiPath);
+                    if (validateApiError) {
+                        return validateApiError;
+                    }
+                }
             }
         }
     }
@@ -636,7 +1016,7 @@ async function validateLocalResource(superblocksRootPath, resource) {
     switch (resource.resourceType) {
         case "APPLICATION": {
             // make sure application config exists
-            const applicationConfigPath = node_path_1.default.resolve(superblocksRootPath, resource.location, util_2.RESOURCE_CONFIG_PATH);
+            const applicationConfigPath = node_path_1.default.resolve(superblocksRootPath, resource.location, util_1.RESOURCE_CONFIG_PATH);
             if (!(await fs.pathExists(applicationConfigPath))) {
                 return `File ${relativeToCurrentDir(applicationConfigPath)} not found. Superblocks CLI commands cannot function without it.`;
             }
@@ -667,7 +1047,7 @@ async function validateLocalResource(superblocksRootPath, resource) {
         }
         case "BACKEND": {
             // make sure the backend config exists
-            const backendConfigPath = node_path_1.default.resolve(superblocksRootPath, resource.location, util_2.RESOURCE_CONFIG_PATH);
+            const backendConfigPath = node_path_1.default.resolve(superblocksRootPath, resource.location, util_1.RESOURCE_CONFIG_PATH);
             if (!(await fs.pathExists(backendConfigPath))) {
                 return `File ${relativeToCurrentDir(backendConfigPath)} not found. Superblocks CLI commands cannot function without it.`;
             }