@tenonhq/dovetail-core 0.0.85

package/dist/appUtils.js

@@ -0,0 +1,929 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createAndAssignUpdateSet = exports.swapScope = exports.buildFiles = exports.summarizeRecord = exports.pushFiles = exports.getAppFileList = exports.groupAppFiles = exports.processMissingFiles = exports.refreshAllFiles = exports.findMissingFiles = exports.syncManifest = exports.processManifest = exports.normalizeManifestKeys = void 0;
+const path_1 = __importDefault(require("path"));
+const fs_1 = __importDefault(require("fs"));
+const progress_1 = __importDefault(require("progress"));
+const fUtils = __importStar(require("./FileUtils"));
+const ConfigManager = __importStar(require("./config"));
+const constants_1 = require("./constants");
+const PluginManager_1 = __importDefault(require("./PluginManager"));
+const FileLogger_1 = require("./FileLogger");
+const projectFiles_1 = require("./projectFiles");
+const snClient_1 = require("./snClient");
+const benchmark_1 = require("./benchmark");
+const Logger_1 = require("./Logger");
+const genericUtils_1 = require("./genericUtils");
+const getUpdateSetConfig = () => {
+    const configPath = (0, projectFiles_1.getUpdateSetsConfigPath)();
+    try {
+        if (fs_1.default.existsSync(configPath)) {
+            return JSON.parse(fs_1.default.readFileSync(configPath, "utf8"));
+        }
+    }
+    catch (e) {
+        Logger_1.logger.warn(`Failed to parse update set config at ${configPath}: ${e instanceof Error ? e.message : String(e)}`);
+    }
+    return {};
+};
+// Merge _lastUpdatedOn into the server-provided metadata content. Preserves all
+// record fields (sys_id, sys_scope, field value/display_value pairs, etc.) so
+// the local metaData.json is a full snapshot of the record, not just a stub.
+const stampMetadataContent = (file) => {
+    if (file.name !== "metaData" || file.type !== "json")
+        return file;
+    const stamp = new Date().toISOString();
+    if (!file.content) {
+        return { ...file, content: JSON.stringify({ _lastUpdatedOn: stamp }, null, 2) };
+    }
+    try {
+        const metadata = JSON.parse(file.content);
+        if (metadata.sys_updated_on && metadata.sys_updated_on.value) {
+            metadata._lastUpdatedOn = metadata.sys_updated_on.value;
+        }
+        else {
+            metadata._lastUpdatedOn = stamp;
+        }
+        return { ...file, content: JSON.stringify(metadata, null, 2) };
+    }
+    catch (e) {
+        // Content isn't JSON — leave as-is, it will be written verbatim.
+        return file;
+    }
+};
+const hasServerMetadata = (files) => files.some((f) => f.name === "metaData" && f.type === "json" && !!f.content);
+const processFilesInManRec = async (recPath, rec, forceWrite) => {
+    FileLogger_1.fileLogger.debug("Processing record: " + rec.name + " (" + rec.files.length + " files)");
+    const fileWrite = fUtils.writeSNFileCurry(forceWrite);
+    // If the server did not provide a metadata file, fall back to a timestamp-only
+    // stub so the record directory always has a metaData.json.
+    if (!hasServerMetadata(rec.files)) {
+        const stubMetadata = {
+            name: "metaData",
+            type: "json",
+            content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
+        };
+        await fileWrite(stubMetadata, recPath);
+    }
+    const writeResults = await (0, genericUtils_1.allSettledBatched)(rec.files, constants_1.CONCURRENCY_FILES, function (file) { return fileWrite(stampMetadataContent(file), recPath); });
+    const writeFailures = writeResults.filter((r) => r.status === "rejected");
+    if (writeFailures.length > 0) {
+        writeFailures.forEach((f) => {
+            FileLogger_1.fileLogger.error("File write failed: " + f.reason);
+        });
+    }
+    // Remove content from ALL files (metadata is not included in manifest)
+    rec.files = rec.files.map((file) => {
+        const fileCopy = { ...file };
+        delete fileCopy.content;
+        return fileCopy;
+    });
+};
+const processRecsInManTable = async (tablePath, table, forceWrite, onRecordProcessed) => {
+    const { records } = table;
+    const recKeys = Object.keys(records);
+    const recKeyToPath = (key) => path_1.default.join(tablePath, records[key].name);
+    const recPathPromises = recKeys
+        .map(recKeyToPath)
+        .map(fUtils.createDirRecursively);
+    await Promise.all(recPathPromises);
+    await (0, genericUtils_1.processBatched)(recKeys, constants_1.CONCURRENCY_RECORDS, function (recKey) {
+        return processFilesInManRec(recKeyToPath(recKey), records[recKey], forceWrite).then(function () {
+            if (onRecordProcessed)
+                onRecordProcessed();
+        });
+    });
+};
+const countRecordsInTables = (tables) => {
+    return Object.keys(tables).reduce(function (sum, tableName) {
+        return sum + Object.keys(tables[tableName].records).length;
+    }, 0);
+};
+const processTablesInManifest = async (tables, forceWrite, sourcePath, onRecordProcessed, scope) => {
+    var basePath = sourcePath || ConfigManager.getSourcePath();
+    var tableNames = Object.keys(tables);
+    // Defense-in-depth: filter out any table not in the resolved whitelist before
+    // writing to disk. Protects against upstream defects that let non-whitelisted
+    // tables through (e.g. server-side manifest fanout, stale manifest entries).
+    if (scope) {
+        try {
+            var resolved = ConfigManager.resolveConfigForScope(scope);
+            var allowed = resolved.tables;
+            if (allowed && allowed.length > 0) {
+                var skipped = [];
+                tableNames = tableNames.filter(function (t) {
+                    if (allowed.indexOf(t) === -1) {
+                        skipped.push(t);
+                        return false;
+                    }
+                    return true;
+                });
+                if (skipped.length > 0) {
+                    FileLogger_1.fileLogger.debug("processTablesInManifest: dropped " + skipped.length +
+                        " non-whitelisted tables for scope '" + scope + "': " + skipped.join(", "));
+                }
+            }
+        }
+        catch (e) {
+            // Config resolution can fail for legacy single-scope manifests; fall
+            // through and process whatever the manifest contains.
+            FileLogger_1.fileLogger.debug("processTablesInManifest: could not resolve whitelist for scope '" + scope + "'");
+        }
+    }
+    await (0, genericUtils_1.processBatched)(tableNames, constants_1.CONCURRENCY_TABLES, function (tableName) {
+        return processRecsInManTable(path_1.default.join(basePath, tableName), tables[tableName], forceWrite, onRecordProcessed);
+    });
+};
+/**
+ * Re-keys manifest records from sys_id to record.name (display value).
+ * Some ServiceNow tables return records keyed by sys_id instead of display name.
+ * This ensures consistent naming for directories and manifest lookups.
+ */
+const normalizeManifestKeys = (manifest) => {
+    var tables = manifest.tables || {};
+    var tableNames = Object.keys(tables);
+    for (var i = 0; i < tableNames.length; i++) {
+        var tableName = tableNames[i];
+        var records = tables[tableName].records || {};
+        var recordKeys = Object.keys(records);
+        var normalized = {};
+        for (var j = 0; j < recordKeys.length; j++) {
+            var key = recordKeys[j];
+            var record = records[key];
+            var displayKey = record.name || key;
+            // Handle duplicate display names by appending sys_id suffix
+            if (normalized[displayKey]) {
+                displayKey = displayKey + " (" + record.sys_id.substring(0, 8) + ")";
+            }
+            normalized[displayKey] = record;
+        }
+        tables[tableName].records = normalized;
+    }
+    return manifest;
+};
+exports.normalizeManifestKeys = normalizeManifestKeys;
+const processManifest = async (manifest, forceWrite = false, sourcePath) => {
+    const tableCount = Object.keys(manifest.tables).length;
+    FileLogger_1.fileLogger.debug("Processing manifest: " + (manifest.scope || "legacy") + " (" + tableCount + " tables)");
+    var recordCount = countRecordsInTables(manifest.tables);
+    var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
+        scope: manifest.scope || "default",
+        total: recordCount,
+    });
+    await processTablesInManifest(manifest.tables, forceWrite, sourcePath, progress.tick, manifest.scope);
+    if (manifest.scope) {
+        await fUtils.writeScopeManifest(manifest.scope, manifest);
+    }
+    else {
+        await fUtils.writeFileForce(ConfigManager.getManifestPath(), JSON.stringify(manifest, null, 2));
+    }
+};
+exports.processManifest = processManifest;
+const syncManifest = async (scope, options = {}) => {
+    // Top-level entry owns the collector lifecycle. Recursive calls (all-scopes
+    // → per-scope) inherit the collector via options._benchmarkCollector.
+    var isBenchmarkOwner = false;
+    var collector = options._benchmarkCollector;
+    if (options.benchmark && !collector) {
+        collector = new benchmark_1.BenchmarkCollector();
+        (0, snClient_1.setBenchmarkSink)(collector);
+        isBenchmarkOwner = true;
+    }
+    try {
+        const curManifest = await ConfigManager.getManifest();
+        if (!curManifest)
+            throw new Error("No manifest file loaded!");
+        const config = ConfigManager.getConfig();
+        const declaredScopes = (config.scopes && Object.keys(config.scopes)) || [];
+        // If a specific scope is provided, sync only that scope
+        if (scope) {
+            // Scope whitelist gate: refuse to refresh scopes not declared in dove.config.js.
+            // Without this, stale entries in dove.manifest.json leak undeclared scopes into
+            // the refresh loop (see RFC-0004 / sys_alias debris incident 2026-04-14).
+            if (declaredScopes.length > 0 && declaredScopes.indexOf(scope) === -1) {
+                Logger_1.logger.warn("Skipping scope '" + scope + "' — not declared in dove.config.js `scopes`. " +
+                    "Add it to config.scopes to sync, or remove its manifest file.");
+                FileLogger_1.fileLogger.debug("syncManifest: skipped undeclared scope '" + scope + "'");
+                return;
+            }
+            Logger_1.logger.info("Refreshing scope: " + scope + "...");
+            const client = (0, snClient_1.defaultClient)();
+            // Resolve scope-specific source directory + table whitelist
+            var scopeSourcePath = ConfigManager.getSourcePathForScope(scope);
+            var resolvedConfig = ConfigManager.resolveConfigForScope(scope);
+            var allowedTables = resolvedConfig.tables;
+            const newManifest = (0, exports.normalizeManifestKeys)(await (0, snClient_1.unwrapSNResponse)(client.getManifest(scope, config)));
+            // Table whitelist gate: drop any table the server returned that is not in
+            // the resolved _tables whitelist for this scope. Mirrors the filter in
+            // commands.ts downloadCommand() and allScopesCommands.ts processScope().
+            if (allowedTables && allowedTables.length > 0) {
+                var manifestTableNames = Object.keys(newManifest.tables || {});
+                var filteredTables = {};
+                var skippedCount = 0;
+                for (var t = 0; t < manifestTableNames.length; t++) {
+                    var tName = manifestTableNames[t];
+                    if (allowedTables.indexOf(tName) !== -1) {
+                        filteredTables[tName] = newManifest.tables[tName];
+                    }
+                    else {
+                        skippedCount++;
+                    }
+                }
+                if (skippedCount > 0) {
+                    FileLogger_1.fileLogger.debug("syncManifest: filtered " + skippedCount + " tables not in _tables whitelist for " +
+                        scope + " (kept " + Object.keys(filteredTables).length + " of " + manifestTableNames.length + ")");
+                }
+                newManifest.tables = filteredTables;
+            }
+            else {
+                Logger_1.logger.warn("No _tables whitelist defined — writing ALL tables for " + scope);
+            }
+            const refreshTableCount = Object.keys(newManifest.tables).length;
+            FileLogger_1.fileLogger.debug("Refreshed manifest for " + scope + ": " + refreshTableCount + " tables");
+            await fUtils.writeScopeManifest(scope, newManifest);
+            if (collector)
+                collector.startScope(scope);
+            await (0, exports.refreshAllFiles)(newManifest, scopeSourcePath, {
+                force: options.force,
+                benchmarkCollector: collector,
+            });
+            // Update the in-memory manifest for this scope
+            if (ConfigManager.isMultiScopeManifest(curManifest)) {
+                curManifest[scope] = newManifest;
+                ConfigManager.updateManifest(curManifest);
+            }
+        }
+        else {
+            // Sync all scopes. Prefer the declared-scopes list (config.scopes) over
+            // the persisted manifest keys — the manifest may contain stale undeclared
+            // scopes that leaked in before the whitelist gate existed.
+            var childOptions = {
+                force: options.force,
+                _benchmarkCollector: collector,
+            };
+            if (declaredScopes.length > 0) {
+                for (var d = 0; d < declaredScopes.length; d++) {
+                    await (0, exports.syncManifest)(declaredScopes[d], childOptions);
+                }
+            }
+            else if (ConfigManager.isMultiScopeManifest(curManifest)) {
+                // No declared scopes — fall back to the persisted manifest's scopes.
+                for (const scopeName of Object.keys(curManifest)) {
+                    await (0, exports.syncManifest)(scopeName, childOptions);
+                }
+            }
+            else if (curManifest.scope) {
+                // Single scope manifest
+                await (0, exports.syncManifest)(curManifest.scope, childOptions);
+            }
+        }
+    }
+    catch (e) {
+        let message;
+        if (e instanceof Error)
+            message = e.message;
+        else
+            message = String(e);
+        Logger_1.logger.error("Refresh failed: " + message);
+    }
+    finally {
+        if (isBenchmarkOwner && collector) {
+            (0, snClient_1.setBenchmarkSink)(null);
+            Logger_1.logger.info(collector.formatSummary());
+        }
+    }
+};
+exports.syncManifest = syncManifest;
+const markFileMissing = (missingObj) => (table) => (recordId) => (file) => {
+    if (!missingObj[table]) {
+        missingObj[table] = {};
+    }
+    if (!missingObj[table][recordId]) {
+        missingObj[table][recordId] = [];
+    }
+    const { name, type } = file;
+    missingObj[table][recordId].push({ name, type });
+};
+const markRecordMissing = (record, missingFunc) => {
+    record.files.forEach((file) => {
+        missingFunc(record.sys_id)(file);
+    });
+};
+const markTableMissing = (table, tableName, missingFunc) => {
+    Object.keys(table.records).forEach((recName) => {
+        markRecordMissing(table.records[recName], missingFunc(tableName));
+    });
+};
+const checkFilesForMissing = async (recPath, files, missingFunc) => {
+    const checkPromises = files.map(fUtils.SNFileExists(recPath));
+    const checks = await Promise.all(checkPromises);
+    checks.forEach((check, index) => {
+        if (!check) {
+            missingFunc(files[index]);
+        }
+    });
+};
+const checkRecordsForMissing = async (tablePath, records, missingFunc) => {
+    const recNames = Object.keys(records);
+    const recPaths = recNames.map(fUtils.appendToPath(tablePath));
+    const checkPromises = recNames.map((recName, index) => fUtils.pathExists(recPaths[index]));
+    const checks = await Promise.all(checkPromises);
+    const fileCheckPromises = checks.map(async (check, index) => {
+        const recName = recNames[index];
+        const record = records[recName];
+        if (!check) {
+            markRecordMissing(record, missingFunc);
+            return;
+        }
+        await checkFilesForMissing(recPaths[index], record.files, missingFunc(record.sys_id));
+    });
+    await Promise.all(fileCheckPromises);
+};
+const checkTablesForMissing = async (topPath, tables, missingFunc) => {
+    const tableNames = Object.keys(tables);
+    const tablePaths = tableNames.map(fUtils.appendToPath(topPath));
+    const checkPromises = tableNames.map((tableName, index) => fUtils.pathExists(tablePaths[index]));
+    const checks = await Promise.all(checkPromises);
+    const recCheckPromises = checks.map(async (check, index) => {
+        const tableName = tableNames[index];
+        if (!check) {
+            markTableMissing(tables[tableName], tableName, missingFunc);
+            return;
+        }
+        await checkRecordsForMissing(tablePaths[index], tables[tableName].records, missingFunc(tableName));
+    });
+    await Promise.all(recCheckPromises);
+};
+const findMissingFiles = async (manifest, sourcePath) => {
+    const missing = {};
+    const { tables } = manifest;
+    const missingTableFunc = markFileMissing(missing);
+    await checkTablesForMissing(sourcePath || ConfigManager.getSourcePath(), tables, missingTableFunc);
+    // missing gets mutated along the way as things get processed
+    return missing;
+};
+exports.findMissingFiles = findMissingFiles;
+// Chunk bulkDownload by table to stay under ServiceNow's 10 MB REST payload cap.
+// A single unchunked call 500s on large scopes (e.g. x_cadso_automate at ~29 MB).
+// Must mirror the chunk size used by allScopesCommands.ts (watch path) so behaviour
+// is consistent across `refresh` and `watch`.
+const BULK_DOWNLOAD_TABLE_CHUNK_SIZE = 5;
+/**
+ * Builds a MissingFileTableMap containing EVERY file in the manifest — ignores
+ * local disk state. Used by `sinc refresh` to pull instance-side edits down.
+ */
+const buildAllFilesMap = (manifest) => {
+    const result = {};
+    const { tables } = manifest;
+    const tableNames = Object.keys(tables);
+    for (var t = 0; t < tableNames.length; t++) {
+        var tableName = tableNames[t];
+        var records = tables[tableName].records;
+        var recNames = Object.keys(records);
+        if (recNames.length === 0)
+            continue;
+        var recMap = {};
+        for (var r = 0; r < recNames.length; r++) {
+            var rec = records[recNames[r]];
+            if (!rec.files || rec.files.length === 0)
+                continue;
+            // Strip any content that may be lingering on manifest file entries; the
+            // bulkDownload endpoint only needs name + type to resolve each field.
+            recMap[rec.sys_id] = rec.files.map(function (f) {
+                return { name: f.name, type: f.type };
+            });
+        }
+        if (Object.keys(recMap).length > 0)
+            result[tableName] = recMap;
+    }
+    return result;
+};
+/**
+ * Refreshes local files against the ServiceNow instance for every file in the
+ * given manifest. Unlike `processMissingFiles` (which only writes files absent
+ * from disk), this walks ALL manifest files, fetches their current content from
+ * the instance, and writes when content differs.
+ *
+ * @param options.force — when true, always overwrite local files even if their
+ * content matches the instance. Use for deliberate "reset local to instance".
+ */
+const refreshAllFiles = async (newManifest, sourcePath, options = {}) => {
+    try {
+        const allFiles = buildAllFilesMap(newManifest);
+        const tableNames = Object.keys(allFiles);
+        if (tableNames.length === 0)
+            return;
+        FileLogger_1.fileLogger.debug("Refreshing file content for " + tableNames.length + " tables (force=" + !!options.force + ")");
+        const { tableOptions = {} } = ConfigManager.getConfig();
+        const client = (0, snClient_1.defaultClient)();
+        const totalChunks = Math.ceil(tableNames.length / BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
+        const filesToProcess = {};
+        for (var i = 0; i < tableNames.length; i += BULK_DOWNLOAD_TABLE_CHUNK_SIZE) {
+            const chunkTableNames = tableNames.slice(i, i + BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
+            const chunkMissing = {};
+            for (var j = 0; j < chunkTableNames.length; j++) {
+                chunkMissing[chunkTableNames[j]] = allFiles[chunkTableNames[j]];
+            }
+            const batchNum = Math.floor(i / BULK_DOWNLOAD_TABLE_CHUNK_SIZE) + 1;
+            FileLogger_1.fileLogger.debug("Refresh download batch " + batchNum + "/" + totalChunks +
+                " (" + chunkTableNames.length + " tables): " + chunkTableNames.join(", "));
+            const chunkResult = await (0, snClient_1.unwrapSNResponse)(client.getMissingFiles(chunkMissing, tableOptions));
+            for (var tableName in chunkResult) {
+                filesToProcess[tableName] = chunkResult[tableName];
+            }
+        }
+        var basePath = sourcePath || ConfigManager.getSourcePath();
+        var recordCount = countRecordsInTables(filesToProcess);
+        var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
+            scope: newManifest.scope || "default",
+            total: recordCount,
+        });
+        var writtenCount = 0;
+        var unchangedCount = 0;
+        const forceWrite = !!options.force;
+        const forceWriter = fUtils.writeSNFileCurry(false);
+        const processedTableNames = Object.keys(filesToProcess);
+        await (0, genericUtils_1.processBatched)(processedTableNames, constants_1.CONCURRENCY_TABLES, async function (tableName) {
+            var tablePath = path_1.default.join(basePath, tableName);
+            var recs = filesToProcess[tableName].records;
+            var recKeys = Object.keys(recs);
+            await Promise.all(recKeys.map(function (k) {
+                return fUtils.createDirRecursively(path_1.default.join(tablePath, recs[k].name));
+            }));
+            await (0, genericUtils_1.processBatched)(recKeys, constants_1.CONCURRENCY_RECORDS, async function (recKey) {
+                var rec = recs[recKey];
+                var recPath = path_1.default.join(tablePath, rec.name);
+                // Split server-provided metadata off from the regular files so we can
+                // track whether any regular file actually changed — metaData shouldn't
+                // be the trigger for "this record changed" since we stamp it on every
+                // touch.
+                var metadataFiles = [];
+                var regularFiles = [];
+                for (var mi = 0; mi < rec.files.length; mi++) {
+                    var rf = rec.files[mi];
+                    if (rf.name === "metaData" && rf.type === "json") {
+                        metadataFiles.push(rf);
+                    }
+                    else {
+                        regularFiles.push(rf);
+                    }
+                }
+                var results = await (0, genericUtils_1.allSettledBatched)(regularFiles, constants_1.CONCURRENCY_FILES, async function (file) {
+                    if (forceWrite) {
+                        await forceWriter(file, recPath);
+                        return true;
+                    }
+                    return fUtils.writeSNFileIfDifferent(file, recPath);
+                });
+                var anyChanged = false;
+                for (var f = 0; f < results.length; f++) {
+                    var res = results[f];
+                    if (res.status === "rejected") {
+                        FileLogger_1.fileLogger.error("File write failed: " + res.reason);
+                        continue;
+                    }
+                    if (res.value) {
+                        anyChanged = true;
+                        writtenCount++;
+                    }
+                    else {
+                        unchangedCount++;
+                    }
+                }
+                // Only touch metaData when at least one regular file in the record
+                // actually changed. Avoids rewriting _lastUpdatedOn for records that
+                // were already in sync with the instance. Prefer the server-provided
+                // metadata (full field snapshot) over a stub; fall back to a stub only
+                // when the server didn't send metadata at all.
+                if (anyChanged || forceWrite) {
+                    let metadataFile;
+                    if (metadataFiles.length > 0 && metadataFiles[0].content) {
+                        metadataFile = stampMetadataContent(metadataFiles[0]);
+                    }
+                    else {
+                        metadataFile = {
+                            name: "metaData",
+                            type: "json",
+                            content: JSON.stringify({ _lastUpdatedOn: new Date().toISOString() }, null, 2),
+                        };
+                    }
+                    await forceWriter(metadataFile, recPath);
+                }
+                // Strip content from manifest entries to keep memory bounded.
+                rec.files = rec.files.map(function (file) {
+                    var copy = Object.assign({}, file);
+                    delete copy.content;
+                    return copy;
+                });
+                progress.tick();
+            });
+        });
+        FileLogger_1.fileLogger.debug("Refresh complete: " + writtenCount + " written, " + unchangedCount + " unchanged");
+        if (writtenCount > 0) {
+            Logger_1.logger.info("Refreshed " + writtenCount + " file(s) from instance" +
+                (unchangedCount > 0 ? " (" + unchangedCount + " already in sync)" : ""));
+        }
+        else {
+            Logger_1.logger.debug("No file changes detected from instance (" + unchangedCount + " checked)");
+        }
+        if (options.benchmarkCollector) {
+            options.benchmarkCollector.endScope(writtenCount, unchangedCount);
+        }
+    }
+    catch (e) {
+        if (options.benchmarkCollector) {
+            options.benchmarkCollector.endScope(0, 0);
+        }
+        throw e;
+    }
+};
+exports.refreshAllFiles = refreshAllFiles;
+const processMissingFiles = async (newManifest, sourcePath) => {
+    try {
+        const missing = await (0, exports.findMissingFiles)(newManifest, sourcePath);
+        const missingTableCount = Object.keys(missing).length;
+        if (missingTableCount === 0)
+            return;
+        FileLogger_1.fileLogger.debug("Downloading missing files from " + missingTableCount + " tables");
+        const { tableOptions = {} } = ConfigManager.getConfig();
+        const client = (0, snClient_1.defaultClient)();
+        // Chunk the bulkDownload request: ServiceNow rejects REST payloads > 10 MB,
+        // so send table batches and merge the results before processing.
+        const tableNames = Object.keys(missing);
+        const totalChunks = Math.ceil(tableNames.length / BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
+        const filesToProcess = {};
+        for (var i = 0; i < tableNames.length; i += BULK_DOWNLOAD_TABLE_CHUNK_SIZE) {
+            const chunkTableNames = tableNames.slice(i, i + BULK_DOWNLOAD_TABLE_CHUNK_SIZE);
+            const chunkMissing = {};
+            for (var j = 0; j < chunkTableNames.length; j++) {
+                chunkMissing[chunkTableNames[j]] = missing[chunkTableNames[j]];
+            }
+            const batchNum = Math.floor(i / BULK_DOWNLOAD_TABLE_CHUNK_SIZE) + 1;
+            FileLogger_1.fileLogger.debug("Bulk download batch " + batchNum + "/" + totalChunks +
+                " (" + chunkTableNames.length + " tables): " + chunkTableNames.join(", "));
+            const chunkResult = await (0, snClient_1.unwrapSNResponse)(client.getMissingFiles(chunkMissing, tableOptions));
+            // Chunks are partitioned by table key, so merging is a simple assign.
+            for (var tableName in chunkResult) {
+                filesToProcess[tableName] = chunkResult[tableName];
+            }
+        }
+        var recordCount = countRecordsInTables(filesToProcess);
+        var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
+            scope: newManifest.scope || "default",
+            total: recordCount,
+        });
+        await processTablesInManifest(filesToProcess, false, sourcePath, progress.tick, newManifest.scope);
+    }
+    catch (e) {
+        throw e;
+    }
+};
+exports.processMissingFiles = processMissingFiles;
+const groupAppFiles = (fileCtxs) => {
+    const combinedFiles = fileCtxs.reduce((groupMap, cur) => {
+        const { tableName, targetField, sys_id } = cur;
+        const key = `${tableName}-${sys_id}`;
+        const entry = groupMap[key] ?? {
+            table: tableName,
+            sysId: sys_id,
+            fields: {},
+        };
+        const newEntry = {
+            ...entry,
+            fields: { ...entry.fields, [targetField]: cur ?? "" },
+        };
+        return { ...groupMap, [key]: newEntry };
+    }, {});
+    return Object.values(combinedFiles);
+};
+exports.groupAppFiles = groupAppFiles;
+const getAppFileList = async (paths) => {
+    const validPaths = typeof paths === "object"
+        ? paths
+        : await fUtils.encodedPathsToFilePaths(paths);
+    const appFileCtxs = [];
+    validPaths.forEach(function (filePath) {
+        var result = fUtils.getFileContextWithSkipReason(filePath);
+        if (result.context) {
+            appFileCtxs.push(result.context);
+        }
+        else {
+            var reason = result.skipReason || "unknown";
+            if (reason === "not in manifest") {
+                Logger_1.logger.info(`Skipped: ${filePath} (${reason})`);
+            }
+            else {
+                Logger_1.logger.warn(`Skipped: ${filePath} (${reason})`);
+            }
+        }
+    });
+    return (0, exports.groupAppFiles)(appFileCtxs);
+};
+exports.getAppFileList = getAppFileList;
+const buildRec = async (rec) => {
+    const fields = Object.keys(rec.fields);
+    const buildPromises = fields.map((field) => {
+        return PluginManager_1.default.getFinalFileContents(rec.fields[field]);
+    });
+    const builtFiles = await (0, genericUtils_1.allSettled)(buildPromises);
+    const buildSuccess = !builtFiles.find((buildRes) => buildRes.status === "rejected");
+    if (!buildSuccess) {
+        return {
+            success: false,
+            message: (0, genericUtils_1.aggregateErrorMessages)(builtFiles
+                .filter((b) => b.status === "rejected")
+                .map((b) => b.reason), "Failed to build!", (_, index) => `${index}`),
+        };
+    }
+    const builtRec = builtFiles.reduce((acc, buildRes, index) => {
+        const { value: content } = buildRes;
+        const fieldName = fields[index];
+        return { ...acc, [fieldName]: content };
+    }, {});
+    return {
+        success: true,
+        builtRec,
+    };
+};
+const pushRec = async (client, table, sysId, builtRec, summary, scope, updateSetConfig) => {
+    const recSummary = summary ?? `${table} > ${sysId}`;
+    try {
+        // Use the batch-level config passed from pushFiles() to avoid re-reading per record
+        const config = updateSetConfig || {};
+        const updateSet = scope ? config[scope] : undefined;
+        const pushFn = updateSet
+            ? () => {
+                Logger_1.logger.debug(`Pushing ${recSummary} via update set: ${updateSet.name}`);
+                return client.pushWithUpdateSet(updateSet.sys_id, table, sysId, builtRec);
+            }
+            : () => client.updateRecord(table, sysId, builtRec);
+        const pushRes = await (0, snClient_1.retryOnHttpErr)(pushFn, recSummary);
+        return (0, snClient_1.processPushResponse)(pushRes, recSummary);
+    }
+    catch (e) {
+        let message;
+        if (e instanceof Error)
+            message = e.message;
+        else
+            message = String(e);
+        const errMsg = message || "Too many retries";
+        return { success: false, message: `${recSummary} : ${errMsg}` };
+    }
+};
+const pushFiles = async (recs) => {
+    const client = (0, snClient_1.defaultClient)();
+    const updateSetConfig = getUpdateSetConfig();
+    const hasUpdateSets = Object.keys(updateSetConfig).length > 0;
+    if (hasUpdateSets) {
+        const activeScopes = Object.entries(updateSetConfig)
+            .map(([scope, us]) => `${scope} -> ${us.name}`)
+            .join(", ");
+        Logger_1.logger.info(`Update set routing active: ${activeScopes}`);
+    }
+    // Pre-resolve read-only table sets for every scope present in this batch.
+    // Cheaper than re-resolving per record. Tables flagged in _readOnlyTables
+    // are pulled normally but pushes are skipped.
+    const readOnlyByScope = {};
+    for (const rec of recs) {
+        const fieldNames = Object.keys(rec.fields);
+        if (fieldNames.length === 0)
+            continue;
+        const scope = rec.fields[fieldNames[0]].scope;
+        if (scope && !readOnlyByScope[scope]) {
+            readOnlyByScope[scope] = ConfigManager.getReadOnlyTablesForScope(scope);
+        }
+    }
+    const announcedSkipTables = {};
+    const tick = getProgTick(Logger_1.logger.getLogLevel(), recs.length * 2) || (() => { });
+    const results = await (0, genericUtils_1.allSettledBatched)(recs, constants_1.CONCURRENCY_PUSH, async function (rec) {
+        const fieldNames = Object.keys(rec.fields);
+        const firstField = rec.fields[fieldNames[0]];
+        const recSummary = (0, exports.summarizeRecord)(rec.table, firstField.name);
+        const scope = firstField.scope;
+        const readOnlySet = scope ? readOnlyByScope[scope] : undefined;
+        if (readOnlySet && readOnlySet.has(rec.table)) {
+            tick();
+            tick();
+            const skipKey = `${scope}:${rec.table}`;
+            if (!announcedSkipTables[skipKey]) {
+                announcedSkipTables[skipKey] = true;
+                Logger_1.logger.info(`Read-only table ${rec.table} in scope ${scope}: push skipped`);
+            }
+            return { success: true, message: `${recSummary} : skipped (read-only table)` };
+        }
+        const buildRes = await buildRec(rec);
+        tick();
+        if (!buildRes.success) {
+            tick();
+            return { success: false, message: `${recSummary} : ${buildRes.message}` };
+        }
+        const pushRes = await pushRec(client, rec.table, rec.sysId, buildRes.builtRec, recSummary, scope, updateSetConfig);
+        tick();
+        return pushRes;
+    });
+    return results.map(function (result) {
+        if (result.status === "fulfilled") {
+            return result.value;
+        }
+        return { success: false, message: `Push failed: ${result.reason}` };
+    });
+};
+exports.pushFiles = pushFiles;
+const summarizeRecord = (table, recDescriptor) => `${table} > ${recDescriptor}`;
+exports.summarizeRecord = summarizeRecord;
+const createScopeProgress = (logLevel, options) => {
+    if (logLevel !== "info" || options.total === 0) {
+        return { tick: function () { }, setTotal: function () { } };
+    }
+    var progBar = new progress_1.default(":scope :bar :current/:total (:percent)", {
+        total: options.total,
+        width: 40,
+        complete: "=",
+        incomplete: "-",
+    });
+    return {
+        tick: function () {
+            progBar.tick({ scope: options.scope });
+        },
+        setTotal: function (n) {
+            progBar.total = n;
+        },
+    };
+};
+const getProgTick = (logLevel, total) => {
+    if (logLevel === "info") {
+        const progBar = new progress_1.default(":bar (:percent)", {
+            total,
+            width: 60,
+        });
+        return () => {
+            progBar.tick();
+        };
+    }
+    // no-op at other log levels
+    return undefined;
+};
+const writeBuildFile = async (preBuild, buildRes, summary) => {
+    const { fields, table, sysId } = preBuild;
+    const recSummary = summary ?? `${table} > ${sysId}`;
+    const sourcePath = ConfigManager.getSourcePath();
+    const buildPath = ConfigManager.getBuildPath();
+    const fieldNames = Object.keys(fields);
+    const writePromises = fieldNames.map(async (field) => {
+        const fieldCtx = fields[field];
+        const srcFilePath = fieldCtx.filePath;
+        const relativePath = path_1.default.relative(sourcePath, srcFilePath);
+        const relPathNoExt = relativePath.split(".").slice(0, -1).join();
+        const buildExt = fUtils.getBuildExt(fieldCtx.tableName, fieldCtx.name, fieldCtx.targetField, fieldCtx.scope);
+        const relPathNewExt = `${relPathNoExt}.${buildExt}`;
+        const buildFilePath = path_1.default.join(buildPath, relPathNewExt);
+        await fUtils.createDirRecursively(path_1.default.dirname(buildFilePath));
+        const writeResult = await fUtils.writeFileForce(buildFilePath, buildRes.builtRec[fieldCtx.targetField]);
+        return writeResult;
+    });
+    try {
+        await (0, genericUtils_1.processBatched)(fieldNames, constants_1.CONCURRENCY_FILES, async function (field) {
+            const fieldCtx = fields[field];
+            const srcFilePath = fieldCtx.filePath;
+            const relativePath = path_1.default.relative(sourcePath, srcFilePath);
+            const relPathNoExt = relativePath.split(".").slice(0, -1).join();
+            const buildExt = fUtils.getBuildExt(fieldCtx.tableName, fieldCtx.name, fieldCtx.targetField);
+            const relPathNewExt = `${relPathNoExt}.${buildExt}`;
+            const buildFilePath = path_1.default.join(buildPath, relPathNewExt);
+            await fUtils.createDirRecursively(path_1.default.dirname(buildFilePath));
+            await fUtils.writeFileForce(buildFilePath, buildRes.builtRec[fieldCtx.targetField]);
+        });
+        return { success: true, message: `${recSummary} built successfully` };
+    }
+    catch (e) {
+        return {
+            success: false,
+            message: `${recSummary} : ${e}`,
+        };
+    }
+};
+const buildFiles = async (fileList) => {
+    const tick = getProgTick(Logger_1.logger.getLogLevel(), fileList.length * 2) || (() => { });
+    const results = await (0, genericUtils_1.allSettledBatched)(fileList, constants_1.CONCURRENCY_BUILD, async function (rec) {
+        const { fields, table } = rec;
+        const fieldNames = Object.keys(fields);
+        const recSummary = (0, exports.summarizeRecord)(table, fields[fieldNames[0]].name);
+        const buildRes = await buildRec(rec);
+        tick();
+        if (!buildRes.success) {
+            tick();
+            return { success: false, message: `${recSummary} : ${buildRes.message}` };
+        }
+        // writeFile
+        const writeRes = await writeBuildFile(rec, buildRes, recSummary);
+        tick();
+        return writeRes;
+    });
+    return results.map(function (result) {
+        if (result.status === "fulfilled") {
+            return result.value;
+        }
+        return { success: false, message: `Build failed: ${result.reason}` };
+    });
+};
+exports.buildFiles = buildFiles;
+const swapScope = async (currentScope) => {
+    try {
+        const client = (0, snClient_1.defaultClient)();
+        const scopeId = await (0, snClient_1.unwrapTableAPIFirstItem)(client.getScopeId(currentScope), "sys_id");
+        await swapServerScope(scopeId);
+        const scopeObj = await (0, snClient_1.unwrapSNResponse)(client.getCurrentScope());
+        return scopeObj;
+    }
+    catch (e) {
+        throw e;
+    }
+};
+exports.swapScope = swapScope;
+const swapServerScope = async (scopeId) => {
+    try {
+        const client = (0, snClient_1.defaultClient)();
+        const userSysId = await (0, snClient_1.unwrapTableAPIFirstItem)(client.getUserSysId(), "sys_id");
+        const curAppUserPrefId = (await (0, snClient_1.unwrapTableAPIFirstItem)(client.getCurrentAppUserPrefSysId(userSysId), "sys_id")) || "";
+        // If not user pref record exists, create it.
+        if (curAppUserPrefId !== "")
+            await client.updateCurrentAppUserPref(scopeId, curAppUserPrefId);
+        else
+            await client.createCurrentAppUserPref(scopeId, userSysId);
+    }
+    catch (e) {
+        let message;
+        if (e instanceof Error)
+            message = e.message;
+        else
+            message = String(e);
+        Logger_1.logger.error(message);
+        throw e;
+    }
+};
+/**
+ * Creates a new update set and assigns it to the current user.
+ * @param updateSetName - does not create update set if value is blank
+ * @param scope - optional scope name (e.g. x_cadso_work) to create the update set in
+ */
+const createAndAssignUpdateSet = async (updateSetName = "", scope) => {
+    Logger_1.logger.info(`Update Set Name: ${updateSetName}` + (scope ? ` (scope: ${scope})` : ""));
+    const client = (0, snClient_1.defaultClient)();
+    var scopeSysId;
+    if (scope) {
+        var scopeResult = await (0, snClient_1.unwrapSNResponse)(client.getScopeId(scope));
+        if (scopeResult.length > 0) {
+            scopeSysId = scopeResult[0].sys_id;
+        }
+    }
+    const { sys_id: updateSetSysId } = await (0, snClient_1.unwrapSNResponse)(client.createUpdateSet(updateSetName, scopeSysId));
+    const userSysId = await (0, snClient_1.unwrapTableAPIFirstItem)(client.getUserSysId(), "sys_id");
+    const curUpdateSetUserPrefId = await (0, snClient_1.unwrapTableAPIFirstItem)(client.getCurrentUpdateSetUserPref(userSysId), "sys_id");
+    if (curUpdateSetUserPrefId !== "") {
+        await client.updateCurrentUpdateSetUserPref(updateSetSysId, curUpdateSetUserPrefId);
+    }
+    else {
+        await client.createCurrentUpdateSetUserPref(updateSetSysId, userSysId);
+    }
+    return {
+        name: updateSetName,
+        id: updateSetSysId,
+    };
+};
+exports.createAndAssignUpdateSet = createAndAssignUpdateSet;