@tenonhq/sincronia-core 0.0.72 → 0.0.73

package/dist/allScopesCommands.js

@@ -32,6 +32,9 @@ var __importStar = (this && this.__importStar) || (function () {
         return result;
     };
 })();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.initScopesCommand = initScopesCommand;
 exports.watchAllScopesCommand = watchAllScopesCommand;
@@ -44,6 +47,7 @@ const commands_1 = require("./commands");
 const path = __importStar(require("path"));
 const fs = __importStar(require("fs"));
 const child_process_1 = require("child_process");
+const progress_1 = __importDefault(require("progress"));
 const fsp = fs.promises;
 // Custom function to process manifest with specific source directory
 async function processManifestForScope(manifest, sourceDirectory, forceWrite = false) {
@@ -58,7 +62,20 @@ async function processManifestForScope(manifest, sourceDirectory, forceWrite = f
         }
         const tables = manifest.tables || {};
         const tableNames = Object.keys(tables);
-        Logger_1.logger.info("Processing " + tableNames.length + " tables for " + sourceDirectory);
+        var totalRecords = 0;
+        for (var tn = 0; tn < tableNames.length; tn++) {
+            totalRecords += Object.keys(tables[tableNames[tn]].records || {}).length;
+        }
+        var scopeLabel = sourceDirectory.split(path.sep).pop() || "scope";
+        var progBar = null;
+        if (Logger_1.logger.getLogLevel() === "info" && totalRecords > 0) {
+            progBar = new progress_1.default(":scope :bar :current/:total (:percent)", {
+                total: totalRecords,
+                width: 40,
+                complete: "=",
+                incomplete: "-",
+            });
+        }
         for (const tableName of tableNames) {
             const tableRecords = tables[tableName];
             const tablePath = path.join(sourceDirectory, tableName);
@@ -102,9 +119,10 @@ async function processManifestForScope(manifest, sourceDirectory, forceWrite = f
                         Logger_1.logger.error("Failed to write metadata: " + metadataFilePath);
                     }
                 }
+                if (progBar)
+                    progBar.tick({ scope: scopeLabel });
             }
         }
-        Logger_1.logger.info("Files written to " + sourceDirectory);
     }
     catch (error) {
         Logger_1.logger.error("Error processing files for " + sourceDirectory + ": " + error);
@@ -340,7 +358,7 @@ async function watchAllScopesCommand(args) {
         }
         // Start dashboard unless --no-dashboard flag is set
         if (!args.noDashboard) {
-            dashboardProcess = startDashboardProcess();
+            dashboardProcess = startDashboardProcess(args.port);
         }
         // Import and start the multi-scope watcher
         const { startMultiScopeWatching } = await Promise.resolve().then(() => __importStar(require("./MultiScopeWatcher")));
@@ -365,7 +383,7 @@ async function watchAllScopesCommand(args) {
         throw error;
     }
 }
-function startDashboardProcess() {
+function startDashboardProcess(portOverride) {
     var serverPath;
     try {
         serverPath = require.resolve("@tenonhq/sincronia-dashboard/server.js");
@@ -374,11 +392,11 @@ function startDashboardProcess() {
         Logger_1.logger.warn("Dashboard package not installed. Run: npm install @tenonhq/sincronia-dashboard");
         return null;
     }
-    var port = process.env.DASHBOARD_PORT || "3456";
+    var port = portOverride ? String(portOverride) : (process.env.DASHBOARD_PORT || "3456");
     var server = (0, child_process_1.spawn)("node", [serverPath], {
         cwd: process.cwd(),
         stdio: "ignore",
-        env: { ...process.env },
+        env: { ...process.env, DASHBOARD_PORT: port },
         detached: false,
     });
     server.on("error", function (err) {

package/dist/appUtils.js

@@ -72,10 +72,7 @@ const processFilesInManRec = async (recPath, rec, forceWrite) => {
         }, null, 2)
     };
     await fileWrite(metadataFile, recPath);
-    const regularPromises = rec.files.map((file) => {
-        return fileWrite(file, recPath);
-    });
-    const writeResults = await Promise.allSettled(regularPromises);
+    const writeResults = await (0, genericUtils_1.allSettledBatched)(rec.files, constants_1.CONCURRENCY_FILES, function (file) { return fileWrite(file, recPath); });
     const writeFailures = writeResults.filter((r) => r.status === "rejected");
     if (writeFailures.length > 0) {
         writeFailures.forEach((f) => {
@@ -89,7 +86,7 @@ const processFilesInManRec = async (recPath, rec, forceWrite) => {
         return fileCopy;
     });
 };
-const processRecsInManTable = async (tablePath, table, forceWrite) => {
+const processRecsInManTable = async (tablePath, table, forceWrite, onRecordProcessed) => {
     const { records } = table;
     const recKeys = Object.keys(records);
     const recKeyToPath = (key) => path_1.default.join(tablePath, records[key].name);
@@ -97,26 +94,34 @@ const processRecsInManTable = async (tablePath, table, forceWrite) => {
         .map(recKeyToPath)
         .map(fUtils.createDirRecursively);
     await Promise.all(recPathPromises);
-    const filePromises = recKeys.reduce((acc, recKey) => {
-        return [
-            ...acc,
-            processFilesInManRec(recKeyToPath(recKey), records[recKey], forceWrite),
-        ];
-    }, []);
-    return Promise.all(filePromises);
+    await (0, genericUtils_1.processBatched)(recKeys, constants_1.CONCURRENCY_RECORDS, function (recKey) {
+        return processFilesInManRec(recKeyToPath(recKey), records[recKey], forceWrite).then(function () {
+            if (onRecordProcessed)
+                onRecordProcessed();
+        });
+    });
+};
+const countRecordsInTables = (tables) => {
+    return Object.keys(tables).reduce(function (sum, tableName) {
+        return sum + Object.keys(tables[tableName].records).length;
+    }, 0);
 };
-const processTablesInManifest = async (tables, forceWrite, sourcePath) => {
+const processTablesInManifest = async (tables, forceWrite, sourcePath, onRecordProcessed) => {
     var basePath = sourcePath || ConfigManager.getSourcePath();
     const tableNames = Object.keys(tables);
-    const tablePromises = tableNames.map((tableName) => {
-        return processRecsInManTable(path_1.default.join(basePath, tableName), tables[tableName], forceWrite);
+    await (0, genericUtils_1.processBatched)(tableNames, constants_1.CONCURRENCY_TABLES, function (tableName) {
+        return processRecsInManTable(path_1.default.join(basePath, tableName), tables[tableName], forceWrite, onRecordProcessed);
     });
-    await Promise.all(tablePromises);
 };
 const processManifest = async (manifest, forceWrite = false, sourcePath) => {
     const tableCount = Object.keys(manifest.tables).length;
     FileLogger_1.fileLogger.debug("Processing manifest: " + (manifest.scope || "legacy") + " (" + tableCount + " tables)");
-    await processTablesInManifest(manifest.tables, forceWrite, sourcePath);
+    var recordCount = countRecordsInTables(manifest.tables);
+    var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
+        scope: manifest.scope || "default",
+        total: recordCount,
+    });
+    await processTablesInManifest(manifest.tables, forceWrite, sourcePath, progress.tick);
     if (manifest.scope) {
         await fUtils.writeScopeManifest(manifest.scope, manifest);
     }
@@ -245,13 +250,18 @@ const processMissingFiles = async (newManifest, sourcePath) => {
     try {
         const missing = await (0, exports.findMissingFiles)(newManifest, sourcePath);
         const missingTableCount = Object.keys(missing).length;
-        if (missingTableCount > 0) {
-            FileLogger_1.fileLogger.debug("Downloading missing files from " + missingTableCount + " tables");
-        }
+        if (missingTableCount === 0)
+            return;
+        FileLogger_1.fileLogger.debug("Downloading missing files from " + missingTableCount + " tables");
         const { tableOptions = {} } = ConfigManager.getConfig();
         const client = (0, snClient_1.defaultClient)();
         const filesToProcess = await (0, snClient_1.unwrapSNResponse)(client.getMissingFiles(missing, tableOptions));
-        await processTablesInManifest(filesToProcess, false, sourcePath);
+        var recordCount = countRecordsInTables(filesToProcess);
+        var progress = createScopeProgress(Logger_1.logger.getLogLevel(), {
+            scope: newManifest.scope || "default",
+            total: recordCount,
+        });
+        await processTablesInManifest(filesToProcess, false, sourcePath, progress.tick);
     }
     catch (e) {
         throw e;
@@ -349,7 +359,7 @@ const pushFiles = async (recs) => {
         Logger_1.logger.info(`Update set routing active: ${activeScopes}`);
     }
     const tick = getProgTick(Logger_1.logger.getLogLevel(), recs.length * 2) || (() => { });
-    const pushResultPromises = recs.map(async (rec) => {
+    const results = await (0, genericUtils_1.allSettledBatched)(recs, constants_1.CONCURRENCY_PUSH, async function (rec) {
         const fieldNames = Object.keys(rec.fields);
         const firstField = rec.fields[fieldNames[0]];
         const recSummary = (0, exports.summarizeRecord)(rec.table, firstField.name);
@@ -364,8 +374,7 @@ const pushFiles = async (recs) => {
         tick();
         return pushRes;
     });
-    const results = await Promise.allSettled(pushResultPromises);
-    return results.map((result) => {
+    return results.map(function (result) {
         if (result.status === "fulfilled") {
             return result.value;
         }
@@ -375,6 +384,25 @@ const pushFiles = async (recs) => {
 exports.pushFiles = pushFiles;
 const summarizeRecord = (table, recDescriptor) => `${table} > ${recDescriptor}`;
 exports.summarizeRecord = summarizeRecord;
+const createScopeProgress = (logLevel, options) => {
+    if (logLevel !== "info" || options.total === 0) {
+        return { tick: function () { }, setTotal: function () { } };
+    }
+    var progBar = new progress_1.default(":scope :bar :current/:total (:percent)", {
+        total: options.total,
+        width: 40,
+        complete: "=",
+        incomplete: "-",
+    });
+    return {
+        tick: function () {
+            progBar.tick({ scope: options.scope });
+        },
+        setTotal: function (n) {
+            progBar.total = n;
+        },
+    };
+};
 const getProgTick = (logLevel, total) => {
     if (logLevel === "info") {
         const progBar = new progress_1.default(":bar (:percent)", {
@@ -394,20 +422,18 @@ const writeBuildFile = async (preBuild, buildRes, summary) => {
     const sourcePath = ConfigManager.getSourcePath();
     const buildPath = ConfigManager.getBuildPath();
     const fieldNames = Object.keys(fields);
-    const writePromises = fieldNames.map(async (field) => {
-        const fieldCtx = fields[field];
-        const srcFilePath = fieldCtx.filePath;
-        const relativePath = path_1.default.relative(sourcePath, srcFilePath);
-        const relPathNoExt = relativePath.split(".").slice(0, -1).join();
-        const buildExt = fUtils.getBuildExt(fieldCtx.tableName, fieldCtx.name, fieldCtx.targetField);
-        const relPathNewExt = `${relPathNoExt}.${buildExt}`;
-        const buildFilePath = path_1.default.join(buildPath, relPathNewExt);
-        await fUtils.createDirRecursively(path_1.default.dirname(buildFilePath));
-        const writeResult = await fUtils.writeFileForce(buildFilePath, buildRes.builtRec[fieldCtx.targetField]);
-        return writeResult;
-    });
     try {
-        await Promise.all(writePromises);
+        await (0, genericUtils_1.processBatched)(fieldNames, constants_1.CONCURRENCY_FILES, async function (field) {
+            const fieldCtx = fields[field];
+            const srcFilePath = fieldCtx.filePath;
+            const relativePath = path_1.default.relative(sourcePath, srcFilePath);
+            const relPathNoExt = relativePath.split(".").slice(0, -1).join();
+            const buildExt = fUtils.getBuildExt(fieldCtx.tableName, fieldCtx.name, fieldCtx.targetField);
+            const relPathNewExt = `${relPathNoExt}.${buildExt}`;
+            const buildFilePath = path_1.default.join(buildPath, relPathNewExt);
+            await fUtils.createDirRecursively(path_1.default.dirname(buildFilePath));
+            await fUtils.writeFileForce(buildFilePath, buildRes.builtRec[fieldCtx.targetField]);
+        });
         return { success: true, message: `${recSummary} built successfully` };
     }
     catch (e) {
@@ -419,7 +445,7 @@ const writeBuildFile = async (preBuild, buildRes, summary) => {
 };
 const buildFiles = async (fileList) => {
     const tick = getProgTick(Logger_1.logger.getLogLevel(), fileList.length * 2) || (() => { });
-    const buildPromises = fileList.map(async (rec) => {
+    const results = await (0, genericUtils_1.allSettledBatched)(fileList, constants_1.CONCURRENCY_BUILD, async function (rec) {
         const { fields, table } = rec;
         const fieldNames = Object.keys(fields);
         const recSummary = (0, exports.summarizeRecord)(table, fields[fieldNames[0]].name);
@@ -434,8 +460,7 @@ const buildFiles = async (fileList) => {
         tick();
         return writeRes;
     });
-    const results = await Promise.allSettled(buildPromises);
-    return results.map((result) => {
+    return results.map(function (result) {
         if (result.status === "fulfilled") {
             return result.value;
         }

package/dist/commander.js

@@ -30,6 +30,11 @@ async function initCommands() {
                 default: false,
                 describe: "Skip launching the dashboard web UI",
             },
+            port: {
+                alias: "p",
+                type: "number",
+                describe: "Dashboard port (default: DASHBOARD_PORT env or 3456)",
+            },
         });
         return cmdArgs;
     }, async (args) => {
@@ -286,7 +291,19 @@ async function initCommands() {
         });
         return cmdArgs;
     }, updateSetCommands_1.showCurrentScopeCommand)
-        .command("dashboard", "Launch the Update Set Dashboard web UI", sharedOptions, dashboardCommand_1.dashboardCommand)
+        .command("dashboard", "Launch the Update Set Dashboard web UI", (cmdArgs) => {
+        cmdArgs.options({
+            ...sharedOptions,
+            port: {
+                alias: "p",
+                type: "number",
+                describe: "Dashboard port (default: DASHBOARD_PORT env or 3456)",
+            },
+        });
+        return cmdArgs;
+    }, async (args) => {
+        await (0, dashboardCommand_1.dashboardCommand)(args);
+    })
         .command("schema <subcommand>", "Manage ServiceNow table schemas (subcommands: pull)", (cmdArgs) => {
         cmdArgs.positional("subcommand", {
             describe: "Schema subcommand to run",

package/dist/commands.js

@@ -233,7 +233,6 @@ async function downloadCommand(args) {
         });
         Logger_1.logger.info("Received " + tableCount + " tables, " + recordCount + " records");
         FileLogger_1.fileLogger.debug("Download manifest: " + tableCount + " tables, " + recordCount + " records");
-        Logger_1.logger.info("Writing files to disk...");
         await AppUtils.processManifest(man, true);
         Logger_1.logger.success("Download complete — " + recordCount + " records written");
     }

package/dist/constants.js

@@ -3,8 +3,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PUSH_RETRY_LIMIT = exports.PUSH_RETRY_WAIT = exports.PATH_DELIMITER = void 0;
+exports.CONCURRENCY_BUILD = exports.CONCURRENCY_PUSH = exports.CONCURRENCY_FILES = exports.CONCURRENCY_RECORDS = exports.CONCURRENCY_TABLES = exports.PUSH_RETRY_LIMIT = exports.PUSH_RETRY_WAIT = exports.PATH_DELIMITER = void 0;
 const path_1 = __importDefault(require("path"));
 exports.PATH_DELIMITER = `${path_1.default.delimiter}${path_1.default.delimiter}`;
 exports.PUSH_RETRY_WAIT = 3000;
 exports.PUSH_RETRY_LIMIT = 3;
+exports.CONCURRENCY_TABLES = 2;
+exports.CONCURRENCY_RECORDS = 5;
+exports.CONCURRENCY_FILES = 10;
+exports.CONCURRENCY_PUSH = 3;
+exports.CONCURRENCY_BUILD = 5;

package/dist/dashboardCommand.js

@@ -13,16 +13,16 @@ async function dashboardCommand(args) {
     catch (e) {
         throw new Error("Dashboard package not installed. Run: npm install @tenonhq/sincronia-dashboard");
     }
+    var port = args.port ? String(args.port) : (process.env.DASHBOARD_PORT || "3456");
     Logger_1.logger.info("Starting Update Set Dashboard...");
     const server = (0, child_process_1.spawn)("node", [serverPath], {
         cwd: process.cwd(),
         stdio: "inherit",
-        env: { ...process.env },
+        env: { ...process.env, DASHBOARD_PORT: port },
     });
     // Open browser after a short delay
     setTimeout(() => {
-        const port = process.env.DASHBOARD_PORT || "3456";
-        const url = `http://localhost:${port}`;
+        const url = "http://localhost:" + port;
         (0, child_process_1.spawn)("open", [url]);
     }, 1000);
     server.on("close", (code) => {

package/dist/genericUtils.js

@@ -41,6 +41,8 @@ exports.parseFileNameParams = parseFileNameParams;
 exports.getParsedFilesPayload = getParsedFilesPayload;
 exports.wait = wait;
 exports.chunkArr = chunkArr;
+exports.processBatched = processBatched;
+exports.allSettledBatched = allSettledBatched;
 const path_1 = __importDefault(require("path"));
 const ConfigManager = __importStar(require("./config"));
 async function _getConfigFromPath(params) {
@@ -125,6 +127,48 @@ function chunkArr(arr, chunkSize) {
     }
     return chunks;
 }
+async function processBatched(items, concurrency, fn) {
+    var results = new Array(items.length);
+    var index = 0;
+    async function runNext() {
+        while (index < items.length) {
+            var currentIndex = index;
+            index++;
+            results[currentIndex] = await fn(items[currentIndex]);
+        }
+    }
+    var workers = [];
+    var workerCount = Math.min(concurrency, items.length);
+    for (var i = 0; i < workerCount; i++) {
+        workers.push(runNext());
+    }
+    await Promise.all(workers);
+    return results;
+}
+async function allSettledBatched(items, concurrency, fn) {
+    var results = new Array(items.length);
+    var index = 0;
+    async function runNext() {
+        while (index < items.length) {
+            var currentIndex = index;
+            index++;
+            try {
+                var value = await fn(items[currentIndex]);
+                results[currentIndex] = { status: "fulfilled", value: value };
+            }
+            catch (e) {
+                results[currentIndex] = { status: "rejected", reason: e };
+            }
+        }
+    }
+    var workers = [];
+    var workerCount = Math.min(concurrency, items.length);
+    for (var i = 0; i < workerCount; i++) {
+        workers.push(runNext());
+    }
+    await Promise.all(workers);
+    return results;
+}
 const allSettled = (promises) => {
     return Promise.all(promises.map((prom) => prom
         .then((value) => ({

package/dist/initSystem/discovery.js

@@ -37,7 +37,16 @@ exports.discoverPlugins = discoverPlugins;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const Logger_1 = require("../Logger");
-const SKIP_PACKAGES = new Set(["sincronia-core", "sincronia-types"]);
+// Skip packages that are not init plugins:
+// - core/types: the discovery system itself
+// - dashboard: starts Express server on require (side effect)
+// - schema: library imported directly by schemaCommand, not a plugin
+const SKIP_PACKAGES = new Set([
+    "sincronia-core",
+    "sincronia-types",
+    "sincronia-dashboard",
+    "sincronia-schema",
+]);
 const MAX_PARENT_DEPTH = 3;
 /**
  * @description Scans node_modules for @tenonhq/sincronia-* packages that export a sincPlugin.

package/dist/tests/discovery.test.js

@@ -53,4 +53,15 @@ describe("discoverPlugins", function () {
         const names = plugins.map(function (p) { return p.name; });
         expect(names).not.toContain("sass-plugin");
     });
+    it("skips sincronia-dashboard and sincronia-schema", function () {
+        Logger_1.logger.warn.mockClear();
+        mockReaddirSync.mockReturnValue([
+            "sincronia-dashboard",
+            "sincronia-schema",
+        ]);
+        const plugins = (0, discovery_1.discoverPlugins)();
+        expect(plugins).toEqual([]);
+        // No require() attempted, so no warn from load failure
+        expect(Logger_1.logger.warn).not.toHaveBeenCalled();
+    });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tenonhq/sincronia-core",
-  "version": "0.0.72",
+  "version": "0.0.73",
   "description": "Next-gen file syncer",
   "license": "GPL-3.0",
   "main": "./dist/index.js",