datagrok-tools 4.14.33 → 4.14.35

package/bin/commands/test.js

@@ -129,7 +129,7 @@ async function runTesting(args) {
       let testsLeft = [];
       let testsToReproduce = [];
       for (let testData of organized) {
-        if (!r.verbosePassed.includes(`${testData.params.category}: ${testData.params.test}`) && !r.verboseSkipped.includes(`${testData.params.category}: ${testData.params.test}`) && !r.verboseFailed.includes(`${testData.params.category}: ${testData.params.test}`) && !new RegExp(`${testData.params.category.trim()}[^\\n]*: (before|after)`).test(r.verboseFailed)) testsLeft.push(testData);
+        if (!r.verbosePassed.includes(`${testData.params.category}: ${testData.params.test}`) && !r.verboseSkipped.includes(`${testData.params.category}: ${testData.params.test}`) && !r.verboseFailed.includes(`${testData.params.category}: ${testData.params.test}`) && !new RegExp(`${testUtils.escapeRegex(testData.params.category.trim())}[^\n]*: *?(before|after)(\\(\\))?`).test(r.verboseFailed)) testsLeft.push(testData);
         if (r.verboseFailed.includes(`${testData.params.category}: ${testData.params.test} :  Error:`)) {
           testsToReproduce.push(testData);
         }

package/bin/utils/test-utils.js

@@ -8,6 +8,7 @@ exports.TestContext = void 0;
 exports.addColumnToCsv = addColumnToCsv;
 exports.addLogsToFile = addLogsToFile;
 exports.defaultLaunchParameters = void 0;
+exports.escapeRegex = escapeRegex;
 exports.exitWithCode = exitWithCode;
 exports.getBrowserPage = getBrowserPage;
 exports.getDevKey = getDevKey;
@@ -182,22 +183,22 @@ const recorderConfig = exports.recorderConfig = {
   // aspectRatio: '16:9',
 };
 async function loadPackages(packagesDir, packagesToLoad, host, skipPublish, skipBuild, linkPackage, release) {
-  let packagesToRun = new Map();
-  let hostString = host === undefined ? `` : `${host}`;
-  if (packagesToLoad !== "all") {
-    for (let pacakgeName of (packagesToLoad ?? "").split(' ')) {
+  const packagesToRun = new Map();
+  const hostString = host === undefined ? `` : `${host}`;
+  if (packagesToLoad !== 'all') {
+    for (const pacakgeName of (packagesToLoad ?? '').split(' ')) {
       if ((pacakgeName ?? '').length !== 0) packagesToRun.set((0, _utils.spaceToCamelCase)(pacakgeName).toLocaleLowerCase(), false);
     }
   }
-  for (let dirName of _fs.default.readdirSync(packagesDir)) {
-    let packageDir = _path.default.join(packagesDir, dirName);
+  for (const dirName of _fs.default.readdirSync(packagesDir)) {
+    const packageDir = _path.default.join(packagesDir, dirName);
     if (!_fs.default.lstatSync(packageDir).isFile()) {
       try {
         const packageJsonData = JSON.parse(_fs.default.readFileSync(_path.default.join(packageDir, 'package.json'), {
           encoding: 'utf-8'
         }));
-        const packageFriendlyName = packagesToRun.get((0, _utils.spaceToCamelCase)(packageJsonData["friendlyName"] ?? packageJsonData["name"].split("/")[1] ?? packageJsonData["name"] ?? '').toLocaleLowerCase() ?? "") ?? packagesToRun.get(dirName);
-        if (utils.isPackageDir(packageDir) && (packageFriendlyName !== undefined || packagesToLoad === "all")) {
+        const packageFriendlyName = packagesToRun.get((0, _utils.spaceToCamelCase)(packageJsonData['friendlyName'] ?? packageJsonData['name'].split('/')[1] ?? packageJsonData['name'] ?? '').toLocaleLowerCase() ?? '') ?? packagesToRun.get(dirName);
+        if (utils.isPackageDir(packageDir) && (packageFriendlyName !== undefined || packagesToLoad === 'all')) {
           try {
             process.stdout.write(`Building and publishing ${dirName}...`);
             if (skipPublish != true) {
@@ -213,7 +214,7 @@ async function loadPackages(packagesDir, packagesToLoad, host, skipPublish, skip
           }
         }
       } catch (e) {
-        if (utils.isPackageDir(packageDir) && (packagesToRun.get((0, _utils.spaceToCamelCase)(dirName).toLocaleLowerCase()) !== undefined || packagesToLoad === "all")) console.log(`Couldn't read package.json  ${dirName}`);
+        if (utils.isPackageDir(packageDir) && (packagesToRun.get((0, _utils.spaceToCamelCase)(dirName).toLocaleLowerCase()) !== undefined || packagesToLoad === 'all')) console.log(`Couldn't read package.json  ${dirName}`);
       }
     }
   }
@@ -222,7 +223,7 @@ async function loadPackages(packagesDir, packagesToLoad, host, skipPublish, skip
   ;
 }
 async function loadTestsList(packages, core = false) {
-  var packageTestsData = await timeout(async () => {
+  const packageTestsData = await timeout(async () => {
     const params = Object.assign({}, defaultLaunchParameters);
     // params['headless'] = false;
     const out = await getBrowserPage(_puppeteer.default, params);
@@ -253,7 +254,7 @@ async function loadTestsList(packages, core = false) {
             }
           });
         } catch (err) {
-          console.error("Error during evaluation in browser context:", err);
+          console.error('Error during evaluation in browser context:', err);
           reject();
         }
         Promise.all(promises).then(results => {
@@ -267,16 +268,14 @@ async function loadTestsList(packages, core = false) {
         });
       });
     }, packages, core);
-    if (browser != null) {
-      await browser.close();
-    }
+    if (browser != null) await browser.close();
     return r;
   }, testCollectionTimeout);
-  let testsList = [];
-  for (let testPackage of packageTestsData) {
+  const testsList = [];
+  for (const testPackage of packageTestsData) {
     for (const key in testPackage.tests) {
       if (testPackage.tests.hasOwnProperty(key)) {
-        for (let testValue of testPackage.tests[key].tests) {
+        for (const testValue of testPackage.tests[key].tests) {
           testValue.packageName = testPackage.packageName;
           testsList.push(testValue);
         }
@@ -291,26 +290,24 @@ function addLogsToFile(filePath, stringToSave) {
 function printBrowsersResult(browserResult, verbose = false) {
   if (verbose) {
     if ((browserResult.passedAmount ?? 0) > 0 && (browserResult.verbosePassed ?? []).length > 0) {
-      console.log("Passed: ");
+      console.log('Passed: ');
       console.log(browserResult.verbosePassed);
     }
     if ((browserResult.skippedAmount ?? 0) > 0 && (browserResult.verboseSkipped ?? []).length > 0) {
-      console.log("Skipped: ");
+      console.log('Skipped: ');
       console.log(browserResult.verboseSkipped);
     }
   }
   if ((browserResult.failedAmount ?? 0) > 0 && (browserResult.verboseFailed ?? []).length > 0) {
-    console.log("Failed: ");
+    console.log('Failed: ');
     console.log(browserResult.verboseFailed);
   }
-  console.log("Passed amount:  " + browserResult?.passedAmount);
-  console.log("Skipped amount: " + browserResult?.skippedAmount);
-  console.log("Failed amount:  " + browserResult?.failedAmount);
+  console.log('Passed amount:  ' + browserResult?.passedAmount);
+  console.log('Skipped amount: ' + browserResult?.skippedAmount);
+  console.log('Failed amount:  ' + browserResult?.failedAmount);
   if (browserResult.failed) {
     if (browserResult.verboseFailed === 'Package not found') color.fail('Tests not found');else color.fail('Tests failed.');
-  } else {
-    color.success('Tests passed.');
-  }
+  } else color.success('Tests passed.');
 }
 function saveCsvResults(stringToSave, csvReportDir) {
   const modifiedStrings = stringToSave.map((str, index) => {
@@ -322,10 +319,10 @@ function saveCsvResults(stringToSave, csvReportDir) {
 }
 async function runTests(testsParams, stopOnFail) {
   let failed = false;
-  let verbosePassed = "";
-  let verboseSkipped = "";
-  let verboseFailed = "";
-  let error = "";
+  let verbosePassed = '';
+  let verboseSkipped = '';
+  let verboseFailed = '';
+  let error = '';
   let countPassed = 0;
   let countSkipped = 0;
   let countFailed = 0;
@@ -333,13 +330,15 @@ async function runTests(testsParams, stopOnFail) {
   let lastTest = null;
   let res = '';
   try {
-    for (let testParam of testsParams) {
+    for (const testParam of testsParams) {
       lastTest = testParam;
-      let df = await window.grok.functions.call(testParam.package + ':test', testParam.params);
-      let flakingCol = window.DG.Column.fromType(window.DG.COLUMN_TYPE.BOOL, 'flaking', df.rowCount);
+      const df = await window.grok.functions.call(testParam.package + ':test', testParam.params);
+      const flakingCol = window.DG.Column.fromType(window.DG.COLUMN_TYPE.BOOL, 'flaking', df.rowCount);
       df.columns.add(flakingCol);
-      let packageNameCol = window.DG.Column.fromList(window.DG.COLUMN_TYPE.STRING, 'package', Array(df.rowCount).fill(testParam.package));
-      df.columns.add(packageNameCol);
+      if (!df.getCol('package')) {
+        const packageNameCol = window.DG.Column.fromList(window.DG.COLUMN_TYPE.STRING, 'package', Array(df.rowCount).fill(testParam.package));
+        df.columns.add(packageNameCol);
+      }
       if (df.rowCount === 0) {
         verboseFailed += `Test result : Invocation Fail : ${testParam.params.category}: ${testParam.params.test}\n`;
         countFailed += 1;
@@ -348,29 +347,30 @@ async function runTests(testsParams, stopOnFail) {
       }
       let row = df.rows.get(0);
       if (df.rowCount > 1) {
-        let unhandledErrorRow = df.rows.get(1);
-        if (!unhandledErrorRow.get("success")) {
-          unhandledErrorRow["category"] = row.get("category");
-          unhandledErrorRow["name"] = row.get("name");
+        const unhandledErrorRow = df.rows.get(1);
+        if (!unhandledErrorRow.get('success')) {
+          unhandledErrorRow['category'] = row.get('category');
+          unhandledErrorRow['name'] = row.get('name');
           row = unhandledErrorRow;
         }
       }
-      const category = row.get("category");
-      const testName = row.get("name");
-      const time = row.get("ms");
-      const result = row.get("result");
-      const success = row.get("success");
-      const skipped = row.get("skipped");
-      row["flaking"] = success && window.DG.Test.isReproducing;
+      const category = row.get('category');
+      const testName = row.get('name');
+      const time = row.get('ms');
+      const result = row.get('result');
+      const success = row.get('success');
+      const skipped = row.get('skipped');
+      row['flaking'] = success && window.DG.Test.isReproducing;
       df.changeColumnType('result', window.DG.COLUMN_TYPE.STRING);
       df.changeColumnType('logs', window.DG.COLUMN_TYPE.STRING);
+      df.changeColumnType('widgetsDifference', window.DG.COLUMN_TYPE.STRING);
       // df.changeColumnType('memoryDelta', (<any>window).DG.COLUMN_TYPE.BIG_INT);
 
       if (resultDF === undefined) resultDF = df;else resultDF = resultDF.append(df);
-      if (row["skipped"]) {
+      if (row['skipped']) {
         verboseSkipped += `Test result : Skipped : ${time} : ${category}: ${testName} :  ${result}\n`;
         countSkipped += 1;
-      } else if (row["success"]) {
+      } else if (row['success']) {
         verbosePassed += `Test result : Success : ${time} : ${category}: ${testName} :  ${result}\n`;
         countPassed += 1;
       } else {
@@ -389,9 +389,7 @@ async function runTests(testsParams, stopOnFail) {
       const bs = window.DG.BitSet.create(resultDF.rowCount);
       bs.setAll(true);
       for (let i = 0; i < resultDF.rowCount; i++) {
-        if (resultDF.rows.get(i).get('category') === 'Unhandled exceptions') {
-          bs.set(i, false);
-        }
+        if (resultDF.rows.get(i).get('category') === 'Unhandled exceptions') bs.set(i, false);
       }
       resultDF = resultDF.clone(bs);
       res = resultDF.toCsv();
@@ -414,7 +412,7 @@ async function runTests(testsParams, stopOnFail) {
   };
 }
 async function runBrowser(testExecutionData, browserOptions, browsersId, testInvocationTimeout = 3600000) {
-  let testsToRun = {
+  const testsToRun = {
     func: runTests.toString(),
     tests: testExecutionData
   };
@@ -432,7 +430,7 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
     const recordDir = `./test-record-${currentBrowserNum}.mp4`;
     if (browserOptions.record) {
       await recorder.start(recordDir);
-      await page.exposeFunction("addLogsToFile", addLogsToFile);
+      await page.exposeFunction('addLogsToFile', addLogsToFile);
       _fs.default.writeFileSync(logsDir, ``);
       page.on('console', msg => {
         addLogsToFile(logsDir, `CONSOLE LOG ENTRY: ${msg.text()}\n`);
@@ -444,7 +442,7 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
         addLogsToFile(logsDir, `CONSOLE LOG REQUEST: ${response.status()}, ${response.url()}\n`);
       });
     }
-    let testingResults = await page.evaluate((testData, options) => {
+    const testingResults = await page.evaluate((testData, options) => {
       if (options.benchmark) window.DG.Test.isInBenchmark = true;
       if (options.reproduce) window.DG.Test.isReproducing = true;
       if (options.ciCd) window.DG.Test.isCiCd = true;
@@ -456,14 +454,14 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
         }).catch(e => {
           resolve({
             failed: true,
-            verbosePassed: "",
-            verboseSkipped: "",
-            verboseFailed: "",
+            verbosePassed: '',
+            verboseSkipped: '',
+            verboseFailed: '',
             error: JSON.stringify(e),
             passedAmount: 0,
             skippedAmount: 0,
             failedAmount: 1,
-            csv: "",
+            csv: '',
             df: undefined
           });
         });
@@ -487,17 +485,13 @@ async function runBrowser(testExecutionData, browserOptions, browsersId, testInv
         //   });
       });
     }, testsToRun, browserOptions);
-    if (browserOptions.record) {
-      await recorder.stop();
-    }
-    if (browser != null) {
-      await browser.close();
-    }
+    if (browserOptions.record) await recorder.stop();
+    if (browser != null) await browser.close();
     return testingResults;
   }, testInvocationTimeout);
 }
 async function mergeBrowsersResults(browsersResults) {
-  let mergedResult = {
+  const mergedResult = {
     failed: browsersResults[0].failed,
     verbosePassed: browsersResults[0].verbosePassed,
     verboseSkipped: browsersResults[0].verboseSkipped,
@@ -508,7 +502,7 @@ async function mergeBrowsersResults(browsersResults) {
     csv: browsersResults[0].csv,
     error: ''
   };
-  for (let browsersResult of browsersResults) {
+  for (const browsersResult of browsersResults) {
     if (mergedResult.csv === browsersResult.csv) continue;
     mergedResult.failed = mergedResult.failed || browsersResult.failed;
     mergedResult.verbosePassed = `${mergedResult.verbosePassed.trim()}\n${browsersResult.verbosePassed.trim()}`;
@@ -549,4 +543,7 @@ async function addColumnToCsv(csv, columnName, defaultValue) {
     }
   });
   return result;
+}
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }

package/package-template/package.json

@@ -10,7 +10,7 @@
     "@datagrok-libraries/utils": "^4.6.0"
   },
   "devDependencies": {
-    "datagrok-tools": "^4.14.32",
+    "datagrok-tools": "^4.14.34",
     "webpack": "^5.95.0",
     "webpack-cli": "^5.1.4"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datagrok-tools",
-  "version": "4.14.33",
+  "version": "4.14.35",
   "description": "Utility to upload and publish packages to Datagrok",
   "homepage": "https://github.com/datagrok-ai/public/tree/master/tools#readme",
   "dependencies": {
@@ -30,7 +30,8 @@
     "link": "npm link",
     "prepublishOnly": "babel bin --extensions .ts -d bin",
     "babel": "babel bin --extensions .ts -d bin",
-    "build": "babel bin --extensions .ts -d bin"
+    "build": "babel bin --extensions .ts -d bin",
+    "debug-source-map": "babel bin --extensions .ts -d bin --source-maps true"
   },
   "bin": {
     "datagrok-upload": "./bin/_deprecated/upload.js",