@testim/testim-cli 3.221.0 → 3.225.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testim/testim-cli",
-  "version": "3.221.0",
+  "version": "3.225.0",
   "description": "Command line interface for running Testing on your CI",
   "author": "Oren Rubin",
   "contributors": [{

package/reports/debugReporter.js

@@ -25,11 +25,12 @@ DebugReporter.prototype.onTestStarted = function (test, workerId) {
 };
 
 DebugReporter.prototype.onTestFinished = function (test, workerId) {
-    const provider = this.options.gridData.provider;
-    const host = this.options.gridData.host;
-    const gridId = this.options.gridData.gridId;
-    const gridType = this.options.gridData.type;
-    const failedGetBrowserAttempts = this.options.gridData.failedGetBrowserAttempts;
+    const gridData = this.options.gridData || {};
+    const provider = gridData.provider;
+    const host = gridData.host;
+    const gridId = gridData.gridId;
+    const gridType = gridData.type;
+    const failedGetBrowserAttempts = gridData.failedGetBrowserAttempts;
     logger.info('Test Finished', {
         testId: test.testId,
         testName: test.name,

package/runOptions.js

@@ -158,16 +158,16 @@ program
     .option('-o --options-file [options-file.json]', '')
     .option('-c --config-file [config-file.js]', '')
     .option('--test-config [test-config]', 'test config name to override for all tests in current execution', collect, [])
-    .option('--test-config-id [test-config-id]', 'test config id to override for all tests in current execution', collect, [])
+    .option('--test-config-id [test-config-id]', 'test config ID to override for all tests in current execution', collect, [])
     .option('--params-file [params-file.json]', '')
     .option('--params [params-json-string]', '')
-    .option('-t, --testId [test-id]', 'test id to run', collect, [])
+    .option('-t, --testId [test-id]', 'test ID to run', collect, [])
     .option('run [file-glob-pattern]', 'codeful test files to run', collect, [])
     .option('-w, --webpackConfig [webpack-configuration]', 'webpack configuration used to build the code based project')
-    .option('--test-id [test-id]', 'test id to run', collect, [])
+    .option('--test-id [test-id]', 'test ID to run', collect, [])
     .option('-l, --label [label]', 'labels to search test by', collect, [])
     .option('-n, --name [test-name]', 'test name to run', collect, [])
-    .option('--project [project-id]', 'project id')
+    .option('--project [project-id]', 'project ID')
     .option('-r, --report-file [report junit xml path]', 'where to save junit xml results file')
     .option('--override-report-file-classname [override-report-file-classname]', 'custom junit class name for the junit reporter')
     .option('--reporters [names]', 'report types', list)
@@ -178,7 +178,7 @@ program
     .option('--protocol [grid-protocol]', 'grid protocol http or https')
     .option('--grid-username [grid-username]', 'grid http basic auth username')
     .option('--grid-password [grid-password]', 'grid http basic auth password')
-    .option('-gi --grid-id [grid-id]', 'grid id')
+    .option('-gi --grid-id [grid-id]', 'grid ID')
     .option('-b, --browser [browser-type]', 'browser type (chrome/firefox)')
     .option('-h, --headless [headless]', 'run in headless mode')
     .option('-m, --mode [runner-mode]', 'use extension or selenium mode (extension/selenium/appium)')
@@ -191,7 +191,7 @@ program
     .option('--test-plan [test-plan-name]', 'test plan to run', collect, [])
     .option('--test-plan-id [test-plan-id]', 'test plan to run', collect, [])
     .option('--suite [suite-name]', 'suite to run', collect, [])
-    .option('--suite-id [suite-id]', 'suite id to run', collect, [])
+    .option('--suite-id [suite-id]', 'suite ID to run', collect, [])
     .option('--rerun-failed-by-run-id [run-id]', 'allows re-running failed tests from a specific run ID')
     .option('--disable-grid-check [boolean]', 'disable checking if selenium grid is available', false)
     .option('--disable-native-events [boolean]', 'pass nativeEvents=false capability to the selenium browser (in selenium mode)', false)
@@ -203,7 +203,7 @@ program
     .option('-oen --override-execution-name [execution-name]', 'override the default execution name', '')
     .option('--retries [max_num_of_retries]', 'number of retires failure test defaults to not retrying', 0)
     .option('--set-retention [retention-in-days]', 'set the number of days for results retention')
-    .option('--user [user-id]', 'user id for local Testim-CLI')
+    .option('--user [user-id]', 'user ID for local Testim-CLI')
     .option('--pass-zero-tests', 'don\'t fail the run if no tests were found')
 
     .option('-str --suppress-tms-reporting [suppress-tms-reporting]', 'disable test management reporting', false)
@@ -301,7 +301,7 @@ program
     .option('--tunnel-region [tunnel-region]', 'ngrok tunnel region')
     .option('--tunnel-diagnostics', 'collect ngrok tunnel diagnostics')
     .option('--tunnel-use-http-address [tunnel-use-http-address]', 'use http:// address instead of https://', false)
-    .option('--external-lambdatest-tunnel-id [tunnel-id]', 'use existing lambdatest tunnel id')
+    .option('--external-lambdatest-tunnel-id [tunnel-id]', 'use existing lambdatest tunnel ID')
     .option('--external-lambdatest-use-wss', 'use wss instead of ssh for LT', false)
     .option('--external-lambdatest-disable-automation-tunneling', 'don\'t tunnel Testim calls in LT tunnel', true)
 
@@ -337,13 +337,18 @@ program
     .option('--high-speed', 'DEPRECATED: use --turbo-mode instead') // When removing, remove from the program.help output filter
     .option('--turbo-mode', 'run in turbo mode')
     .option('--lightweight-mode [settings]', 'run lightweight mode')
-    .option('--create-prefeched-data [location]', 'prefech data into local cache file')
-    .option('--use-prefeched-data [location]', 'use prefeched data from local cache file, and force using only cached data')
+    .option('--create-prefeched-data [location]', 'prefetch data into local cache file')
+    .option('--use-prefeched-data [location]', 'use prefetched data from local cache file, and force using only cached data')
     .option('--save-rca-locally [path]', 'save root cause analysis assets locally')
 
     .option('--exit-code-ignore-failing-tests', 'dont return non zero exit code when tests fail. non zero exit code will mean a real error occurred')
+
+    .option('--intersect-with-label [label]', 'Out of the execution\'s test list, run only those tests that have the specified label', collect, [])
+    .option('--intersect-with-suite [suiteName]', 'Out of the execution\'s test list, run only those tests that are included in the specified suite (by suite name)', collect, [])
+    .option('--intersect-with-suite-id [suiteId]', 'Out of the execution\'s test list, run only those tests that are included in the specified suite (by suite ID)', collect, [])
     .parse(process.argv);
 
+
 module.exports = {
     async process() {
         if (program.inspect) {
@@ -1179,6 +1184,13 @@ module.exports = {
             exitCodeIgnoreFailingTests: program.exitCodeIgnoreFailingTests,
 
             disableSockets: program.disableSockets,
+
+            // intersections
+            intersections: {
+                labels: program.intersectWithLabel.length ? [program.intersectWithLabel].flat() : undefined,
+                suiteNames: program.intersectWithSuite.length ? [program.intersectWithSuite].flat() : undefined,
+                suiteIds: program.intersectWithSuiteId.length ? [program.intersectWithSuiteId].flat() : undefined,
+            },
         });
     },
 };

package/runner.js

@@ -223,7 +223,7 @@ function runRunner(options, customExtensionLocalLocation) {
 }
 
 function showFreeGridRunWarningIfNeeded(options) {
-    if (featureAvailabilityService.shouldShowFreeGridRunWarning(options.gridData.type)) {
+    if (featureAvailabilityService.shouldShowFreeGridRunWarning(options.gridData && options.gridData.type)) {
         const CYAN = '\x1b[36m';
         const UNDERSCORE = '\x1b[4m';
         const RESET = '\x1b[0m';

package/runners/TestPlanRunner.js

@@ -11,6 +11,7 @@ const testimServicesApi = require('../commons/testimServicesApi');
 const testimCustomToken = require('../commons/testimCustomToken');
 const TestRunStatus = require('../testRunStatus');
 const analyticsService = require('../services/analyticsService');
+const gridService = require('../services/gridService');
 const branchService = require('../services/branchService');
 const config = require('../commons/config');
 const ParallelWorkerManager = require('./ParallelWorkerManager');
@@ -230,7 +231,7 @@ class TestPlanRunner {
         const testPlansTests = {};
         const projectId = options.project;
 
-        const data = await testimServicesApi.getTestPlanTestList(projectId, options.testPlan, options.testPlanIds, branchToUse);
+        const data = await testimServicesApi.getTestPlanTestList(projectId, options.testPlan, options.testPlanIds, branchToUse, options.intersections);
         const testPlans = data.testPlans;
         const testPlansData = data.testPlansData;
         if (!testPlans || testPlans.length === 0) {
@@ -258,6 +259,9 @@ class TestPlanRunner {
                 delete tpOptions.gridId;
             }
 
+
+            tpOptions.gridData = await gridService.getTestPlanGridData(options, testPlan);
+
             const testPlanName = tpOptions.overrideExecutionName || testPlan.name;
             return await Promise.all(testPlansData[id].map(async testPlanTests => {
                 const res = await this.runTestPlan(testPlanTests.beforeTests, testPlanTests.tests, testPlanTests.afterTests, tpOptions, testPlanName, id, branchToUse);

package/runners/runnerUtils.js

@@ -35,6 +35,7 @@ async function getSuite(options, branchToUse) {
         branch: branchToUse,
         rerunFailedByRunId: options.rerunFailedByRunId,
         testConfigIds: options.testConfigIds,
+        intersections: options.intersections,
     });
 }
 

package/services/gridService.js

@@ -235,16 +235,27 @@ function getOptionGrid(options) {
     return Promise.resolve({ host, port, path, protocol, type, user, key });
 }
 
-async function getTestPlanGridData(options) {
-    const testPlanDatas = await servicesApi.getTestPlan(options.project, options.testPlan);
-    if (testPlanDatas.length === 0) {
-        throw new ArgError(`no test plan to run ${options.testPlan}`);
-    }
-    const testPlanGrids = testPlanDatas.map(testPlanData => options.allGrids.find(grid => grid._id === testPlanData.gridId));
-    if (testPlanGrids.includes(undefined)) {
-        throw new ArgError('failed to find one of the test plan defined grid');
+async function getTestPlanGridData(options, testPlanData) {
+    const companyId = options.company.companyId;
+    try {
+        return await getGridDataByGridId(companyId, testPlanData.gridId, options.allGrids);
+    } catch (err) {
+        if (err instanceof ArgError) {
+            // this is a fallback to an old behavior that is wrong,
+            // we should remove it in the future
+            const testPlanDatas = await servicesApi.getTestPlan(options.project, options.testPlan);
+            if (testPlanDatas.length === 0) {
+                throw new ArgError(`no test plan to run ${options.testPlan}`);
+            }
+            const testPlanGrids = testPlanDatas.map(d => options.allGrids.find(grid => grid._id === d.gridId));
+            if (testPlanGrids.includes(undefined)) {
+                throw new ArgError('failed to find one of the test plan defined grid');
+            }
+            logger.warn('getTestPlanGridData used fallback and might cause test plans run on a unintended grid', { companyId });
+            return getSerializableObject(_.first(testPlanGrids));
+        }
+        throw err;
     }
-    return getSerializableObject(_.first(testPlanGrids));
 }
 
 async function getGridData(options) {
@@ -253,10 +264,10 @@ async function getGridData(options) {
         host, useLocalChromeDriver, useChromeLauncher, gridId, grid,
     } = options;
     if (useLocalChromeDriver || useChromeLauncher) {
-        return Promise.resolve({ mode: 'local' });
+        return { mode: 'local' };
     }
     if (host) {
-        return Promise.resolve(getOptionGrid(options));
+        return getOptionGrid(options);
     }
     const companyId = company.companyId;
     if (gridId) {
@@ -266,7 +277,8 @@ async function getGridData(options) {
         return getGridDataByGridName(companyId, grid, allGrids);
     }
     if (hasTestPlanFlag(options)) {
-        return getTestPlanGridData(options);
+        logger.info('skipping getting grid, as it is set on test plan', { companyId });
+        return undefined;
     }
 
     throw new GridError('Missing host or grid configuration');
@@ -300,7 +312,7 @@ async function _getGridSlot(browser, executionId, testResultId, onGridSlot, opti
     return gridInfo;
 }
 
-const handleHybridOrVendorIfNeeded = async (runnerOptions = { }, gridInfo = {}, testRunConfig = {}, lambdatestService, retryConfig = {}) => {
+const handleHybridOrVendorIfNeeded = async (runnerOptions = { }, gridInfo = {}, testRunConfig = {}, lambdatestService = {}, retryConfig = {}) => {
     const { company = {} } = runnerOptions;
     const companyId = company.companyId;
     const { gridId, type } = gridInfo;
@@ -337,6 +349,7 @@ module.exports = {
     getGridSlot,
     releaseGridSlot,
     getGridData,
+    getTestPlanGridData,
     addItemToGridCache,
     keepAlive: {
         start: startKeepAlive,

package/services/gridService.test.js

@@ -150,21 +150,14 @@ describe('gridService', () => {
                 .to.eventually.be.rejectedWith('Failed to find grid name: gridName');
         });
 
-        it('should get grid by id when passing test plan id', async () => {
-            const grid = await gridService.getGridData({ testPlan: 'testPlan', company: { companyId: 'companyId' }, allGrids: [{ _id: 'gridId', type: 'gridId' }] });
-            expect(grid).to.shallowDeepEqual({ type: 'gridId', gridId: 'gridId' });
-            sinon.assert.calledOnce(getTestPlanStub);
-        });
-
-        it('should handle grid not found error when passing test plan', async () => {
-            await expect(gridService.getGridData({ testPlan: 'testPlan', company: { companyId: 'companyId' }, allGrids: [] }))
-                .to.eventually.be.rejectedWith('failed to find one of the test plan defined grid');
+        it('should not assign a grid when using a test plan', async () => {
+            const grid = await gridService.getGridData({ testPlan: ['testPlan'], company: { companyId: 'companyId' }, allGrids: [{ _id: 'gridId', type: 'gridId' }] });
+            expect(grid).to.be.undefined;
         });
 
-        it('should handle empty test plan list', async () => {
-            getTestPlanStub.resolves([]);
-            await expect(gridService.getGridData({ testPlan: 'testPlan', company: { companyId: 'companyId' } }))
-                .to.eventually.be.rejectedWith('no test plan to run testPlan');
+        it('should not assign a grid when using a test plan id', async () => {
+            const grid = await gridService.getGridData({ testPlanIds: ['testPlan'], company: { companyId: 'companyId' }, allGrids: [{ _id: 'gridId', type: 'gridId' }] });
+            expect(grid).to.be.undefined;
         });
 
         it('should throw when no grid selected', async () => {

package/testRunStatus.js

@@ -397,6 +397,7 @@ RunStatus.prototype.executionStart = function (executionId, projectId, startTime
                         remoteRunId: options.remoteRunId,
                         localRunUserId: options.user,
                         isLocalRun,
+                        intersections: options.intersections,
                     };
                     const ret = servicesApi.reportExecutionStarted(data);
                     this.executionStartedPromise = ret;