@testim/testim-cli 3.253.0 → 3.255.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/OverrideTestDataBuilder.js +1 -1
- package/agent/routers/cliJsCode/index.js +4 -4
- package/agent/routers/cliJsCode/router.js +46 -42
- package/agent/routers/cliJsCode/service.js +18 -13
- package/agent/routers/codim/router.js +14 -17
- package/agent/routers/codim/router.test.js +19 -21
- package/agent/routers/codim/service.js +16 -16
- package/agent/routers/general/index.js +4 -8
- package/agent/routers/hybrid/registerRoutes.js +18 -18
- package/agent/routers/index.js +7 -7
- package/agent/routers/playground/router.js +11 -10
- package/agent/routers/playground/service.js +22 -23
- package/agent/routers/standalone-browser/registerRoutes.js +10 -10
- package/cdpTestRunner.js +4 -3
- package/chromiumInstaller.js +4 -5
- package/cli/onExit.js +2 -2
- package/cli.js +11 -10
- package/cliAgentMode.js +8 -8
- package/codim/codim-cli.js +20 -17
- package/codim/hybrid-utils.js +1 -1
- package/codim/measure-perf.js +9 -6
- package/codim/template.js/tests/examples/01-simple-text-validation.test.js +6 -6
- package/codim/template.js/tests/examples/02-using-locators.test.js +13 -15
- package/codim/template.js/tests/examples/03-using-hooks.test.js +17 -19
- package/codim/template.js/tests/examples/04-skip-and-only.test.js +16 -17
- package/codim/template.js/tests/examples/05-multiple-windows.test.js +16 -17
- package/codim/template.js/webpack.config.js +1 -1
- package/codim/template.ts/webpack.config.js +3 -3
- package/commons/AbortError.js +4 -4
- package/commons/detectDebugger.js +4 -2
- package/commons/featureFlags.js +8 -0
- package/commons/httpRequest.js +5 -1
- package/commons/httpRequestCounters.js +21 -10
- package/commons/lazyRequire.js +14 -12
- package/commons/logger.js +4 -4
- package/commons/performance-logger.js +14 -8
- package/commons/preloadTests.js +2 -2
- package/commons/prepareRunner.js +4 -2
- package/commons/prepareRunnerAndTestimStartUtils.js +40 -42
- package/commons/runnerFileCache.js +1 -1
- package/commons/socket/baseSocketServiceSocketIO.js +32 -34
- package/commons/socket/realDataService.js +6 -5
- package/commons/socket/realDataServiceSocketIO.js +4 -4
- package/commons/socket/remoteStepService.js +4 -3
- package/commons/socket/remoteStepServiceSocketIO.js +11 -12
- package/commons/socket/socketService.js +50 -52
- package/commons/socket/testResultServiceSocketIO.js +11 -11
- package/commons/testimDesiredCapabilitiesBuilder.js +3 -2
- package/commons/testimNgrok.js +2 -2
- package/commons/testimNgrok.test.js +1 -1
- package/commons/testimServicesApi.js +27 -20
- package/commons/testimTunnel.test.js +2 -1
- package/commons/xhr2.js +97 -100
- package/coverage/SummaryToObjectReport.js +0 -1
- package/coverage/jsCoverage.js +12 -10
- package/errors.js +5 -0
- package/fixLocalBuild.js +2 -0
- package/inputFileUtils.js +11 -9
- package/npm-shrinkwrap.json +2286 -1284
- package/package.json +9 -8
- package/player/appiumTestPlayer.js +1 -1
- package/player/chromeLauncherTestPlayer.js +0 -1
- package/player/services/tabService.js +15 -1
- package/player/services/tabServiceMock.js +166 -0
- package/player/stepActions/locateStepAction.js +2 -0
- package/player/stepActions/navigationStepAction.js +11 -10
- package/player/stepActions/sleepStepAction.js +4 -5
- package/player/stepActions/textStepAction.js +4 -11
- package/player/utils/imageCaptureUtils.js +81 -120
- package/player/utils/windowUtils.js +4 -3
- package/player/webdriver.js +26 -23
- package/processHandler.js +3 -3
- package/processHandler.test.js +1 -1
- package/reports/consoleReporter.js +3 -2
- package/reports/junitReporter.js +7 -9
- package/reports/reporter.js +34 -39
- package/runOptions.d.ts +260 -0
- package/runOptions.js +59 -44
- package/runner.js +14 -0
- package/runners/ParallelWorkerManager.js +9 -10
- package/runners/TestPlanRunner.js +142 -78
- package/runners/buildCodeTests.js +38 -37
- package/runners/runnerUtils.js +3 -3
- package/services/gridService.js +36 -40
- package/services/lambdatestService.js +3 -5
- package/stepPlayers/cliJsStepPlayback.js +22 -17
- package/testRunHandler.js +8 -0
- package/testRunStatus.js +9 -6
- package/utils/argsUtils.js +86 -0
- package/utils/argsUtils.test.js +32 -0
- package/utils/fsUtils.js +154 -0
- package/{utils.js → utils/index.js} +19 -262
- package/utils/promiseUtils.js +89 -0
- package/utils/stringUtils.js +98 -0
- package/utils/stringUtils.test.js +22 -0
- package/utils/timeUtils.js +25 -0
- package/utils/utils.test.js +27 -0
- package/workers/BaseWorker.js +16 -14
- package/workers/WorkerAppium.js +1 -1
- package/workers/WorkerExtension.js +6 -7
- package/workers/WorkerExtensionSingleBrowser.js +4 -4
- package/workers/WorkerSelenium.js +5 -2
- package/utils.test.js +0 -68
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const Bluebird = require('bluebird');
|
|
4
3
|
const _ = require('lodash');
|
|
5
4
|
const constants = require('../commons/constants');
|
|
6
5
|
|
|
@@ -26,100 +25,131 @@ const logger = Logger.getLogger('test-plan-runner');
|
|
|
26
25
|
const TDK_CHILD_RESULTS_TIMEOUT = 1000 * 60 * 5;
|
|
27
26
|
|
|
28
27
|
class TestPlanRunner {
|
|
28
|
+
/**
|
|
29
|
+
* @param {string=} customExtensionLocalLocation
|
|
30
|
+
*/
|
|
29
31
|
constructor(customExtensionLocalLocation) {
|
|
30
32
|
this.workerManager = new ParallelWorkerManager(customExtensionLocalLocation);
|
|
31
33
|
this.startTime = Date.now();
|
|
32
34
|
}
|
|
33
|
-
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @private
|
|
38
|
+
* @param {any[]} beforeTests
|
|
39
|
+
* @param {any[]} tests
|
|
40
|
+
* @param {any[]} afterTests
|
|
41
|
+
* @param {string} branchToUse
|
|
42
|
+
* @param {*} tpOptions
|
|
43
|
+
* @param {string} executionId
|
|
44
|
+
* @param {string} executionName
|
|
45
|
+
* @param {TestRunStatus} testStatus
|
|
46
|
+
*/
|
|
47
|
+
async runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, executionName, testStatus) {
|
|
34
48
|
const executionResults = {};
|
|
35
49
|
const authData = testimCustomToken.getTokenV3UserData();
|
|
36
50
|
|
|
37
|
-
const runBeforeTests = () => {
|
|
51
|
+
const runBeforeTests = async () => {
|
|
38
52
|
const workerCount = tpOptions.beforeParallel || 1;
|
|
39
53
|
const stopOnError = true;
|
|
40
|
-
|
|
41
|
-
|
|
54
|
+
const beforeTestsResults = await this.workerManager.runTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
55
|
+
Object.assign(executionResults, beforeTestsResults);
|
|
42
56
|
};
|
|
43
57
|
|
|
44
|
-
const runTestPlanTests = () => {
|
|
58
|
+
const runTestPlanTests = async () => {
|
|
45
59
|
const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
|
|
46
60
|
const stopOnError = false;
|
|
47
61
|
perf.log('right before this.workerManager.runTests');
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
62
|
+
const testsResults = await this.workerManager.runTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
63
|
+
perf.log('right after this.workerManager.runTests');
|
|
64
|
+
Object.assign(executionResults, testsResults);
|
|
51
65
|
};
|
|
52
66
|
|
|
53
|
-
const runAfterTests = () => {
|
|
67
|
+
const runAfterTests = async () => {
|
|
54
68
|
const workerCount = tpOptions.afterParallel || 1;
|
|
55
69
|
const stopOnError = false;
|
|
56
|
-
|
|
57
|
-
|
|
70
|
+
const afterTestsResults = await this.workerManager.runTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
71
|
+
Object.assign(executionResults, afterTestsResults);
|
|
58
72
|
};
|
|
59
73
|
|
|
60
|
-
function catchBeforeTestsFailed() {
|
|
61
|
-
return testStatus.markAllQueuedTests(executionId, constants.runnerTestStatus.ABORTED, 'aborted', false);
|
|
62
|
-
}
|
|
63
|
-
|
|
64
74
|
const sessionType = utils.getSessionType(tpOptions);
|
|
65
75
|
analyticsService.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
|
|
66
76
|
perf.log('right before runBeforeTests');
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
.
|
|
70
|
-
|
|
71
|
-
.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
77
|
+
try {
|
|
78
|
+
await runBeforeTests();
|
|
79
|
+
perf.log('right before runTestPlanTests');
|
|
80
|
+
await runTestPlanTests();
|
|
81
|
+
perf.log('right after runTestPlanTests');
|
|
82
|
+
await runAfterTests();
|
|
83
|
+
return executionResults;
|
|
84
|
+
} catch (err) {
|
|
85
|
+
logger.error('error running test plan', { err });
|
|
86
|
+
if (err instanceof StopRunOnError) {
|
|
87
|
+
return testStatus.markAllQueuedTests(executionId, constants.runnerTestStatus.ABORTED, 'aborted', false);
|
|
88
|
+
}
|
|
89
|
+
throw err;
|
|
90
|
+
} finally {
|
|
91
|
+
await handlePixelValidationBatches();
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function handlePixelValidationBatches() {
|
|
95
|
+
if (tpOptions.lightweightMode?.disablePixelValidation) {
|
|
96
|
+
return;
|
|
97
|
+
}
|
|
98
|
+
// When sessionPlayer is available, use it - as it only attempts to close batches that exist.
|
|
99
|
+
if (tpOptions.mode === constants.CLI_MODE.SELENIUM) {
|
|
100
|
+
const { EyeSdkBuilder } = require('../commons/getSessionPlayerRequire');
|
|
101
|
+
await EyeSdkBuilder.closeBatch(executionId);
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
/** @type {Awaited<ReturnType<typeof testimServicesApi['getApplitoolsIntegrationData']>>} */
|
|
105
|
+
let applitoolsIntegrationData;
|
|
106
|
+
try {
|
|
107
|
+
if (!tpOptions.company || !tpOptions.company.activePlan || !tpOptions.company.activePlan.premiumFeatures || !tpOptions.company.activePlan.premiumFeatures.applitools) {
|
|
82
108
|
return;
|
|
83
109
|
}
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
const { EyeSdkBuilder } = require('../commons/getSessionPlayerRequire');
|
|
87
|
-
await EyeSdkBuilder.closeBatch(executionId);
|
|
110
|
+
applitoolsIntegrationData = await testimServicesApi.getApplitoolsIntegrationData(tpOptions.project);
|
|
111
|
+
if (_.isEmpty(applitoolsIntegrationData) || !executionId) {
|
|
88
112
|
return;
|
|
89
113
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (_.isEmpty(applitoolsIntegrationData) || !executionId) {
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
const { runKey: apiKey, url: serverUrl } = applitoolsIntegrationData;
|
|
101
|
-
const tmpSDK = require('@applitools/eyes-sdk-core').makeSDK({ name: 'Testim.io', version: '4.0.0', spec: {} });
|
|
102
|
-
await tmpSDK.closeBatches({ batchIds: [executionId], serverUrl, apiKey });
|
|
103
|
-
} catch (err) {
|
|
104
|
-
// If a batch with this name did not exist, do not log an error.
|
|
105
|
-
if (err.message && (err.message.startsWith('Request failed with status code 404') || err.message.startsWith('no batchIds were set'))) {
|
|
106
|
-
return;
|
|
107
|
-
}
|
|
108
|
-
logger.error('Failed closing batch in extension mode', { err, projectId: tpOptions.project, applitoolsIntegrationData, batchIds: [executionId] });
|
|
114
|
+
const { runKey: apiKey, url: serverUrl } = applitoolsIntegrationData;
|
|
115
|
+
const tmpSDK = require('@applitools/eyes-sdk-core').makeSDK({ name: 'Testim.io', version: '4.0.0', spec: {} });
|
|
116
|
+
await tmpSDK.closeBatches({ batchIds: [executionId], serverUrl, apiKey });
|
|
117
|
+
} catch (err) {
|
|
118
|
+
// If a batch with this name did not exist, do not log an error.
|
|
119
|
+
if (err.message && (err.message.startsWith('Request failed with status code 404') || err.message.startsWith('no batchIds were set'))) {
|
|
120
|
+
return;
|
|
109
121
|
}
|
|
110
|
-
|
|
122
|
+
logger.error('Failed closing batch in extension mode', { err, projectId: tpOptions.project, applitoolsIntegrationData, batchIds: [executionId] });
|
|
123
|
+
}
|
|
124
|
+
}
|
|
111
125
|
}
|
|
112
126
|
|
|
127
|
+
/**
|
|
128
|
+
* @private
|
|
129
|
+
* @param {string} projectId
|
|
130
|
+
*/
|
|
113
131
|
async initRealDataService(projectId) {
|
|
114
132
|
const realDataService = new RealDataService();
|
|
115
133
|
await realDataService.init(projectId);
|
|
116
134
|
return realDataService;
|
|
117
135
|
}
|
|
118
136
|
|
|
137
|
+
/**
|
|
138
|
+
* @private
|
|
139
|
+
* @param {RealDataService} realDataService
|
|
140
|
+
* @param {string} projectId
|
|
141
|
+
* @param {string} runId
|
|
142
|
+
* @param {TestRunStatus} testStatus
|
|
143
|
+
*/
|
|
119
144
|
async listenToTestCreatedInFile(realDataService, projectId, runId, testStatus) {
|
|
120
145
|
const childTestResults = {};
|
|
121
146
|
realDataService.joinToTestResultsByRunId(runId, projectId);
|
|
122
|
-
|
|
147
|
+
let isPromisePending = true;
|
|
148
|
+
const promise = new Promise(_resolve => {
|
|
149
|
+
const resolve = (val) => {
|
|
150
|
+
isPromisePending = false;
|
|
151
|
+
_resolve(val);
|
|
152
|
+
};
|
|
123
153
|
realDataService.listenToTestResultsByRunId(runId, testResult => {
|
|
124
154
|
const resultId = testResult.id;
|
|
125
155
|
if (!testStatus.getTestResult(resultId)) {
|
|
@@ -151,9 +181,9 @@ class TestPlanRunner {
|
|
|
151
181
|
|
|
152
182
|
if (allParentTestResultCompleted && !allChildTestResultCompleted) {
|
|
153
183
|
// wait 10 sec to handle race condition when parent test result (file) finished before child test result
|
|
154
|
-
return
|
|
184
|
+
return utils.delay(10000)
|
|
155
185
|
.then(() => {
|
|
156
|
-
if (
|
|
186
|
+
if (isPromisePending) {
|
|
157
187
|
// TODO(Benji) we are missing the child test results here.
|
|
158
188
|
// we are resolving here with partial data - we should consider fetching it
|
|
159
189
|
// from the server
|
|
@@ -168,6 +198,28 @@ class TestPlanRunner {
|
|
|
168
198
|
return await promise;
|
|
169
199
|
}
|
|
170
200
|
|
|
201
|
+
/**
|
|
202
|
+
* @private
|
|
203
|
+
* @param {string} projectId
|
|
204
|
+
* @param {string} executionId
|
|
205
|
+
* @param {TestRunStatus} testStatus
|
|
206
|
+
*/
|
|
207
|
+
async initRealDataServiceAndListenToTestsCreatedInFile(projectId, executionId, testStatus) {
|
|
208
|
+
const realDataService = await this.initRealDataService(projectId);
|
|
209
|
+
return this.listenToTestCreatedInFile(realDataService, projectId, executionId, testStatus);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* @private
|
|
214
|
+
* @param {any[]} beforeTests
|
|
215
|
+
* @param {any[]} tests
|
|
216
|
+
* @param {any[]} afterTests
|
|
217
|
+
* @param {import('../runOptions').RunnerOptions} tpOptions
|
|
218
|
+
* @param {string} testPlanName
|
|
219
|
+
* @param {string | null} testPlanId
|
|
220
|
+
* @param {string} branch
|
|
221
|
+
* @param {boolean=} isAnonymous
|
|
222
|
+
*/
|
|
171
223
|
async runTestPlan(beforeTests, tests, afterTests, tpOptions, testPlanName, testPlanId, branch, isAnonymous) {
|
|
172
224
|
const executionId = guid();
|
|
173
225
|
const projectId = tpOptions.project;
|
|
@@ -176,37 +228,36 @@ class TestPlanRunner {
|
|
|
176
228
|
afterTests.forEach(test => { test.isAfterTestPlan = true; });
|
|
177
229
|
const testStatus = new TestRunStatus(_.concat(beforeTests, tests, afterTests), tpOptions, testPlanId, branch);
|
|
178
230
|
|
|
179
|
-
const configs = _(_.concat(beforeTests, tests, afterTests))
|
|
231
|
+
const configs = _.chain(_.concat(beforeTests, tests, afterTests))
|
|
232
|
+
.map(test => test.overrideTestConfig?.name || '')
|
|
233
|
+
.uniq()
|
|
234
|
+
.compact()
|
|
180
235
|
.value();
|
|
181
|
-
const configName = configs
|
|
236
|
+
const configName = configs?.length === 1 ? configs[0] : null;
|
|
182
237
|
|
|
183
238
|
const isCodeMode = tpOptions.files.length > 0;
|
|
184
|
-
const testNames = tpOptions.lightweightMode
|
|
239
|
+
const testNames = tpOptions.lightweightMode?.onlyTestIdsNoSuite ? [] : _.concat(beforeTests, tests, afterTests).map(test => test.name);
|
|
185
240
|
|
|
186
|
-
const testListInfoPromise = tpOptions.lightweightMode
|
|
241
|
+
const testListInfoPromise = tpOptions.lightweightMode?.onlyTestIdsNoSuite ?
|
|
187
242
|
{ beforeTests, tests, afterTests } :
|
|
188
243
|
testStatus.executionStart(executionId, projectId, this.startTime, testPlanName, testNames);
|
|
189
244
|
let childTestResults;
|
|
190
245
|
if (isCodeMode) {
|
|
191
|
-
childTestResults =
|
|
192
|
-
const realDataService = await this.initRealDataService(projectId);
|
|
193
|
-
return this.listenToTestCreatedInFile(realDataService, projectId, executionId, testStatus);
|
|
194
|
-
});
|
|
246
|
+
childTestResults = this.initRealDataServiceAndListenToTestsCreatedInFile(projectId, executionId, testStatus);
|
|
195
247
|
}
|
|
196
248
|
perf.log('before testListInfoPromise');
|
|
197
249
|
const testListInfo = await testListInfoPromise;
|
|
198
|
-
if (!
|
|
250
|
+
if (!tpOptions.lightweightMode?.onlyTestIdsNoSuite) {
|
|
199
251
|
reporter.onTestPlanStarted(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, testPlanName, executionId, isAnonymous, configName, isCodeMode);
|
|
200
252
|
}
|
|
201
253
|
|
|
202
254
|
perf.log('before runTestAllPhases');
|
|
203
255
|
const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testPlanName || 'All Tests', testStatus);
|
|
204
|
-
const childResults = await
|
|
205
|
-
.timeout(TDK_CHILD_RESULTS_TIMEOUT)
|
|
256
|
+
const childResults = await utils.promiseTimeout(childTestResults, TDK_CHILD_RESULTS_TIMEOUT)
|
|
206
257
|
.catch(async () => {
|
|
207
|
-
logger.warn('timed out waiting for child
|
|
258
|
+
logger.warn('timed out waiting for child results on websocket. using REST fallback', { projectId, executionId });
|
|
208
259
|
const testResults = await testimServicesApi.getRealData(projectId, 'testResult', `runId=${executionId}&sort=runOrder`);
|
|
209
|
-
return _.chain((testResults
|
|
260
|
+
return _.chain((testResults?.data?.docs) || [])
|
|
210
261
|
.groupBy('parentResultId')
|
|
211
262
|
.omit('undefined')
|
|
212
263
|
.values()
|
|
@@ -219,6 +270,11 @@ class TestPlanRunner {
|
|
|
219
270
|
return { results, executionId, testPlanName, configName, childTestResults: childResults };
|
|
220
271
|
}
|
|
221
272
|
|
|
273
|
+
/**
|
|
274
|
+
* @private
|
|
275
|
+
* @param {import('../runOptions').RunnerOptions} options
|
|
276
|
+
* @param {string} branchToUse
|
|
277
|
+
*/
|
|
222
278
|
async runTestPlans(options, branchToUse) {
|
|
223
279
|
logger.info('start to run test plan', {
|
|
224
280
|
options: Object.assign({}, options, { token: undefined, userParamsData: undefined }),
|
|
@@ -246,7 +302,7 @@ class TestPlanRunner {
|
|
|
246
302
|
throw new ArgError(`no test to run in test plan ${options.testPlan}`);
|
|
247
303
|
}
|
|
248
304
|
await validateConfig(options, flattenTestListData(testPlansData));
|
|
249
|
-
return await
|
|
305
|
+
return await utils.promiseMap(testPlans, async testPlan => {
|
|
250
306
|
const id = testPlan.testPlanId;
|
|
251
307
|
testPlansResults[id] = {};
|
|
252
308
|
|
|
@@ -265,7 +321,7 @@ class TestPlanRunner {
|
|
|
265
321
|
tpOptions.gridData = await gridService.getTestPlanGridData(options, testPlan);
|
|
266
322
|
|
|
267
323
|
const testPlanName = tpOptions.overrideExecutionName || testPlan.name;
|
|
268
|
-
return await
|
|
324
|
+
return await utils.promiseMap(testPlansData[id], async testPlanTests => {
|
|
269
325
|
const res = await this.runTestPlan(testPlanTests.beforeTests, testPlanTests.tests, testPlanTests.afterTests, tpOptions, testPlanName, id, branchToUse);
|
|
270
326
|
const isCodeMode = options.files.length > 0;
|
|
271
327
|
reporter.onTestPlanFinished(res.results, testPlan.name, this.startTime, res.executionId, false, isCodeMode, res.childTestResults);
|
|
@@ -281,10 +337,15 @@ class TestPlanRunner {
|
|
|
281
337
|
const executionId = success ? executions[0].executionId : executions.find(exec => !exec.success).executionId;
|
|
282
338
|
await testimServicesApi.saveTestPlanResult(projectId, id, { success, executions, executionId });
|
|
283
339
|
return res;
|
|
284
|
-
})
|
|
285
|
-
})
|
|
340
|
+
});
|
|
341
|
+
});
|
|
286
342
|
}
|
|
287
343
|
|
|
344
|
+
/**
|
|
345
|
+
* @private
|
|
346
|
+
* @param {import('../runOptions').RunnerOptions} options
|
|
347
|
+
* @param {string} branchToUse
|
|
348
|
+
*/
|
|
288
349
|
async runAnonymousTestPlan(options, branchToUse) {
|
|
289
350
|
logger.info('start to run anonymous', {
|
|
290
351
|
options: Object.assign({}, options, { token: undefined }),
|
|
@@ -295,7 +356,7 @@ class TestPlanRunner {
|
|
|
295
356
|
const suiteResult = await getSuite(options, branchToUse);
|
|
296
357
|
perf.log('after getSuite');
|
|
297
358
|
|
|
298
|
-
if (!suiteResult.tests[0]
|
|
359
|
+
if (!suiteResult.tests[0]?.length) {
|
|
299
360
|
if (options.rerunFailedByRunId) {
|
|
300
361
|
throw new ArgError('No failed tests found in the provided run');
|
|
301
362
|
}
|
|
@@ -317,7 +378,7 @@ class TestPlanRunner {
|
|
|
317
378
|
const testPlanName = options.overrideExecutionName || suiteResult.runName || _.concat(options.label, options.name, options.suites).join(' ');
|
|
318
379
|
const isAnonymous = true;
|
|
319
380
|
perf.log('Right before validateConfig + runAnonymousTestPlan tests map');
|
|
320
|
-
return await
|
|
381
|
+
return await utils.promiseMap(suiteResult.tests, async suiteTests => { // array of results per execution
|
|
321
382
|
//override result id for remote run mode and run only the first test data
|
|
322
383
|
if (options.resultId) {
|
|
323
384
|
const firstTest = _.first(suiteTests);
|
|
@@ -331,9 +392,12 @@ class TestPlanRunner {
|
|
|
331
392
|
const isCodeMode = options.files.length > 0;
|
|
332
393
|
await reporter.onTestPlanFinished(res.results, testPlanName, this.startTime, res.executionId, isAnonymous, isCodeMode, res.childTestResults);
|
|
333
394
|
return res;
|
|
334
|
-
})
|
|
395
|
+
});
|
|
335
396
|
}
|
|
336
397
|
|
|
398
|
+
/**
|
|
399
|
+
* @param {import('../runOptions').RunnerOptions} options
|
|
400
|
+
*/
|
|
337
401
|
async run(options) {
|
|
338
402
|
const branchToUse = branchService.getCurrentBranch();
|
|
339
403
|
let results = [];
|
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
'use strict';
|
|
2
2
|
|
|
3
|
-
const
|
|
3
|
+
const _ = require('lodash');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const utils = require('../utils');
|
|
6
5
|
const Promise = require('bluebird');
|
|
7
|
-
const
|
|
6
|
+
const MemoryFS = require('memory-fs');
|
|
7
|
+
const utils = require('../utils');
|
|
8
8
|
const lazyRequire = require('../commons/lazyRequire');
|
|
9
9
|
const { AbortError } = require('../commons/AbortError');
|
|
10
10
|
|
|
11
|
-
const { isEqual, cloneDeep } = require('lodash');
|
|
12
|
-
|
|
13
11
|
// compiler instance we can reuse between calls
|
|
14
12
|
const state = {
|
|
15
13
|
compiler: null,
|
|
16
|
-
webpackConfig: null
|
|
14
|
+
webpackConfig: null,
|
|
17
15
|
};
|
|
18
16
|
|
|
19
17
|
exports.buildCodeTests = async function buildCodeTestsGuarded(
|
|
@@ -36,36 +34,40 @@ exports.buildCodeTests = async function buildCodeTestsGuarded(
|
|
|
36
34
|
}
|
|
37
35
|
throw e;
|
|
38
36
|
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runnerOptionsToMaybeCopyToTestEnvironment, fileSystem, optionalAbortSignal) {
|
|
37
|
+
};
|
|
42
38
|
|
|
39
|
+
async function buildCodeTests(files, webpackConfig = { mode: 'development' }, runnerOptionsToMaybeCopyToTestEnvironment = undefined, fileSystem = undefined, optionalAbortSignal = undefined) {
|
|
43
40
|
const webpack = await lazyRequire('webpack');
|
|
44
41
|
|
|
45
42
|
const suite = {};
|
|
46
|
-
const webpackConfigBeforeOurChanges = cloneDeep(webpackConfig);
|
|
43
|
+
const webpackConfigBeforeOurChanges = _.cloneDeep(webpackConfig);
|
|
47
44
|
|
|
48
45
|
webpackConfig.externals = { // define testim as an external
|
|
49
|
-
|
|
46
|
+
testim: '__testim',
|
|
50
47
|
// 'chai': '__chai'
|
|
51
48
|
};
|
|
52
49
|
webpackConfig.devtool = webpackConfig.devtool || 'inline-source-map';
|
|
53
50
|
|
|
54
51
|
webpackConfig.plugins = webpackConfig.plugins || [];
|
|
55
52
|
|
|
56
|
-
webpackConfig.plugins.push(new webpack.DefinePlugin(getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment))
|
|
57
|
-
|
|
58
|
-
'process.argv': JSON.stringify(process.argv)
|
|
53
|
+
webpackConfig.plugins.push(new webpack.DefinePlugin(getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment)), new webpack.DefinePlugin({
|
|
54
|
+
'process.argv': JSON.stringify(process.argv),
|
|
59
55
|
}));
|
|
60
56
|
files = files.map(f => path.resolve(f));
|
|
61
57
|
|
|
62
|
-
const fileHashes = files.map(
|
|
58
|
+
const fileHashes = files.map(() => utils.guid(30));
|
|
63
59
|
|
|
64
60
|
webpackConfig.optimization = { minimize: false };
|
|
65
61
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
62
|
+
const majorNodeVersion = Number(process.versions.node.split('.')[0]);
|
|
63
|
+
if (majorNodeVersion >= 17) {
|
|
64
|
+
// TODO ugly hack to be removed once using Webpack 5 - https://stackoverflow.com/a/73465262 + https://github.com/webpack/webpack/issues/14532
|
|
65
|
+
const crypto = require('crypto');
|
|
66
|
+
const originalCryptoCreateHash = crypto.createHash;
|
|
67
|
+
crypto.createHash = (algorithm, options) => originalCryptoCreateHash(algorithm === 'md4' ? 'sha256' : algorithm, options);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
webpackConfig.entry = Object.fromEntries(_.zip(files, fileHashes).map(([filename, hash]) => [hash, filename]));
|
|
69
71
|
webpackConfig.output = Object.assign({
|
|
70
72
|
devtoolModuleFilenameTemplate: (info) => `file:///${info.absoluteResourcePath}`,
|
|
71
73
|
filename: '[name].bundle.js',
|
|
@@ -76,7 +78,7 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
76
78
|
// if we are passed a filesystem, assume reuse between calls and turn on watch mode
|
|
77
79
|
if (fileSystem) {
|
|
78
80
|
// were we passed a filesystem before to compile the same thing?
|
|
79
|
-
if (isEqual(state.webpackConfig, webpackConfigBeforeOurChanges) && state.compiler) {
|
|
81
|
+
if (_.isEqual(state.webpackConfig, webpackConfigBeforeOurChanges) && state.compiler) {
|
|
80
82
|
// we already have a compiler up and running
|
|
81
83
|
compiler = state.compiler;
|
|
82
84
|
} else {
|
|
@@ -92,9 +94,9 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
92
94
|
compiler.outputFileSystem = mfs; // no need to write compiled tests to disk
|
|
93
95
|
|
|
94
96
|
// This can only reject
|
|
95
|
-
const abortSignalPromise =
|
|
97
|
+
const abortSignalPromise = utils.promiseFromCallback(cb => {
|
|
96
98
|
if (optionalAbortSignal) {
|
|
97
|
-
optionalAbortSignal.addEventListener(
|
|
99
|
+
optionalAbortSignal.addEventListener('abort', () => {
|
|
98
100
|
cb(new AbortError());
|
|
99
101
|
});
|
|
100
102
|
}
|
|
@@ -102,12 +104,12 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
102
104
|
|
|
103
105
|
// run compiler:
|
|
104
106
|
try {
|
|
105
|
-
const stats = await Promise.race([
|
|
107
|
+
const stats = await Promise.race([utils.promiseFromCallback(cb => compiler.run(cb)), abortSignalPromise]);
|
|
106
108
|
if (stats.hasErrors()) {
|
|
107
109
|
throw new Error(stats.toJson().errors);
|
|
108
110
|
}
|
|
109
111
|
} catch (e) {
|
|
110
|
-
const {ArgError} = require('../errors');
|
|
112
|
+
const { ArgError } = require('../errors');
|
|
111
113
|
|
|
112
114
|
const cantFindFile = e.message.match(/Entry module not found: Error: Can't resolve '(.*)'/);
|
|
113
115
|
if (cantFindFile && cantFindFile.length === 2) {
|
|
@@ -117,40 +119,39 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
117
119
|
throw new ArgError(`Can't find test files in: '${cantFindFile[1]}'`);
|
|
118
120
|
}
|
|
119
121
|
|
|
120
|
-
throw new ArgError(
|
|
122
|
+
throw new ArgError(`Compilation Webpack Error in tests: ${e.message}`);
|
|
121
123
|
}
|
|
122
124
|
|
|
123
|
-
const fileResults = files.map((file, i) => ({code: mfs.readFileSync(path.resolve('./dist', `${fileHashes[i]}.bundle.js`)), name: file })); // read all files
|
|
125
|
+
const fileResults = files.map((file, i) => ({ code: mfs.readFileSync(path.resolve('./dist', `${fileHashes[i]}.bundle.js`)), name: file })); // read all files
|
|
124
126
|
|
|
125
|
-
suite.tests = [fileResults.map(({code, name}) => ({
|
|
127
|
+
suite.tests = [fileResults.map(({ code, name }) => ({
|
|
126
128
|
code: code.toString(),
|
|
127
|
-
baseUrl:
|
|
129
|
+
baseUrl: '', // not supported at the moment
|
|
128
130
|
name: path.resolve(name),
|
|
129
131
|
testConfig: {},
|
|
130
132
|
testConfigId: null,
|
|
131
133
|
testId: utils.guid(),
|
|
132
134
|
resultId: utils.guid(),
|
|
133
|
-
isTestsContainer: true
|
|
135
|
+
isTestsContainer: true,
|
|
134
136
|
}))];
|
|
135
|
-
suite.runName =
|
|
137
|
+
suite.runName = `Testim Dev Kit Run ${new Date().toLocaleString()}`;
|
|
136
138
|
return suite;
|
|
137
139
|
}
|
|
138
140
|
|
|
139
141
|
// copied mostly from facebook/create-react-app/blob/8b7b819b4b9e6ba457e011e92e33266690e26957/packages/react-scripts/config/env.js
|
|
140
142
|
function getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment) {
|
|
141
|
-
|
|
142
|
-
let fromEnvironment = _.fromPairs(
|
|
143
|
+
const fromEnvironment = _.fromPairs(
|
|
143
144
|
Object.keys(process.env)
|
|
144
|
-
|
|
145
|
-
|
|
145
|
+
.filter(key => /^TDK_/i.test(key) || key === 'BASE_URL')
|
|
146
|
+
.map(key => [key, process.env[key]])
|
|
146
147
|
);
|
|
147
148
|
|
|
148
|
-
|
|
149
|
-
|
|
149
|
+
const fromConfig = {
|
|
150
|
+
BASE_URL: runnerOptionsToMaybeCopyToTestEnvironment.baseUrl,
|
|
150
151
|
};
|
|
151
152
|
|
|
152
153
|
return {
|
|
153
|
-
'process.env': stringifyValues({ ...fromConfig, ...fromEnvironment})
|
|
154
|
+
'process.env': stringifyValues({ ...fromConfig, ...fromEnvironment }),
|
|
154
155
|
};
|
|
155
156
|
}
|
|
156
157
|
function stringifyValues(object) {
|
package/runners/runnerUtils.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
const _ = require('lodash');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const utils = require('../utils
|
|
5
|
+
const utils = require('../utils');
|
|
6
6
|
const analytics = require('../commons/testimAnalytics');
|
|
7
7
|
const { ArgError } = require('../errors');
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
async function getSuite(options, branchToUse) {
|
|
11
|
-
if (options.lightweightMode
|
|
11
|
+
if (options.lightweightMode?.onlyTestIdsNoSuite && options.testId) {
|
|
12
12
|
return { tests: [options.testId.map(testId => ({ testId, testConfig: { }, resultId: utils.guid() }))] };
|
|
13
13
|
}
|
|
14
14
|
// local code test
|
|
@@ -17,7 +17,7 @@ async function getSuite(options, branchToUse) {
|
|
|
17
17
|
let webpackConfig = {};
|
|
18
18
|
if (options.webpackConfig) {
|
|
19
19
|
const webpackConfigPath = path.join(process.cwd(), options.webpackConfig);
|
|
20
|
-
webpackConfig = require(webpackConfigPath);
|
|
20
|
+
webpackConfig = require(webpackConfigPath); // eslint-disable-line import/no-dynamic-require
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
return buildCodeTests(options.files, webpackConfig, { baseUrl: options.baseUrl });
|