@testim/testim-cli 3.252.0 → 3.254.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/OverrideTestDataBuilder.js +1 -1
- package/agent/routers/cliJsCode/index.js +4 -4
- package/agent/routers/cliJsCode/router.js +46 -42
- package/agent/routers/cliJsCode/service.js +18 -13
- package/agent/routers/codim/router.js +14 -17
- package/agent/routers/codim/router.test.js +15 -14
- package/agent/routers/codim/service.js +1 -1
- package/agent/routers/general/index.js +4 -8
- package/agent/routers/hybrid/registerRoutes.js +18 -18
- package/agent/routers/index.js +9 -8
- package/agent/routers/playground/router.js +12 -11
- package/agent/routers/playground/service.js +19 -18
- package/agent/routers/standalone-browser/registerRoutes.js +10 -10
- package/cdpTestRunner.js +4 -3
- package/chromiumInstaller.js +4 -5
- package/cli/onExit.js +2 -2
- package/cli.js +7 -6
- package/cliAgentMode.js +4 -5
- package/codim/codim-cli.js +11 -10
- package/codim/hybrid-utils.js +1 -1
- package/codim/measure-perf.js +9 -6
- package/codim/template.js/tests/examples/01-simple-text-validation.test.js +6 -6
- package/codim/template.js/tests/examples/02-using-locators.test.js +13 -15
- package/codim/template.js/tests/examples/03-using-hooks.test.js +17 -19
- package/codim/template.js/tests/examples/04-skip-and-only.test.js +16 -17
- package/codim/template.js/tests/examples/05-multiple-windows.test.js +16 -17
- package/codim/template.js/webpack.config.js +1 -1
- package/codim/template.ts/webpack.config.js +3 -3
- package/commons/AbortError.js +4 -4
- package/commons/chrome-launcher.js +6 -6
- package/commons/constants.js +2 -0
- package/commons/detectDebugger.js +4 -2
- package/commons/getSessionPlayerRequire.js +2 -20
- package/commons/initializeUserWithAuth.js +2 -2
- package/commons/lazyRequire.js +10 -9
- package/commons/logger.js +4 -4
- package/commons/performance-logger.js +14 -8
- package/commons/prepareRunnerAndTestimStartUtils.js +6 -7
- package/commons/socket/baseSocketServiceSocketIO.js +32 -34
- package/commons/socket/realDataService.js +6 -5
- package/commons/socket/realDataServiceSocketIO.js +4 -4
- package/commons/socket/remoteStepService.js +4 -3
- package/commons/socket/remoteStepServiceSocketIO.js +11 -12
- package/commons/socket/socketService.js +50 -52
- package/commons/socket/testResultServiceSocketIO.js +11 -11
- package/commons/testimDesiredCapabilitiesBuilder.js +44 -0
- package/commons/testimNgrok.js +2 -2
- package/commons/testimNgrok.test.js +1 -1
- package/commons/testimServicesApi.js +37 -21
- package/commons/xhr2.js +97 -100
- package/credentialsManager.js +17 -20
- package/errors.js +5 -0
- package/fixLocalBuild.js +2 -0
- package/npm-shrinkwrap.json +4455 -1576
- package/package.json +9 -7
- package/player/WebdriverioWebDriverApi.js +7 -2
- package/player/appiumTestPlayer.js +102 -0
- package/player/chromeLauncherTestPlayer.js +0 -1
- package/player/seleniumTestPlayer.js +3 -2
- package/player/services/frameLocator.js +2 -1
- package/player/services/mobileFrameLocatorMock.js +32 -0
- package/player/services/playbackTimeoutCalculator.js +1 -0
- package/player/services/portSelector.js +10 -8
- package/player/services/tabService.js +29 -0
- package/player/services/tabServiceMock.js +166 -0
- package/player/stepActions/navigationStepAction.js +11 -10
- package/player/stepActions/sleepStepAction.js +4 -5
- package/player/stepActions/stepAction.js +15 -1
- package/player/stepActions/textStepAction.js +4 -11
- package/player/utils/stepActionUtils.js +4 -2
- package/player/utils/windowUtils.js +139 -125
- package/player/webdriver.js +40 -26
- package/processHandler.js +3 -3
- package/processHandler.test.js +1 -1
- package/reports/consoleReporter.js +3 -2
- package/reports/debugReporter.js +41 -39
- package/reports/jsonReporter.js +53 -50
- package/reports/junitReporter.js +1 -2
- package/reports/reporter.js +135 -136
- package/runOptions.js +8 -7
- package/runner.js +13 -0
- package/runners/ParallelWorkerManager.js +2 -0
- package/runners/TestPlanRunner.js +142 -74
- package/runners/buildCodeTests.js +38 -37
- package/runners/runnerUtils.js +3 -3
- package/services/lambdatestService.js +3 -5
- package/stepPlayers/cliJsStepPlayback.js +22 -17
- package/testRunHandler.js +8 -0
- package/testRunStatus.js +458 -460
- package/{utils.js → utils/index.js} +25 -117
- package/utils/promiseUtils.js +78 -0
- package/utils/stringUtils.js +96 -0
- package/{utils.test.js → utils/utils.test.js} +2 -2
- package/workers/BaseWorker.js +29 -20
- package/workers/WorkerAppium.js +123 -0
- package/workers/WorkerExtensionSingleBrowser.js +4 -4
- package/workers/WorkerSelenium.js +5 -2
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
const Bluebird = require('bluebird');
|
|
4
3
|
const _ = require('lodash');
|
|
5
4
|
const constants = require('../commons/constants');
|
|
6
5
|
|
|
@@ -26,35 +25,50 @@ const logger = Logger.getLogger('test-plan-runner');
|
|
|
26
25
|
const TDK_CHILD_RESULTS_TIMEOUT = 1000 * 60 * 5;
|
|
27
26
|
|
|
28
27
|
class TestPlanRunner {
|
|
28
|
+
/**
|
|
29
|
+
* @param {string=} customExtensionLocalLocation
|
|
30
|
+
*/
|
|
29
31
|
constructor(customExtensionLocalLocation) {
|
|
30
32
|
this.workerManager = new ParallelWorkerManager(customExtensionLocalLocation);
|
|
31
33
|
this.startTime = Date.now();
|
|
32
34
|
}
|
|
33
|
-
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @private
|
|
38
|
+
* @param {any[]} beforeTests
|
|
39
|
+
* @param {any[]} tests
|
|
40
|
+
* @param {any[]} afterTests
|
|
41
|
+
* @param {string} branchToUse
|
|
42
|
+
* @param {*} tpOptions
|
|
43
|
+
* @param {string} executionId
|
|
44
|
+
* @param {string} executionName
|
|
45
|
+
* @param {TestRunStatus} testStatus
|
|
46
|
+
*/
|
|
47
|
+
async runTestAllPhases(beforeTests, tests, afterTests, branchToUse, tpOptions, executionId, executionName, testStatus) {
|
|
34
48
|
const executionResults = {};
|
|
35
49
|
const authData = testimCustomToken.getTokenV3UserData();
|
|
36
50
|
|
|
37
|
-
const runBeforeTests = () => {
|
|
51
|
+
const runBeforeTests = async () => {
|
|
38
52
|
const workerCount = tpOptions.beforeParallel || 1;
|
|
39
53
|
const stopOnError = true;
|
|
40
|
-
|
|
41
|
-
|
|
54
|
+
const beforeTestsResults = await this.workerManager.runTests(beforeTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
55
|
+
Object.assign(executionResults, beforeTestsResults);
|
|
42
56
|
};
|
|
43
57
|
|
|
44
|
-
const runTestPlanTests = () => {
|
|
58
|
+
const runTestPlanTests = async () => {
|
|
45
59
|
const workerCount = config.TESTIM_CONCURRENT_WORKER_COUNT || tpOptions.parallel;
|
|
46
60
|
const stopOnError = false;
|
|
47
61
|
perf.log('right before this.workerManager.runTests');
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
62
|
+
const testsResults = await this.workerManager.runTests(tests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
63
|
+
perf.log('right after this.workerManager.runTests');
|
|
64
|
+
Object.assign(executionResults, testsResults);
|
|
51
65
|
};
|
|
52
66
|
|
|
53
|
-
const runAfterTests = () => {
|
|
67
|
+
const runAfterTests = async () => {
|
|
54
68
|
const workerCount = tpOptions.afterParallel || 1;
|
|
55
69
|
const stopOnError = false;
|
|
56
|
-
|
|
57
|
-
|
|
70
|
+
const afterTestsResults = await this.workerManager.runTests(afterTests, testStatus, executionId, executionName, tpOptions, branchToUse, authData, workerCount, stopOnError);
|
|
71
|
+
Object.assign(executionResults, afterTestsResults);
|
|
58
72
|
};
|
|
59
73
|
|
|
60
74
|
function catchBeforeTestsFailed() {
|
|
@@ -64,62 +78,82 @@ class TestPlanRunner {
|
|
|
64
78
|
const sessionType = utils.getSessionType(tpOptions);
|
|
65
79
|
analyticsService.analyticsExecsStart({ authData, executionId, projectId: tpOptions.project, sessionType });
|
|
66
80
|
perf.log('right before runBeforeTests');
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
.
|
|
70
|
-
|
|
71
|
-
.
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
81
|
+
try {
|
|
82
|
+
await runBeforeTests();
|
|
83
|
+
perf.log('right before runTestPlanTests');
|
|
84
|
+
await runTestPlanTests();
|
|
85
|
+
perf.log('right after runTestPlanTests');
|
|
86
|
+
await runAfterTests();
|
|
87
|
+
return executionResults;
|
|
88
|
+
} catch (err) {
|
|
89
|
+
logger.error('error running test plan', { err });
|
|
90
|
+
if (err instanceof StopRunOnError) {
|
|
91
|
+
return catchBeforeTestsFailed();
|
|
92
|
+
}
|
|
93
|
+
throw err;
|
|
94
|
+
} finally {
|
|
95
|
+
await handlePixelValidationBatches();
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
async function handlePixelValidationBatches() {
|
|
99
|
+
if (tpOptions.lightweightMode?.disablePixelValidation) {
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
// When sessionPlayer is available, use it - as it only attempts to close batches that exist.
|
|
103
|
+
if (tpOptions.mode === constants.CLI_MODE.SELENIUM) {
|
|
104
|
+
const { EyeSdkBuilder } = require('../commons/getSessionPlayerRequire');
|
|
105
|
+
await EyeSdkBuilder.closeBatch(executionId);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
/** @type {Awaited<ReturnType<typeof testimServicesApi['getApplitoolsIntegrationData']>>} */
|
|
109
|
+
let applitoolsIntegrationData;
|
|
110
|
+
try {
|
|
111
|
+
if (!tpOptions.company || !tpOptions.company.activePlan || !tpOptions.company.activePlan.premiumFeatures || !tpOptions.company.activePlan.premiumFeatures.applitools) {
|
|
82
112
|
return;
|
|
83
113
|
}
|
|
84
|
-
|
|
85
|
-
if (
|
|
86
|
-
const { EyeSdkBuilder } = require('../commons/getSessionPlayerRequire');
|
|
87
|
-
await EyeSdkBuilder.closeBatch(executionId);
|
|
114
|
+
applitoolsIntegrationData = await testimServicesApi.getApplitoolsIntegrationData(tpOptions.project);
|
|
115
|
+
if (_.isEmpty(applitoolsIntegrationData) || !executionId) {
|
|
88
116
|
return;
|
|
89
117
|
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
if (_.isEmpty(applitoolsIntegrationData) || !executionId) {
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
const { runKey: apiKey, url: serverUrl } = applitoolsIntegrationData;
|
|
101
|
-
const tmpSDK = require('@applitools/eyes-sdk-core').makeSDK({ name: 'Testim.io', version: '4.0.0', spec: {} });
|
|
102
|
-
await tmpSDK.closeBatches({ batchIds: [executionId], serverUrl, apiKey });
|
|
103
|
-
} catch (err) {
|
|
104
|
-
// If a batch with this name did not exist, do not log an error.
|
|
105
|
-
if (err.message && (err.message.startsWith('Request failed with status code 404') || err.message.startsWith('no batchIds were set'))) {
|
|
106
|
-
return;
|
|
107
|
-
}
|
|
108
|
-
logger.error('Failed closing batch in extension mode', { err, projectId: tpOptions.project, applitoolsIntegrationData, batchIds: [executionId] });
|
|
118
|
+
const { runKey: apiKey, url: serverUrl } = applitoolsIntegrationData;
|
|
119
|
+
const tmpSDK = require('@applitools/eyes-sdk-core').makeSDK({ name: 'Testim.io', version: '4.0.0', spec: {} });
|
|
120
|
+
await tmpSDK.closeBatches({ batchIds: [executionId], serverUrl, apiKey });
|
|
121
|
+
} catch (err) {
|
|
122
|
+
// If a batch with this name did not exist, do not log an error.
|
|
123
|
+
if (err.message && (err.message.startsWith('Request failed with status code 404') || err.message.startsWith('no batchIds were set'))) {
|
|
124
|
+
return;
|
|
109
125
|
}
|
|
110
|
-
|
|
126
|
+
logger.error('Failed closing batch in extension mode', { err, projectId: tpOptions.project, applitoolsIntegrationData, batchIds: [executionId] });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
111
129
|
}
|
|
112
130
|
|
|
131
|
+
/**
|
|
132
|
+
* @private
|
|
133
|
+
* @param {string} projectId
|
|
134
|
+
*/
|
|
113
135
|
async initRealDataService(projectId) {
|
|
114
136
|
const realDataService = new RealDataService();
|
|
115
137
|
await realDataService.init(projectId);
|
|
116
138
|
return realDataService;
|
|
117
139
|
}
|
|
118
140
|
|
|
141
|
+
/**
|
|
142
|
+
* @private
|
|
143
|
+
* @param {RealDataService} realDataService
|
|
144
|
+
* @param {string} projectId
|
|
145
|
+
* @param {string} runId
|
|
146
|
+
* @param {TestRunStatus} testStatus
|
|
147
|
+
*/
|
|
119
148
|
async listenToTestCreatedInFile(realDataService, projectId, runId, testStatus) {
|
|
120
149
|
const childTestResults = {};
|
|
121
150
|
realDataService.joinToTestResultsByRunId(runId, projectId);
|
|
122
|
-
|
|
151
|
+
let isPromisePending = true;
|
|
152
|
+
const promise = new Promise(_resolve => {
|
|
153
|
+
const resolve = (val) => {
|
|
154
|
+
isPromisePending = false;
|
|
155
|
+
_resolve(val);
|
|
156
|
+
};
|
|
123
157
|
realDataService.listenToTestResultsByRunId(runId, testResult => {
|
|
124
158
|
const resultId = testResult.id;
|
|
125
159
|
if (!testStatus.getTestResult(resultId)) {
|
|
@@ -151,9 +185,9 @@ class TestPlanRunner {
|
|
|
151
185
|
|
|
152
186
|
if (allParentTestResultCompleted && !allChildTestResultCompleted) {
|
|
153
187
|
// wait 10 sec to handle race condition when parent test result (file) finished before child test result
|
|
154
|
-
return
|
|
188
|
+
return utils.delay(10000)
|
|
155
189
|
.then(() => {
|
|
156
|
-
if (
|
|
190
|
+
if (isPromisePending) {
|
|
157
191
|
// TODO(Benji) we are missing the child test results here.
|
|
158
192
|
// we are resolving here with partial data - we should consider fetching it
|
|
159
193
|
// from the server
|
|
@@ -168,6 +202,28 @@ class TestPlanRunner {
|
|
|
168
202
|
return await promise;
|
|
169
203
|
}
|
|
170
204
|
|
|
205
|
+
/**
|
|
206
|
+
* @private
|
|
207
|
+
* @param {string} projectId
|
|
208
|
+
* @param {string} executionId
|
|
209
|
+
* @param {TestRunStatus} testStatus
|
|
210
|
+
*/
|
|
211
|
+
async initRealDataServiceAndListenToTestsCreatedInFile(projectId, executionId, testStatus) {
|
|
212
|
+
const realDataService = await this.initRealDataService(projectId);
|
|
213
|
+
return this.listenToTestCreatedInFile(realDataService, projectId, executionId, testStatus);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/**
|
|
217
|
+
* @private
|
|
218
|
+
* @param {any[]} beforeTests
|
|
219
|
+
* @param {any[]} tests
|
|
220
|
+
* @param {any[]} afterTests
|
|
221
|
+
* @param {Awaited<ReturnType<typeof import('../runOptions')['process']>>} tpOptions
|
|
222
|
+
* @param {string} testPlanName
|
|
223
|
+
* @param {string | null} testPlanId
|
|
224
|
+
* @param {string} branch
|
|
225
|
+
* @param {boolean=} isAnonymous
|
|
226
|
+
*/
|
|
171
227
|
async runTestPlan(beforeTests, tests, afterTests, tpOptions, testPlanName, testPlanId, branch, isAnonymous) {
|
|
172
228
|
const executionId = guid();
|
|
173
229
|
const projectId = tpOptions.project;
|
|
@@ -176,37 +232,36 @@ class TestPlanRunner {
|
|
|
176
232
|
afterTests.forEach(test => { test.isAfterTestPlan = true; });
|
|
177
233
|
const testStatus = new TestRunStatus(_.concat(beforeTests, tests, afterTests), tpOptions, testPlanId, branch);
|
|
178
234
|
|
|
179
|
-
const configs = _(_.concat(beforeTests, tests, afterTests))
|
|
235
|
+
const configs = _.chain(_.concat(beforeTests, tests, afterTests))
|
|
236
|
+
.map(test => test.overrideTestConfig?.name || '')
|
|
237
|
+
.uniq()
|
|
238
|
+
.compact()
|
|
180
239
|
.value();
|
|
181
|
-
const configName = configs
|
|
240
|
+
const configName = configs?.length === 1 ? configs[0] : null;
|
|
182
241
|
|
|
183
242
|
const isCodeMode = tpOptions.files.length > 0;
|
|
184
|
-
const testNames = tpOptions.lightweightMode
|
|
243
|
+
const testNames = tpOptions.lightweightMode?.onlyTestIdsNoSuite ? [] : _.concat(beforeTests, tests, afterTests).map(test => test.name);
|
|
185
244
|
|
|
186
|
-
const testListInfoPromise = tpOptions.lightweightMode
|
|
245
|
+
const testListInfoPromise = tpOptions.lightweightMode?.onlyTestIdsNoSuite ?
|
|
187
246
|
{ beforeTests, tests, afterTests } :
|
|
188
247
|
testStatus.executionStart(executionId, projectId, this.startTime, testPlanName, testNames);
|
|
189
248
|
let childTestResults;
|
|
190
249
|
if (isCodeMode) {
|
|
191
|
-
childTestResults =
|
|
192
|
-
const realDataService = await this.initRealDataService(projectId);
|
|
193
|
-
return this.listenToTestCreatedInFile(realDataService, projectId, executionId, testStatus);
|
|
194
|
-
});
|
|
250
|
+
childTestResults = this.initRealDataServiceAndListenToTestsCreatedInFile(projectId, executionId, testStatus);
|
|
195
251
|
}
|
|
196
252
|
perf.log('before testListInfoPromise');
|
|
197
253
|
const testListInfo = await testListInfoPromise;
|
|
198
|
-
if (!
|
|
254
|
+
if (!tpOptions.lightweightMode?.onlyTestIdsNoSuite) {
|
|
199
255
|
reporter.onTestPlanStarted(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, testPlanName, executionId, isAnonymous, configName, isCodeMode);
|
|
200
256
|
}
|
|
201
257
|
|
|
202
258
|
perf.log('before runTestAllPhases');
|
|
203
259
|
const results = await this.runTestAllPhases(testListInfo.beforeTests, testListInfo.tests, testListInfo.afterTests, branch, tpOptions, executionId, testPlanName || 'All Tests', testStatus);
|
|
204
|
-
const childResults = await
|
|
205
|
-
.timeout(TDK_CHILD_RESULTS_TIMEOUT)
|
|
260
|
+
const childResults = await utils.promiseTimeout(childTestResults, TDK_CHILD_RESULTS_TIMEOUT)
|
|
206
261
|
.catch(async () => {
|
|
207
|
-
logger.warn('timed out waiting for child
|
|
262
|
+
logger.warn('timed out waiting for child results on websocket. using REST fallback', { projectId, executionId });
|
|
208
263
|
const testResults = await testimServicesApi.getRealData(projectId, 'testResult', `runId=${executionId}&sort=runOrder`);
|
|
209
|
-
return _.chain((testResults
|
|
264
|
+
return _.chain((testResults?.data?.docs) || [])
|
|
210
265
|
.groupBy('parentResultId')
|
|
211
266
|
.omit('undefined')
|
|
212
267
|
.values()
|
|
@@ -219,6 +274,11 @@ class TestPlanRunner {
|
|
|
219
274
|
return { results, executionId, testPlanName, configName, childTestResults: childResults };
|
|
220
275
|
}
|
|
221
276
|
|
|
277
|
+
/**
|
|
278
|
+
* @private
|
|
279
|
+
* @param {Awaited<ReturnType<typeof import('../runOptions')['process']>>} options
|
|
280
|
+
* @param {string} branchToUse
|
|
281
|
+
*/
|
|
222
282
|
async runTestPlans(options, branchToUse) {
|
|
223
283
|
logger.info('start to run test plan', {
|
|
224
284
|
options: Object.assign({}, options, { token: undefined, userParamsData: undefined }),
|
|
@@ -246,7 +306,7 @@ class TestPlanRunner {
|
|
|
246
306
|
throw new ArgError(`no test to run in test plan ${options.testPlan}`);
|
|
247
307
|
}
|
|
248
308
|
await validateConfig(options, flattenTestListData(testPlansData));
|
|
249
|
-
return await
|
|
309
|
+
return await utils.promiseMap(testPlans, async testPlan => {
|
|
250
310
|
const id = testPlan.testPlanId;
|
|
251
311
|
testPlansResults[id] = {};
|
|
252
312
|
|
|
@@ -265,7 +325,7 @@ class TestPlanRunner {
|
|
|
265
325
|
tpOptions.gridData = await gridService.getTestPlanGridData(options, testPlan);
|
|
266
326
|
|
|
267
327
|
const testPlanName = tpOptions.overrideExecutionName || testPlan.name;
|
|
268
|
-
return await
|
|
328
|
+
return await utils.promiseMap(testPlansData[id], async testPlanTests => {
|
|
269
329
|
const res = await this.runTestPlan(testPlanTests.beforeTests, testPlanTests.tests, testPlanTests.afterTests, tpOptions, testPlanName, id, branchToUse);
|
|
270
330
|
const isCodeMode = options.files.length > 0;
|
|
271
331
|
reporter.onTestPlanFinished(res.results, testPlan.name, this.startTime, res.executionId, false, isCodeMode, res.childTestResults);
|
|
@@ -281,10 +341,15 @@ class TestPlanRunner {
|
|
|
281
341
|
const executionId = success ? executions[0].executionId : executions.find(exec => !exec.success).executionId;
|
|
282
342
|
await testimServicesApi.saveTestPlanResult(projectId, id, { success, executions, executionId });
|
|
283
343
|
return res;
|
|
284
|
-
})
|
|
285
|
-
})
|
|
344
|
+
});
|
|
345
|
+
});
|
|
286
346
|
}
|
|
287
347
|
|
|
348
|
+
/**
|
|
349
|
+
* @private
|
|
350
|
+
* @param {Awaited<ReturnType<typeof import('../runOptions')['process']>>} options
|
|
351
|
+
* @param {string} branchToUse
|
|
352
|
+
*/
|
|
288
353
|
async runAnonymousTestPlan(options, branchToUse) {
|
|
289
354
|
logger.info('start to run anonymous', {
|
|
290
355
|
options: Object.assign({}, options, { token: undefined }),
|
|
@@ -295,7 +360,7 @@ class TestPlanRunner {
|
|
|
295
360
|
const suiteResult = await getSuite(options, branchToUse);
|
|
296
361
|
perf.log('after getSuite');
|
|
297
362
|
|
|
298
|
-
if (!suiteResult.tests[0]
|
|
363
|
+
if (!suiteResult.tests[0]?.length) {
|
|
299
364
|
if (options.rerunFailedByRunId) {
|
|
300
365
|
throw new ArgError('No failed tests found in the provided run');
|
|
301
366
|
}
|
|
@@ -317,7 +382,7 @@ class TestPlanRunner {
|
|
|
317
382
|
const testPlanName = options.overrideExecutionName || suiteResult.runName || _.concat(options.label, options.name, options.suites).join(' ');
|
|
318
383
|
const isAnonymous = true;
|
|
319
384
|
perf.log('Right before validateConfig + runAnonymousTestPlan tests map');
|
|
320
|
-
return await
|
|
385
|
+
return await utils.promiseMap(suiteResult.tests, async suiteTests => { // array of results per execution
|
|
321
386
|
//override result id for remote run mode and run only the first test data
|
|
322
387
|
if (options.resultId) {
|
|
323
388
|
const firstTest = _.first(suiteTests);
|
|
@@ -331,9 +396,12 @@ class TestPlanRunner {
|
|
|
331
396
|
const isCodeMode = options.files.length > 0;
|
|
332
397
|
await reporter.onTestPlanFinished(res.results, testPlanName, this.startTime, res.executionId, isAnonymous, isCodeMode, res.childTestResults);
|
|
333
398
|
return res;
|
|
334
|
-
})
|
|
399
|
+
});
|
|
335
400
|
}
|
|
336
401
|
|
|
402
|
+
/**
|
|
403
|
+
* @param {Awaited<ReturnType<typeof import('../runOptions')['process']>>} options
|
|
404
|
+
*/
|
|
337
405
|
async run(options) {
|
|
338
406
|
const branchToUse = branchService.getCurrentBranch();
|
|
339
407
|
let results = [];
|
|
@@ -1,19 +1,17 @@
|
|
|
1
|
-
|
|
1
|
+
'use strict';
|
|
2
2
|
|
|
3
|
-
const
|
|
3
|
+
const _ = require('lodash');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const utils = require('../utils');
|
|
6
5
|
const Promise = require('bluebird');
|
|
7
|
-
const
|
|
6
|
+
const MemoryFS = require('memory-fs');
|
|
7
|
+
const utils = require('../utils');
|
|
8
8
|
const lazyRequire = require('../commons/lazyRequire');
|
|
9
9
|
const { AbortError } = require('../commons/AbortError');
|
|
10
10
|
|
|
11
|
-
const { isEqual, cloneDeep } = require('lodash');
|
|
12
|
-
|
|
13
11
|
// compiler instance we can reuse between calls
|
|
14
12
|
const state = {
|
|
15
13
|
compiler: null,
|
|
16
|
-
webpackConfig: null
|
|
14
|
+
webpackConfig: null,
|
|
17
15
|
};
|
|
18
16
|
|
|
19
17
|
exports.buildCodeTests = async function buildCodeTestsGuarded(
|
|
@@ -36,36 +34,40 @@ exports.buildCodeTests = async function buildCodeTestsGuarded(
|
|
|
36
34
|
}
|
|
37
35
|
throw e;
|
|
38
36
|
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runnerOptionsToMaybeCopyToTestEnvironment, fileSystem, optionalAbortSignal) {
|
|
37
|
+
};
|
|
42
38
|
|
|
39
|
+
async function buildCodeTests(files, webpackConfig = { mode: 'development' }, runnerOptionsToMaybeCopyToTestEnvironment = undefined, fileSystem = undefined, optionalAbortSignal = undefined) {
|
|
43
40
|
const webpack = await lazyRequire('webpack');
|
|
44
41
|
|
|
45
42
|
const suite = {};
|
|
46
|
-
const webpackConfigBeforeOurChanges = cloneDeep(webpackConfig);
|
|
43
|
+
const webpackConfigBeforeOurChanges = _.cloneDeep(webpackConfig);
|
|
47
44
|
|
|
48
45
|
webpackConfig.externals = { // define testim as an external
|
|
49
|
-
|
|
46
|
+
testim: '__testim',
|
|
50
47
|
// 'chai': '__chai'
|
|
51
48
|
};
|
|
52
49
|
webpackConfig.devtool = webpackConfig.devtool || 'inline-source-map';
|
|
53
50
|
|
|
54
51
|
webpackConfig.plugins = webpackConfig.plugins || [];
|
|
55
52
|
|
|
56
|
-
webpackConfig.plugins.push(new webpack.DefinePlugin(getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment))
|
|
57
|
-
|
|
58
|
-
'process.argv': JSON.stringify(process.argv)
|
|
53
|
+
webpackConfig.plugins.push(new webpack.DefinePlugin(getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment)), new webpack.DefinePlugin({
|
|
54
|
+
'process.argv': JSON.stringify(process.argv),
|
|
59
55
|
}));
|
|
60
56
|
files = files.map(f => path.resolve(f));
|
|
61
57
|
|
|
62
|
-
const fileHashes = files.map(
|
|
58
|
+
const fileHashes = files.map(() => utils.guid(30));
|
|
63
59
|
|
|
64
60
|
webpackConfig.optimization = { minimize: false };
|
|
65
61
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
62
|
+
const majorNodeVersion = Number(process.versions.node.split('.')[0]);
|
|
63
|
+
if (majorNodeVersion >= 17) {
|
|
64
|
+
// TODO ugly hack to be removed once using Webpack 5 - https://stackoverflow.com/a/73465262 + https://github.com/webpack/webpack/issues/14532
|
|
65
|
+
const crypto = require('crypto');
|
|
66
|
+
const originalCryptoCreateHash = crypto.createHash;
|
|
67
|
+
crypto.createHash = (algorithm, options) => originalCryptoCreateHash(algorithm === 'md4' ? 'sha256' : algorithm, options);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
webpackConfig.entry = Object.fromEntries(_.zip(files, fileHashes).map(([filename, hash]) => [hash, filename]));
|
|
69
71
|
webpackConfig.output = Object.assign({
|
|
70
72
|
devtoolModuleFilenameTemplate: (info) => `file:///${info.absoluteResourcePath}`,
|
|
71
73
|
filename: '[name].bundle.js',
|
|
@@ -76,7 +78,7 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
76
78
|
// if we are passed a filesystem, assume reuse between calls and turn on watch mode
|
|
77
79
|
if (fileSystem) {
|
|
78
80
|
// were we passed a filesystem before to compile the same thing?
|
|
79
|
-
if (isEqual(state.webpackConfig, webpackConfigBeforeOurChanges) && state.compiler) {
|
|
81
|
+
if (_.isEqual(state.webpackConfig, webpackConfigBeforeOurChanges) && state.compiler) {
|
|
80
82
|
// we already have a compiler up and running
|
|
81
83
|
compiler = state.compiler;
|
|
82
84
|
} else {
|
|
@@ -92,9 +94,9 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
92
94
|
compiler.outputFileSystem = mfs; // no need to write compiled tests to disk
|
|
93
95
|
|
|
94
96
|
// This can only reject
|
|
95
|
-
const abortSignalPromise =
|
|
97
|
+
const abortSignalPromise = utils.promiseFromCallback(cb => {
|
|
96
98
|
if (optionalAbortSignal) {
|
|
97
|
-
optionalAbortSignal.addEventListener(
|
|
99
|
+
optionalAbortSignal.addEventListener('abort', () => {
|
|
98
100
|
cb(new AbortError());
|
|
99
101
|
});
|
|
100
102
|
}
|
|
@@ -102,12 +104,12 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
102
104
|
|
|
103
105
|
// run compiler:
|
|
104
106
|
try {
|
|
105
|
-
const stats = await Promise.race([
|
|
107
|
+
const stats = await Promise.race([utils.promiseFromCallback(cb => compiler.run(cb)), abortSignalPromise]);
|
|
106
108
|
if (stats.hasErrors()) {
|
|
107
109
|
throw new Error(stats.toJson().errors);
|
|
108
110
|
}
|
|
109
111
|
} catch (e) {
|
|
110
|
-
const {ArgError} = require('../errors');
|
|
112
|
+
const { ArgError } = require('../errors');
|
|
111
113
|
|
|
112
114
|
const cantFindFile = e.message.match(/Entry module not found: Error: Can't resolve '(.*)'/);
|
|
113
115
|
if (cantFindFile && cantFindFile.length === 2) {
|
|
@@ -117,40 +119,39 @@ async function buildCodeTests(files, webpackConfig = {mode: 'development'}, runn
|
|
|
117
119
|
throw new ArgError(`Can't find test files in: '${cantFindFile[1]}'`);
|
|
118
120
|
}
|
|
119
121
|
|
|
120
|
-
throw new ArgError(
|
|
122
|
+
throw new ArgError(`Compilation Webpack Error in tests: ${e.message}`);
|
|
121
123
|
}
|
|
122
124
|
|
|
123
|
-
const fileResults = files.map((file, i) => ({code: mfs.readFileSync(path.resolve('./dist', `${fileHashes[i]}.bundle.js`)), name: file })); // read all files
|
|
125
|
+
const fileResults = files.map((file, i) => ({ code: mfs.readFileSync(path.resolve('./dist', `${fileHashes[i]}.bundle.js`)), name: file })); // read all files
|
|
124
126
|
|
|
125
|
-
suite.tests = [fileResults.map(({code, name}) => ({
|
|
127
|
+
suite.tests = [fileResults.map(({ code, name }) => ({
|
|
126
128
|
code: code.toString(),
|
|
127
|
-
baseUrl:
|
|
129
|
+
baseUrl: '', // not supported at the moment
|
|
128
130
|
name: path.resolve(name),
|
|
129
131
|
testConfig: {},
|
|
130
132
|
testConfigId: null,
|
|
131
133
|
testId: utils.guid(),
|
|
132
134
|
resultId: utils.guid(),
|
|
133
|
-
isTestsContainer: true
|
|
135
|
+
isTestsContainer: true,
|
|
134
136
|
}))];
|
|
135
|
-
suite.runName =
|
|
137
|
+
suite.runName = `Testim Dev Kit Run ${new Date().toLocaleString()}`;
|
|
136
138
|
return suite;
|
|
137
139
|
}
|
|
138
140
|
|
|
139
141
|
// copied mostly from facebook/create-react-app/blob/8b7b819b4b9e6ba457e011e92e33266690e26957/packages/react-scripts/config/env.js
|
|
140
142
|
function getEnvironmentVariables(runnerOptionsToMaybeCopyToTestEnvironment) {
|
|
141
|
-
|
|
142
|
-
let fromEnvironment = _.fromPairs(
|
|
143
|
+
const fromEnvironment = _.fromPairs(
|
|
143
144
|
Object.keys(process.env)
|
|
144
|
-
|
|
145
|
-
|
|
145
|
+
.filter(key => /^TDK_/i.test(key) || key === 'BASE_URL')
|
|
146
|
+
.map(key => [key, process.env[key]])
|
|
146
147
|
);
|
|
147
148
|
|
|
148
|
-
|
|
149
|
-
|
|
149
|
+
const fromConfig = {
|
|
150
|
+
BASE_URL: runnerOptionsToMaybeCopyToTestEnvironment.baseUrl,
|
|
150
151
|
};
|
|
151
152
|
|
|
152
153
|
return {
|
|
153
|
-
'process.env': stringifyValues({ ...fromConfig, ...fromEnvironment})
|
|
154
|
+
'process.env': stringifyValues({ ...fromConfig, ...fromEnvironment }),
|
|
154
155
|
};
|
|
155
156
|
}
|
|
156
157
|
function stringifyValues(object) {
|
package/runners/runnerUtils.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
const _ = require('lodash');
|
|
4
4
|
const path = require('path');
|
|
5
|
-
const utils = require('../utils
|
|
5
|
+
const utils = require('../utils');
|
|
6
6
|
const analytics = require('../commons/testimAnalytics');
|
|
7
7
|
const { ArgError } = require('../errors');
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
async function getSuite(options, branchToUse) {
|
|
11
|
-
if (options.lightweightMode
|
|
11
|
+
if (options.lightweightMode?.onlyTestIdsNoSuite && options.testId) {
|
|
12
12
|
return { tests: [options.testId.map(testId => ({ testId, testConfig: { }, resultId: utils.guid() }))] };
|
|
13
13
|
}
|
|
14
14
|
// local code test
|
|
@@ -17,7 +17,7 @@ async function getSuite(options, branchToUse) {
|
|
|
17
17
|
let webpackConfig = {};
|
|
18
18
|
if (options.webpackConfig) {
|
|
19
19
|
const webpackConfigPath = path.join(process.cwd(), options.webpackConfig);
|
|
20
|
-
webpackConfig = require(webpackConfigPath);
|
|
20
|
+
webpackConfig = require(webpackConfigPath); // eslint-disable-line import/no-dynamic-require
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
return buildCodeTests(options.files, webpackConfig, { baseUrl: options.baseUrl });
|
|
@@ -4,8 +4,6 @@ const pRetry = require('p-retry');
|
|
|
4
4
|
const fse = require('fs-extra');
|
|
5
5
|
const portfinder = require('portfinder');
|
|
6
6
|
const ms = require('ms');
|
|
7
|
-
|
|
8
|
-
const { guid, isURL } = require('../utils');
|
|
9
7
|
const utils = require('../utils');
|
|
10
8
|
const { gridTypes, CLI_MODE } = require('../commons/constants');
|
|
11
9
|
const httpRequest = require('../commons/httpRequest');
|
|
@@ -98,7 +96,7 @@ class LambdatestService {
|
|
|
98
96
|
const infoAPIPort = await portfinder.getPortPromise();
|
|
99
97
|
const { gridData = {}, gridUsername, gridPassword } = runnerOptions;
|
|
100
98
|
const proxyUri = global.proxyUri;
|
|
101
|
-
LambdatestService.tunnelName = guid();
|
|
99
|
+
LambdatestService.tunnelName = utils.guid();
|
|
102
100
|
|
|
103
101
|
let tunnelArgs = [
|
|
104
102
|
'--tunnelName', LambdatestService.tunnelName,
|
|
@@ -198,11 +196,11 @@ class LambdatestService {
|
|
|
198
196
|
if (!extensionPath && extUrls[browser]) {
|
|
199
197
|
loadExtension = [...loadExtension, extUrls[browser]];
|
|
200
198
|
}
|
|
201
|
-
if (extensionPath && isURL(extensionPath)) {
|
|
199
|
+
if (extensionPath && utils.isURL(extensionPath)) {
|
|
202
200
|
loadExtension = [...loadExtension, extensionPath];
|
|
203
201
|
}
|
|
204
202
|
}
|
|
205
|
-
if (installCustomExtension && isURL(installCustomExtension)) {
|
|
203
|
+
if (installCustomExtension && utils.isURL(installCustomExtension)) {
|
|
206
204
|
loadExtension = [...loadExtension, installCustomExtension];
|
|
207
205
|
}
|
|
208
206
|
|
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
|
|
1
|
+
'use strict';
|
|
2
2
|
|
|
3
3
|
const service = require('../agent/routers/cliJsCode/service');
|
|
4
|
-
const
|
|
4
|
+
const { TimeoutError } = require('bluebird');
|
|
5
5
|
const featureFlags = require('../commons/featureFlags');
|
|
6
|
-
const logger = require('../commons/logger').getLogger('cli-js-step-playback');
|
|
7
6
|
|
|
8
7
|
function isExceedingMaxResultSize(data, project) {
|
|
9
8
|
try {
|
|
10
9
|
const shouldEnforceMaxSize = project.defaults.enforceMaximumJsResultSize;
|
|
11
10
|
const maximumJsResultSize = featureFlags.flags.maximumJsResultSize.getValue();
|
|
12
11
|
const dataSizeExceeded = JSON.stringify(data).length > maximumJsResultSize;
|
|
13
|
-
if(!shouldEnforceMaxSize) {
|
|
12
|
+
if (!shouldEnforceMaxSize) {
|
|
14
13
|
return false;
|
|
15
14
|
}
|
|
16
15
|
return dataSizeExceeded;
|
|
@@ -19,17 +18,23 @@ function isExceedingMaxResultSize(data, project) {
|
|
|
19
18
|
}
|
|
20
19
|
}
|
|
21
20
|
|
|
22
|
-
module.exports.run = (browser, step, projectData) => {
|
|
23
|
-
const {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
}
|
|
34
|
-
|
|
21
|
+
module.exports.run = async (browser, step, projectData) => {
|
|
22
|
+
const {
|
|
23
|
+
code, stepId, incomingParams, context, testResultId, retryIndex, stepResultId, runTimeout, fileDataUrl, s3filepath,
|
|
24
|
+
} = step.data;
|
|
25
|
+
try {
|
|
26
|
+
const data = await service.runCodeWithPackages(code, stepId, incomingParams, context, testResultId, retryIndex, stepResultId, runTimeout, fileDataUrl, s3filepath);
|
|
27
|
+
if (data && isExceedingMaxResultSize({ result: data.result, tstConsoleLogs: data.tstConsoleLogs }, projectData)) {
|
|
28
|
+
return {
|
|
29
|
+
code: 'js-result-max-size-exceeded',
|
|
30
|
+
success: false,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
return { data, success: true };
|
|
34
|
+
} catch (err) {
|
|
35
|
+
if (err instanceof TimeoutError) {
|
|
36
|
+
throw new Error('Timeout while running action');
|
|
37
|
+
}
|
|
38
|
+
throw err;
|
|
39
|
+
}
|
|
35
40
|
};
|