@applitools/eyes-storybook 3.65.8 → 3.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +135 -0
- package/package.json +11 -9
- package/src/buildExecutionSummaryEvent.js +75 -0
- package/src/eyesStorybook.js +103 -1
- package/src/initPage.js +10 -1
- package/src/pagePool.js +2 -1
- package/src/renderStories.js +9 -0
- package/src/renderStory.js +6 -0
- package/src/runMetrics.js +159 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,140 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [3.67.0](https://github.com/Applitools-Dev/sdk/compare/js/eyes-storybook@3.66.0...js/eyes-storybook@3.67.0) (2026-06-28)
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
### Features
|
|
7
|
+
|
|
8
|
+
* per-phase timing in the summary event | AD-14515 ([#3942](https://github.com/Applitools-Dev/sdk/issues/3942)) ([3704a6d](https://github.com/Applitools-Dev/sdk/commit/3704a6d60f07e4338f3d0c5fa1589baf89b45080))
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Dependencies
|
|
12
|
+
|
|
13
|
+
* @applitools/utils bumped to 1.15.1
|
|
14
|
+
#### Bug Fixes
|
|
15
|
+
|
|
16
|
+
* updated extract git fallback logic | FLD-4492 ([#3831](https://github.com/Applitools-Dev/sdk/issues/3831)) ([0d98f3c](https://github.com/Applitools-Dev/sdk/commit/0d98f3cdf828eff756ffbfc5d4cdff7b19a20557))
|
|
17
|
+
* @applitools/logger bumped to 2.3.0
|
|
18
|
+
#### Features
|
|
19
|
+
|
|
20
|
+
* stream SDK logs to Azure Blob per server flag | AD-13401 ([#3849](https://github.com/Applitools-Dev/sdk/issues/3849)) ([7f9dfec](https://github.com/Applitools-Dev/sdk/commit/7f9dfecd874aefc33104db015451a22eee500324))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
* @applitools/core-base bumped to 1.36.0
|
|
25
|
+
#### Features
|
|
26
|
+
|
|
27
|
+
* stream SDK logs to Azure Blob per server flag | AD-13401 ([#3849](https://github.com/Applitools-Dev/sdk/issues/3849)) ([7f9dfec](https://github.com/Applitools-Dev/sdk/commit/7f9dfecd874aefc33104db015451a22eee500324))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
* @applitools/dom-snapshot bumped to 4.17.5
|
|
32
|
+
|
|
33
|
+
* @applitools/socket bumped to 1.3.15
|
|
34
|
+
|
|
35
|
+
* @applitools/req bumped to 1.11.2
|
|
36
|
+
|
|
37
|
+
* @applitools/image bumped to 1.2.13
|
|
38
|
+
|
|
39
|
+
* @applitools/dom-capture bumped to 11.8.3
|
|
40
|
+
|
|
41
|
+
* @applitools/driver bumped to 1.26.5
|
|
42
|
+
|
|
43
|
+
* @applitools/spec-driver-webdriver bumped to 1.6.5
|
|
44
|
+
|
|
45
|
+
* @applitools/spec-driver-selenium bumped to 1.8.5
|
|
46
|
+
|
|
47
|
+
* @applitools/spec-driver-playwright bumped to 1.9.5
|
|
48
|
+
|
|
49
|
+
* @applitools/spec-driver-puppeteer bumped to 1.8.5
|
|
50
|
+
|
|
51
|
+
* @applitools/screenshoter bumped to 3.12.24
|
|
52
|
+
|
|
53
|
+
* @applitools/nml-client bumped to 1.11.34
|
|
54
|
+
|
|
55
|
+
* @applitools/tunnel-client bumped to 1.12.2
|
|
56
|
+
|
|
57
|
+
* @applitools/ufg-client bumped to 1.22.5
|
|
58
|
+
|
|
59
|
+
* @applitools/ec-client bumped to 1.12.36
|
|
60
|
+
|
|
61
|
+
* @applitools/core bumped to 4.66.0
|
|
62
|
+
#### Features
|
|
63
|
+
|
|
64
|
+
* stream SDK logs to Azure Blob per server flag | AD-13401 ([#3849](https://github.com/Applitools-Dev/sdk/issues/3849)) ([7f9dfec](https://github.com/Applitools-Dev/sdk/commit/7f9dfecd874aefc33104db015451a22eee500324))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
#### Bug Fixes
|
|
68
|
+
|
|
69
|
+
* updated extract git fallback logic | FLD-4492 ([#3831](https://github.com/Applitools-Dev/sdk/issues/3831)) ([0d98f3c](https://github.com/Applitools-Dev/sdk/commit/0d98f3cdf828eff756ffbfc5d4cdff7b19a20557))
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
* @applitools/eyes bumped to 1.43.5
|
|
74
|
+
|
|
75
|
+
* @applitools/test-server bumped to 1.4.6
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
## [3.66.0](https://github.com/Applitools-Dev/sdk/compare/js/eyes-storybook@3.65.8...js/eyes-storybook@3.66.0) (2026-06-18)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
### Features
|
|
82
|
+
|
|
83
|
+
* one-time per-execution summary logEvent in storybook | AD-14305 ([#3911](https://github.com/Applitools-Dev/sdk/issues/3911)) ([59e3a5d](https://github.com/Applitools-Dev/sdk/commit/59e3a5dccf74aa1ac1158024316bd305e3515afa))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
### Dependencies
|
|
87
|
+
|
|
88
|
+
* @applitools/utils bumped to 1.15.0
|
|
89
|
+
#### Features
|
|
90
|
+
|
|
91
|
+
* one-time per-execution summary logEvent in storybook | AD-14305 ([#3911](https://github.com/Applitools-Dev/sdk/issues/3911)) ([59e3a5d](https://github.com/Applitools-Dev/sdk/commit/59e3a5dccf74aa1ac1158024316bd305e3515afa))
|
|
92
|
+
* @applitools/req bumped to 1.11.1
|
|
93
|
+
#### Bug Fixes
|
|
94
|
+
|
|
95
|
+
* bundle node-fetch into dist to drop node-domexception dep warning | FLD-4197 ([#3899](https://github.com/Applitools-Dev/sdk/issues/3899)) ([9c3dc52](https://github.com/Applitools-Dev/sdk/commit/9c3dc52866a4a665f1578c39ff43cd0ee5f63492))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
* @applitools/logger bumped to 2.2.13
|
|
100
|
+
|
|
101
|
+
* @applitools/dom-snapshot bumped to 4.17.4
|
|
102
|
+
|
|
103
|
+
* @applitools/socket bumped to 1.3.14
|
|
104
|
+
|
|
105
|
+
* @applitools/image bumped to 1.2.12
|
|
106
|
+
|
|
107
|
+
* @applitools/dom-capture bumped to 11.8.2
|
|
108
|
+
|
|
109
|
+
* @applitools/driver bumped to 1.26.4
|
|
110
|
+
|
|
111
|
+
* @applitools/spec-driver-webdriver bumped to 1.6.4
|
|
112
|
+
|
|
113
|
+
* @applitools/spec-driver-selenium bumped to 1.8.4
|
|
114
|
+
|
|
115
|
+
* @applitools/spec-driver-playwright bumped to 1.9.4
|
|
116
|
+
|
|
117
|
+
* @applitools/spec-driver-puppeteer bumped to 1.8.4
|
|
118
|
+
|
|
119
|
+
* @applitools/screenshoter bumped to 3.12.23
|
|
120
|
+
|
|
121
|
+
* @applitools/nml-client bumped to 1.11.33
|
|
122
|
+
|
|
123
|
+
* @applitools/tunnel-client bumped to 1.12.1
|
|
124
|
+
|
|
125
|
+
* @applitools/ufg-client bumped to 1.22.4
|
|
126
|
+
|
|
127
|
+
* @applitools/core-base bumped to 1.35.3
|
|
128
|
+
|
|
129
|
+
* @applitools/ec-client bumped to 1.12.35
|
|
130
|
+
|
|
131
|
+
* @applitools/core bumped to 4.65.2
|
|
132
|
+
|
|
133
|
+
* @applitools/eyes bumped to 1.43.4
|
|
134
|
+
|
|
135
|
+
* @applitools/test-server bumped to 1.4.5
|
|
136
|
+
|
|
137
|
+
|
|
3
138
|
## [3.65.8](https://github.com/Applitools-Dev/sdk/compare/js/eyes-storybook@3.65.7...js/eyes-storybook@3.65.8) (2026-06-15)
|
|
4
139
|
|
|
5
140
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@applitools/eyes-storybook",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.67.0",
|
|
4
4
|
"description": "",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"applitools",
|
|
@@ -36,6 +36,7 @@
|
|
|
36
36
|
"test:it": "run --top-level mocha 'test/it/*.test.js'",
|
|
37
37
|
"test:versions": "yarn build && yarn test:e2e:versions && yarn lint",
|
|
38
38
|
"test:e2e": "MOCHA_GROUP=e2e run --top-level mocha 'test/e2e/*.e2e.test.js' --require ./test/mocha-global-setup.js",
|
|
39
|
+
"test:performance-logs": "MOCHA_GROUP=performance_logs run --top-level mocha 'test/e2e/execution-summary-event.e2e.test.js' --require ./test/mocha-global-setup.js --exit",
|
|
39
40
|
"test:e2e:versions": "MOCHA_GROUP=e2e_versions run --top-level mocha test/versions/*.e2e.versions.test.js",
|
|
40
41
|
"test:sanity": "run --top-level mocha 'test/sanity/*.test.js'",
|
|
41
42
|
"test:heavy": "node bin/eyes-storybook.js -f test/fixtures/heavyStorybook/applitools.config.js",
|
|
@@ -59,15 +60,15 @@
|
|
|
59
60
|
"up:framework": "cd test/fixtures/storybook-versions/${APPLITOOLS_FRAMEWORK_VERSION} && npm ci"
|
|
60
61
|
},
|
|
61
62
|
"dependencies": {
|
|
62
|
-
"@applitools/core": "4.
|
|
63
|
-
"@applitools/driver": "1.26.
|
|
64
|
-
"@applitools/eyes": "1.43.
|
|
63
|
+
"@applitools/core": "4.66.0",
|
|
64
|
+
"@applitools/driver": "1.26.5",
|
|
65
|
+
"@applitools/eyes": "1.43.5",
|
|
65
66
|
"@applitools/functional-commons": "1.6.0",
|
|
66
|
-
"@applitools/logger": "2.
|
|
67
|
+
"@applitools/logger": "2.3.0",
|
|
67
68
|
"@applitools/monitoring-commons": "1.0.19",
|
|
68
|
-
"@applitools/spec-driver-puppeteer": "1.8.
|
|
69
|
-
"@applitools/ufg-client": "1.22.
|
|
70
|
-
"@applitools/utils": "1.
|
|
69
|
+
"@applitools/spec-driver-puppeteer": "1.8.5",
|
|
70
|
+
"@applitools/ufg-client": "1.22.5",
|
|
71
|
+
"@applitools/utils": "1.15.1",
|
|
71
72
|
"@inquirer/prompts": "7.0.1",
|
|
72
73
|
"boxen": "4.2.0",
|
|
73
74
|
"chalk": "3.0.0",
|
|
@@ -83,7 +84,7 @@
|
|
|
83
84
|
"devDependencies": {
|
|
84
85
|
"@applitools/bongo": "^5.10.0",
|
|
85
86
|
"@applitools/snaptdout": "^1.1.1",
|
|
86
|
-
"@applitools/test-server": "^1.4.
|
|
87
|
+
"@applitools/test-server": "^1.4.6",
|
|
87
88
|
"@applitools/test-utils": "^1.5.17",
|
|
88
89
|
"@storybook/addon-interactions": "^6.4.18",
|
|
89
90
|
"@storybook/react": "^6.5.16",
|
|
@@ -117,6 +118,7 @@
|
|
|
117
118
|
"core-js": false,
|
|
118
119
|
"puppeteer": true,
|
|
119
120
|
"@storybook/react>core-js": false,
|
|
121
|
+
"@storybook/react>@storybook/core-common>webpack>watchpack>watchpack-chokidar2>chokidar>fsevents": false,
|
|
120
122
|
"@applitools/ufg-client>@applitools/image>sharp": false,
|
|
121
123
|
"puppeteer>puppeteer-core>ws>bufferutil": false,
|
|
122
124
|
"puppeteer>puppeteer-core>ws>utf-8-validate": false
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Credentials/connection details that must never be sent to Coralogix. Everything else in
|
|
4
|
+
// the config (puppeteerOptions, hooks, flags, ...) is reported as-is.
|
|
5
|
+
const SETTINGS_DENY_LIST = ['apiKey', 'proxy', 'eyesServerUrl', 'serverUrl', 'accessKey'];
|
|
6
|
+
|
|
7
|
+
function pickSettings(config = {}) {
|
|
8
|
+
const flags = {};
|
|
9
|
+
for (const [key, value] of Object.entries(config)) {
|
|
10
|
+
if (!SETTINGS_DENY_LIST.includes(key)) flags[key] = value;
|
|
11
|
+
}
|
|
12
|
+
return flags;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Assemble the one-time per-execution summary event payload. Pure function — all
|
|
17
|
+
* inputs are plain data; never throws on missing fields (best-effort observability).
|
|
18
|
+
*/
|
|
19
|
+
function buildExecutionSummaryEvent({
|
|
20
|
+
status, // 'finished' | 'failed' | 'aborted'
|
|
21
|
+
config = {},
|
|
22
|
+
environment = {},
|
|
23
|
+
account = {},
|
|
24
|
+
concurrentTabs,
|
|
25
|
+
counts = {},
|
|
26
|
+
summary, // TestResultSummary | undefined
|
|
27
|
+
metrics = {}, // runMetrics.summarize()
|
|
28
|
+
totalRunMs,
|
|
29
|
+
renderStoriesMs,
|
|
30
|
+
}) {
|
|
31
|
+
// keep the scalar counts (passed/failed/unresolved/...); drop the large per-test `results` array
|
|
32
|
+
let tests;
|
|
33
|
+
if (summary) {
|
|
34
|
+
tests = {...summary};
|
|
35
|
+
delete tests.results;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
type: 'eyesStorybookExecutionFinished',
|
|
40
|
+
status,
|
|
41
|
+
sdkVersion: environment.sdk?.currentVersion,
|
|
42
|
+
addonVersion: environment.sdk?.addonVersion,
|
|
43
|
+
storybookVersion: config.sbVersion || environment.versions?.storybook,
|
|
44
|
+
framework: environment.sdk?.framework,
|
|
45
|
+
counts: {
|
|
46
|
+
totalStories: counts.totalStories,
|
|
47
|
+
afterFilter: counts.afterFilter,
|
|
48
|
+
afterVariations: counts.afterVariations,
|
|
49
|
+
normal: counts.normal,
|
|
50
|
+
fakeIE: counts.fakeIE,
|
|
51
|
+
tests,
|
|
52
|
+
},
|
|
53
|
+
timing: {
|
|
54
|
+
totalRunMs,
|
|
55
|
+
renderStoriesMs,
|
|
56
|
+
perStoryTotalMs: metrics.timing?.perStoryTotalMs,
|
|
57
|
+
snapshotMs: metrics.timing?.snapshotMs,
|
|
58
|
+
openMs: metrics.timing?.openMs,
|
|
59
|
+
checkAndCloseMs: metrics.timing?.checkAndCloseMs,
|
|
60
|
+
},
|
|
61
|
+
memory: metrics.memory,
|
|
62
|
+
tabs: {
|
|
63
|
+
concurrentTabs,
|
|
64
|
+
crashed: metrics.tabs?.tabsCrashed,
|
|
65
|
+
replaced: metrics.tabs?.tabsReplaced,
|
|
66
|
+
},
|
|
67
|
+
settings: {
|
|
68
|
+
componentConcurrency: account.serverConcurrency?.componentConcurrency,
|
|
69
|
+
concurrentTabs,
|
|
70
|
+
flags: pickSettings(config),
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
module.exports = {buildExecutionSummaryEvent, pickSettings, SETTINGS_DENY_LIST};
|
package/src/eyesStorybook.js
CHANGED
|
@@ -26,6 +26,8 @@ const {makeCore} = require('@applitools/core');
|
|
|
26
26
|
const makeGetStoriesWithConfig = require('./getStoriesWithConfig');
|
|
27
27
|
const {makeNetworkUtils} = require('./utils/pageNetworkUtils');
|
|
28
28
|
const {readStoriesTimeout: defaultReadStoriesTimeout} = require('./defaultConfig');
|
|
29
|
+
const {makeRunMetrics} = require('./runMetrics');
|
|
30
|
+
const {buildExecutionSummaryEvent} = require('./buildExecutionSummaryEvent');
|
|
29
31
|
|
|
30
32
|
async function eyesStorybook({
|
|
31
33
|
config,
|
|
@@ -43,6 +45,11 @@ async function eyesStorybook({
|
|
|
43
45
|
let transitioning = false;
|
|
44
46
|
logger.log('eyesStorybook started');
|
|
45
47
|
|
|
48
|
+
const runStart = Date.now();
|
|
49
|
+
// Run-scope counts, populated during the pipeline and read by the summary event
|
|
50
|
+
// even if the run later fails partway through (best-effort observability).
|
|
51
|
+
const counts = {};
|
|
52
|
+
|
|
46
53
|
const CONCURRENT_TABS = isNaN(Number(process.env.APPLITOOLS_CONCURRENT_TABS))
|
|
47
54
|
? 3
|
|
48
55
|
: Number(process.env.APPLITOOLS_CONCURRENT_TABS);
|
|
@@ -64,6 +71,11 @@ async function eyesStorybook({
|
|
|
64
71
|
const browser = await puppeteer.launch(config.puppeteerOptions);
|
|
65
72
|
logger.log('browser launched');
|
|
66
73
|
|
|
74
|
+
// Best-effort per-execution metrics (memory / timing / tab reliability). The
|
|
75
|
+
// sampler is unref'd and every sample is guarded — it can never keep the
|
|
76
|
+
// process alive or affect the run.
|
|
77
|
+
const metrics = makeRunMetrics({browser, logger});
|
|
78
|
+
|
|
67
79
|
const environment = extractEnvironment(addonVersion);
|
|
68
80
|
const core = await makeCore({spec, agentId: config.agentId, environment, logger});
|
|
69
81
|
const manager = await core.makeManager({
|
|
@@ -87,6 +99,11 @@ async function eyesStorybook({
|
|
|
87
99
|
throw error;
|
|
88
100
|
});
|
|
89
101
|
|
|
102
|
+
// Start the best-effort metrics sampler only once getAccountInfo has succeeded —
|
|
103
|
+
// its .catch closes the browser and rethrows without reaching the `finally` that
|
|
104
|
+
// would otherwise stop the sampler.
|
|
105
|
+
metrics.start();
|
|
106
|
+
|
|
90
107
|
const getStoriesWithConfig = makeGetStoriesWithConfig({config});
|
|
91
108
|
|
|
92
109
|
const initPage = makeInitPage({
|
|
@@ -96,8 +113,72 @@ async function eyesStorybook({
|
|
|
96
113
|
logger,
|
|
97
114
|
getTransitiongIntoIE,
|
|
98
115
|
getRenderIE,
|
|
116
|
+
onTabCrashed: metrics.incrementTabsCrashed,
|
|
117
|
+
});
|
|
118
|
+
const pagePool = createPagePool({
|
|
119
|
+
initPage,
|
|
120
|
+
logger,
|
|
121
|
+
onPageReplaced: metrics.incrementTabsReplaced,
|
|
99
122
|
});
|
|
100
|
-
|
|
123
|
+
|
|
124
|
+
let summaryEmitted = false;
|
|
125
|
+
let runStatus = 'finished';
|
|
126
|
+
let resultSummary;
|
|
127
|
+
|
|
128
|
+
// Emit the one-time per-execution summary event (-> Coralogix). Best-effort and
|
|
129
|
+
// idempotent: awaited so the batched logEvent actually flushes + sends before we
|
|
130
|
+
// exit, guarded so we never double-send, and fully swallowed on failure so it can
|
|
131
|
+
// never affect the run's outcome.
|
|
132
|
+
async function emitExecutionSummary(status, summary) {
|
|
133
|
+
if (summaryEmitted) return;
|
|
134
|
+
summaryEmitted = true;
|
|
135
|
+
try {
|
|
136
|
+
await metrics.stop(); // final sample while the browser is still alive
|
|
137
|
+
const event = buildExecutionSummaryEvent({
|
|
138
|
+
status,
|
|
139
|
+
config,
|
|
140
|
+
environment,
|
|
141
|
+
account,
|
|
142
|
+
concurrentTabs: CONCURRENT_TABS,
|
|
143
|
+
counts,
|
|
144
|
+
summary,
|
|
145
|
+
metrics: metrics.summarize(),
|
|
146
|
+
totalRunMs: Date.now() - runStart,
|
|
147
|
+
renderStoriesMs: performance['renderStories'],
|
|
148
|
+
});
|
|
149
|
+
logger.log('eyesStorybook sending execution summary event', event);
|
|
150
|
+
await core.logEvent({
|
|
151
|
+
settings: {
|
|
152
|
+
eyesServerUrl: config.eyesServerUrl,
|
|
153
|
+
apiKey: config.apiKey,
|
|
154
|
+
agentId: config.agentId,
|
|
155
|
+
proxy: config.proxy,
|
|
156
|
+
useDnsCache: config.useDnsCache,
|
|
157
|
+
level: 'Notice',
|
|
158
|
+
event,
|
|
159
|
+
},
|
|
160
|
+
logger,
|
|
161
|
+
});
|
|
162
|
+
} catch (err) {
|
|
163
|
+
logger.log('failed to emit execution summary event:', err && err.message);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// process 'exit' is synchronous-only and cannot perform the async, batched HTTP
|
|
168
|
+
// send — so cover hard terminations with signal handlers (which CAN run async).
|
|
169
|
+
// Registered now and removed in `finally` so we never leak a listener.
|
|
170
|
+
const makeSignalHandler = signalName => async () => {
|
|
171
|
+
logger.log(`eyesStorybook received ${signalName}, emitting summary before exit`);
|
|
172
|
+
await emitExecutionSummary('aborted', resultSummary);
|
|
173
|
+
process.exit(1);
|
|
174
|
+
};
|
|
175
|
+
const signalHandlers = {
|
|
176
|
+
SIGINT: makeSignalHandler('SIGINT'),
|
|
177
|
+
SIGTERM: makeSignalHandler('SIGTERM'),
|
|
178
|
+
};
|
|
179
|
+
for (const [signalName, handler] of Object.entries(signalHandlers)) {
|
|
180
|
+
process.once(signalName, handler);
|
|
181
|
+
}
|
|
101
182
|
|
|
102
183
|
try {
|
|
103
184
|
const {stories, page} = await getStoriesWithSpinner();
|
|
@@ -121,6 +202,8 @@ async function eyesStorybook({
|
|
|
121
202
|
});
|
|
122
203
|
|
|
123
204
|
const filteredStories = filterStories({stories, config});
|
|
205
|
+
counts.totalStories = stories.length;
|
|
206
|
+
counts.afterFilter = filteredStories.length;
|
|
124
207
|
|
|
125
208
|
// Log filtering and sharding results
|
|
126
209
|
logger.log(`${stories.length} total stories found`);
|
|
@@ -141,10 +224,14 @@ async function eyesStorybook({
|
|
|
141
224
|
`there are ${storiesIncludingVariations.length} stories after filtering and adding variations `,
|
|
142
225
|
);
|
|
143
226
|
|
|
227
|
+
counts.afterVariations = storiesIncludingVariations.length;
|
|
228
|
+
|
|
144
229
|
const storiesByBrowserWithConfig = getStoriesWithConfig({
|
|
145
230
|
stories: storiesIncludingVariations,
|
|
146
231
|
logger,
|
|
147
232
|
});
|
|
233
|
+
counts.normal = storiesByBrowserWithConfig.stories.length;
|
|
234
|
+
counts.fakeIE = storiesByBrowserWithConfig.storiesWithIE.length;
|
|
148
235
|
|
|
149
236
|
logger.log(
|
|
150
237
|
`starting to run ${storiesByBrowserWithConfig.stories.length} normal stories ("non fake IE") and ${storiesByBrowserWithConfig.storiesWithIE.length} "fake IE stories"`,
|
|
@@ -166,6 +253,8 @@ async function eyesStorybook({
|
|
|
166
253
|
appName: config.appName,
|
|
167
254
|
serverSettings: account.eyesServer,
|
|
168
255
|
signal,
|
|
256
|
+
recordOpenTime: metrics.recordOpenTime,
|
|
257
|
+
recordCheckAndCloseTime: metrics.recordCheckAndCloseTime,
|
|
169
258
|
});
|
|
170
259
|
|
|
171
260
|
const renderStories = makeRenderStories({
|
|
@@ -178,6 +267,8 @@ async function eyesStorybook({
|
|
|
178
267
|
pagePool,
|
|
179
268
|
eventEmitter,
|
|
180
269
|
signal,
|
|
270
|
+
recordStoryTime: metrics.recordStoryTime,
|
|
271
|
+
recordSnapshotTime: metrics.recordSnapshotTime,
|
|
181
272
|
});
|
|
182
273
|
|
|
183
274
|
logger.log('finished creating functions');
|
|
@@ -196,6 +287,7 @@ async function eyesStorybook({
|
|
|
196
287
|
}),
|
|
197
288
|
);
|
|
198
289
|
if (signal.aborted) {
|
|
290
|
+
runStatus = 'aborted';
|
|
199
291
|
if (error) {
|
|
200
292
|
const msg = refineErrorMessage({prefix: 'Error in executeRenders:', error});
|
|
201
293
|
logger.log('Error in executeRenders:', error);
|
|
@@ -211,17 +303,27 @@ async function eyesStorybook({
|
|
|
211
303
|
if (errorInGetResults) {
|
|
212
304
|
logger.log('failed to get results', errorInGetResults);
|
|
213
305
|
}
|
|
306
|
+
resultSummary = testResultsSummary;
|
|
214
307
|
|
|
215
308
|
if (error) {
|
|
309
|
+
runStatus = 'failed';
|
|
216
310
|
const msg = refineErrorMessage({prefix: 'Error in executeRenders:', error});
|
|
217
311
|
logger.log('Error in executeRenders:', error);
|
|
218
312
|
throw new Error(msg);
|
|
219
313
|
} else {
|
|
220
314
|
return {summary: testResultsSummary, results};
|
|
221
315
|
}
|
|
316
|
+
} catch (err) {
|
|
317
|
+
if (runStatus === 'finished') runStatus = 'failed';
|
|
318
|
+
throw err;
|
|
222
319
|
} finally {
|
|
223
320
|
logger.log('total time: ', performance['renderStories']);
|
|
224
321
|
logger.log('perf results', performance);
|
|
322
|
+
// emit BEFORE closing the browser so the final memory sample is still valid
|
|
323
|
+
await emitExecutionSummary(runStatus, resultSummary);
|
|
324
|
+
for (const [signalName, handler] of Object.entries(signalHandlers)) {
|
|
325
|
+
process.removeListener(signalName, handler);
|
|
326
|
+
}
|
|
225
327
|
pagePool.isClosed = true;
|
|
226
328
|
await browser.close();
|
|
227
329
|
}
|
package/src/initPage.js
CHANGED
|
@@ -4,7 +4,15 @@ const fakeIE = require('./fakeIE');
|
|
|
4
4
|
const {makeNetworkUtils} = require('./utils/pageNetworkUtils');
|
|
5
5
|
const utils = require('@applitools/utils');
|
|
6
6
|
|
|
7
|
-
function makeInitPage({
|
|
7
|
+
function makeInitPage({
|
|
8
|
+
iframeUrl,
|
|
9
|
+
config,
|
|
10
|
+
browser,
|
|
11
|
+
logger,
|
|
12
|
+
getTransitiongIntoIE,
|
|
13
|
+
getRenderIE,
|
|
14
|
+
onTabCrashed = () => {},
|
|
15
|
+
}) {
|
|
8
16
|
const networkUtils = makeNetworkUtils({logger});
|
|
9
17
|
return async function initPage({pageId, pagePool}) {
|
|
10
18
|
logger.log('initializing puppeteer page number ', pageId);
|
|
@@ -60,6 +68,7 @@ function makeInitPage({iframeUrl, config, browser, logger, getTransitiongIntoIE,
|
|
|
60
68
|
|
|
61
69
|
page.on('error', async err => {
|
|
62
70
|
logger.log(`Puppeteer error for page ${pageId}:`, err);
|
|
71
|
+
onTabCrashed(); // best-effort metric; the 'error' event is Chromium "Page crashed!"
|
|
63
72
|
await pagePool.removeAndAddPage(pageId);
|
|
64
73
|
});
|
|
65
74
|
|
package/src/pagePool.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
function createPagePool({initPage, logger, freePageTimeout = 300_000}) {
|
|
3
|
+
function createPagePool({initPage, logger, freePageTimeout = 300_000, onPageReplaced = () => {}}) {
|
|
4
4
|
let counter = 0;
|
|
5
5
|
const fullPageObjs = [];
|
|
6
6
|
logger.log(`[page pool] created`);
|
|
@@ -43,6 +43,7 @@ function createPagePool({initPage, logger, freePageTimeout = 300_000}) {
|
|
|
43
43
|
return pagePool;
|
|
44
44
|
|
|
45
45
|
async function removeAndAddPage(pageId) {
|
|
46
|
+
onPageReplaced(); // best-effort metric: a pooled tab was torn down and replaced
|
|
46
47
|
pagePool.removePage(pageId);
|
|
47
48
|
const {pageId: newPageId} = await pagePool.createPage();
|
|
48
49
|
pagePool.addToPool(newPageId);
|
package/src/renderStories.js
CHANGED
|
@@ -17,6 +17,8 @@ function makeRenderStories({
|
|
|
17
17
|
maxPageTTL = 60000,
|
|
18
18
|
eventEmitter = new EventEmitter(),
|
|
19
19
|
signal = new AbortController().signal,
|
|
20
|
+
recordStoryTime = () => {},
|
|
21
|
+
recordSnapshotTime = () => {},
|
|
20
22
|
}) {
|
|
21
23
|
const sparePages = {
|
|
22
24
|
queue: [],
|
|
@@ -142,7 +144,11 @@ function makeRenderStories({
|
|
|
142
144
|
const story = stories[currIndex++];
|
|
143
145
|
const storyUrl = getStoryUrl(story, storybookUrl);
|
|
144
146
|
const title = getStoryBaselineName(story);
|
|
147
|
+
// measure the FULL per-story chain (snapshot + open + checkAndClose),
|
|
148
|
+
// unlike performance[baselineName] which covers only open + checkAndClose.
|
|
149
|
+
const storyStart = Date.now();
|
|
145
150
|
try {
|
|
151
|
+
const snapshotStart = Date.now();
|
|
146
152
|
let [error, storyData] = await presult(
|
|
147
153
|
getStoryData({
|
|
148
154
|
story,
|
|
@@ -151,6 +157,7 @@ function makeRenderStories({
|
|
|
151
157
|
pageId,
|
|
152
158
|
}),
|
|
153
159
|
);
|
|
160
|
+
recordSnapshotTime(Date.now() - snapshotStart);
|
|
154
161
|
|
|
155
162
|
if (
|
|
156
163
|
error &&
|
|
@@ -202,6 +209,8 @@ function makeRenderStories({
|
|
|
202
209
|
} catch (ex) {
|
|
203
210
|
logger.log(`[page ${pageId}] error while processing story "${title}". ${ex}`);
|
|
204
211
|
return onDoneStory(ex, story);
|
|
212
|
+
} finally {
|
|
213
|
+
recordStoryTime(Date.now() - storyStart);
|
|
205
214
|
}
|
|
206
215
|
}
|
|
207
216
|
}
|
package/src/renderStory.js
CHANGED
|
@@ -14,6 +14,8 @@ function makeRenderStory({
|
|
|
14
14
|
serverSettings,
|
|
15
15
|
concurrency,
|
|
16
16
|
signal = new AbortController().signal,
|
|
17
|
+
recordOpenTime = () => {},
|
|
18
|
+
recordCheckAndCloseTime = () => {},
|
|
17
19
|
}) {
|
|
18
20
|
const throttle = throat(storyDataGap);
|
|
19
21
|
return function renderStory({story, snapshots, url}) {
|
|
@@ -26,7 +28,9 @@ function makeRenderStory({
|
|
|
26
28
|
const checkParams = storyToCheckSettings({story, url});
|
|
27
29
|
|
|
28
30
|
return timeItAsync(baselineName, async () => {
|
|
31
|
+
const openStart = Date.now();
|
|
29
32
|
const eyes = await openEyes({settings: openParams});
|
|
33
|
+
recordOpenTime(Date.now() - openStart);
|
|
30
34
|
return new Promise(async (resolve, reject) => {
|
|
31
35
|
if (signal.aborted) {
|
|
32
36
|
return await abortStory();
|
|
@@ -36,11 +40,13 @@ function makeRenderStory({
|
|
|
36
40
|
throttle(async () => {
|
|
37
41
|
try {
|
|
38
42
|
if (snapshots) {
|
|
43
|
+
const checkStart = Date.now();
|
|
39
44
|
await eyes.checkAndClose({
|
|
40
45
|
target: snapshots,
|
|
41
46
|
settings: checkParams,
|
|
42
47
|
});
|
|
43
48
|
const results = await eyes.getResults();
|
|
49
|
+
recordCheckAndCloseTime(Date.now() - checkStart);
|
|
44
50
|
resolve(results);
|
|
45
51
|
} else {
|
|
46
52
|
await eyes.abort({settings: {environments: checkParams.environments}});
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const utils = require('@applitools/utils');
|
|
4
|
+
|
|
5
|
+
// Collects per-execution metrics for the one-time summary logEvent (-> Coralogix).
|
|
6
|
+
//
|
|
7
|
+
// Everything here is ADDITIVE observability and best-effort: a failure to sample
|
|
8
|
+
// any metric is swallowed (logged at debug) and the run is unaffected. The memory
|
|
9
|
+
// sampler runs on an interval that is `.unref()`'d so it can NEVER keep the
|
|
10
|
+
// process alive past the end of a run.
|
|
11
|
+
|
|
12
|
+
const SAMPLE_INTERVAL_MS = 2000;
|
|
13
|
+
|
|
14
|
+
function makeStats() {
|
|
15
|
+
const samples = [];
|
|
16
|
+
return {
|
|
17
|
+
add(n) {
|
|
18
|
+
if (typeof n === 'number' && Number.isFinite(n)) samples.push(n);
|
|
19
|
+
},
|
|
20
|
+
summary() {
|
|
21
|
+
if (samples.length === 0) return null;
|
|
22
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
23
|
+
const n = sorted.length;
|
|
24
|
+
const sum = sorted.reduce((a, b) => a + b, 0);
|
|
25
|
+
const mid = Math.floor(n / 2);
|
|
26
|
+
return {
|
|
27
|
+
min: sorted[0],
|
|
28
|
+
max: sorted[n - 1],
|
|
29
|
+
avg: Math.round(sum / n),
|
|
30
|
+
median: n % 2 ? sorted[mid] : Math.round((sorted[mid - 1] + sorted[mid]) / 2),
|
|
31
|
+
p95: sorted[Math.ceil(n * 0.95) - 1],
|
|
32
|
+
samples: n,
|
|
33
|
+
};
|
|
34
|
+
},
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* @param {object} options
|
|
40
|
+
* @param {import('puppeteer').Browser} [options.browser]
|
|
41
|
+
* @param {{log: Function}} options.logger
|
|
42
|
+
*/
|
|
43
|
+
function makeRunMetrics({browser, logger} = {}) {
|
|
44
|
+
const nodeRss = makeStats();
|
|
45
|
+
const nodeHeapUsed = makeStats();
|
|
46
|
+
const browserTreeRss = makeStats();
|
|
47
|
+
const perStoryTotalMs = makeStats();
|
|
48
|
+
const snapshotMs = makeStats();
|
|
49
|
+
const openMs = makeStats();
|
|
50
|
+
const checkAndCloseMs = makeStats();
|
|
51
|
+
const counters = {tabsCrashed: 0, tabsReplaced: 0};
|
|
52
|
+
|
|
53
|
+
let timer;
|
|
54
|
+
let sampling = false;
|
|
55
|
+
let browserPid;
|
|
56
|
+
let started = false;
|
|
57
|
+
|
|
58
|
+
function start() {
|
|
59
|
+
if (started) return;
|
|
60
|
+
started = true;
|
|
61
|
+
try {
|
|
62
|
+
browserPid = browser?.process?.()?.pid;
|
|
63
|
+
} catch (err) {
|
|
64
|
+
logger?.log(`[metrics] could not resolve browser pid: ${err?.message}`);
|
|
65
|
+
}
|
|
66
|
+
// immediate first sample, then on interval
|
|
67
|
+
void sampleOnce();
|
|
68
|
+
timer = setInterval(() => void sampleOnce(), SAMPLE_INTERVAL_MS);
|
|
69
|
+
// never let the metrics sampler keep the process alive
|
|
70
|
+
timer.unref?.();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
async function sampleOnce() {
|
|
74
|
+
// node memory — synchronous, always cheap, always safe
|
|
75
|
+
try {
|
|
76
|
+
const mem = process.memoryUsage();
|
|
77
|
+
nodeRss.add(mem.rss);
|
|
78
|
+
nodeHeapUsed.add(mem.heapUsed);
|
|
79
|
+
} catch (err) {
|
|
80
|
+
logger?.log(`[metrics] node memory sample failed: ${err?.message}`);
|
|
81
|
+
}
|
|
82
|
+
// browser process-tree memory — async, may be unavailable; skip if a previous
|
|
83
|
+
// browser sample is still in flight to avoid pile-up on slow platforms.
|
|
84
|
+
if (sampling || !browserPid) return;
|
|
85
|
+
sampling = true;
|
|
86
|
+
try {
|
|
87
|
+
const bytes = await utils.process.sampleProcessTreeMemory(browserPid, {logger});
|
|
88
|
+
if (typeof bytes === 'number') browserTreeRss.add(bytes);
|
|
89
|
+
} catch (err) {
|
|
90
|
+
logger?.log(`[metrics] browser memory sample failed: ${err?.message}`);
|
|
91
|
+
} finally {
|
|
92
|
+
sampling = false;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function recordStoryTime(ms) {
|
|
97
|
+
perStoryTotalMs.add(ms);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function recordSnapshotTime(ms) {
|
|
101
|
+
snapshotMs.add(ms);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function recordOpenTime(ms) {
|
|
105
|
+
openMs.add(ms);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function recordCheckAndCloseTime(ms) {
|
|
109
|
+
checkAndCloseMs.add(ms);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function incrementTabsCrashed() {
|
|
113
|
+
counters.tabsCrashed += 1;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function incrementTabsReplaced() {
|
|
117
|
+
counters.tabsReplaced += 1;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Stop sampling and take one final sample. Idempotent.
|
|
121
|
+
async function stop() {
|
|
122
|
+
if (timer) {
|
|
123
|
+
clearInterval(timer);
|
|
124
|
+
timer = undefined;
|
|
125
|
+
}
|
|
126
|
+
await sampleOnce();
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function summarize() {
|
|
130
|
+
return {
|
|
131
|
+
memory: {
|
|
132
|
+
nodeRssBytes: nodeRss.summary(),
|
|
133
|
+
nodeHeapUsedBytes: nodeHeapUsed.summary(),
|
|
134
|
+
browserTreeRssBytes: browserTreeRss.summary(),
|
|
135
|
+
},
|
|
136
|
+
timing: {
|
|
137
|
+
perStoryTotalMs: perStoryTotalMs.summary(),
|
|
138
|
+
snapshotMs: snapshotMs.summary(),
|
|
139
|
+
openMs: openMs.summary(),
|
|
140
|
+
checkAndCloseMs: checkAndCloseMs.summary(),
|
|
141
|
+
},
|
|
142
|
+
tabs: {...counters},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return {
|
|
147
|
+
start,
|
|
148
|
+
stop,
|
|
149
|
+
recordStoryTime,
|
|
150
|
+
recordSnapshotTime,
|
|
151
|
+
recordOpenTime,
|
|
152
|
+
recordCheckAndCloseTime,
|
|
153
|
+
incrementTabsCrashed,
|
|
154
|
+
incrementTabsReplaced,
|
|
155
|
+
summarize,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
module.exports = {makeRunMetrics, makeStats};
|