@camunda/e2e-test-suite 0.0.1129 → 0.0.1131

package/README.md

@@ -419,22 +419,22 @@ See [docs/ci-failure-categories.md](docs/ci-failure-categories.md) for the full
 
 When a PR opens in [camunda/camunda](https://github.com/camunda/camunda) or [camunda/connectors](https://github.com/camunda/connectors), AlwaysGreen dispatches a subset of this suite against that PR's build. This is separate from this repo's own [Automatic test runs on PR](#automatic-test-runs-on-pr) above, which cover changes to the test suite itself rather than to the product.
 
-| leg | scope | blocking | runtime |
-| --- | --- | --- | --- |
-| SaaS smoke | `test-setup`, `navigation`, `smoke-tests` | yes | 9–12 min |
-| SaaS full | remaining specs (minus exclusions below) | no, until Nov 2026 | ~40–48 min |
-| SM smoke | smoke suite | yes | — |
-| SM full | full suite | no | — |
+| leg | scope |
+| --- | --- |
+| SaaS smoke | `test-setup`, `navigation`, `smoke-tests` |
+| SaaS full | remaining specs (minus exclusions below) |
+| SM smoke | smoke suite |
+| SM full | full suite |
 
-**Excluded from PR level entirely — nightly only.** Third-party model-endpoint specs bill an external provider on every run they're included in, so they stay out of the PR-blocking path:
+Whether a given leg gates a merge is decided by the caller repo's own branch protection / merge-queue configuration, not by this suite — that can change independently of anything documented here, so it's deliberately not claimed above. Runtime is likewise omitted: it shifts as specs are added, excluded, or reworked, and a stale number here is worse than none.
+
+**Excluded from PR level entirely — nightly only.** Third-party model-endpoint specs bill an external provider on every run they're included in, so they stay out of the PR-level full-suite pass:
 
 - By spec file: `agentic-ai-user-flows.spec.ts`, `idp-user-flows.spec.ts`, `aws-connectors-user-flows.spec.ts` (AWS Bedrock, AgentCore, Vertex, Azure OpenAI)
-- By tag `@ai-external`: the AI-generated-form tests inside `web-modeler-user-flows.spec.ts` (`Form.js Integration with User Task and AI Generated Form`, `Form.js Integration with AI generated Start form`)
+- By tag `@ai-external`: both `Form.js Integration with AI generated Start form` and `Form.js Integration with User Task and AI Generated Form` inside `web-modeler-user-flows.spec.ts` -- both call `formJsPage.generateAIForm()` (a paid model call), so both are excluded.
 
 **Kept in at PR level despite touching external systems:** Connector Secrets tests (reads from our own AWS/GCP secret managers — no model endpoint, no meaningful per-run cost) and MCP tests (run against our own cluster — no cost).
 
-See [AlwaysGreen Iteration 2 - Alpha 5: Coverage & Optimisation](https://github.com/camunda/team-test-automation/issues/78) for the full rationale and current SaaS/SM stability data behind these boundaries.
-
 ## Test Suite Structure
 
 The test suite is organized using the Page Object Model design pattern, where each test file contains test scenarios related to a specific component or feature of the Camunda C8 product stack. Additionally, it leverages separate page objects for each web page or component being tested.

package/dist/tests/8.10/test-setup.spec.js

@@ -1,6 +1,10 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const test_1 = require("@playwright/test");
+const fs_1 = __importDefault(require("fs"));
 const _8_10_1 = require("../../fixtures/8.10");
 const _setup_1 = require("../../test-setup.js");
 const connectorSecrets_1 = require("../../utils/connectorSecrets");
@@ -38,26 +42,49 @@ _8_10_1.test.describe('Cluster Setup Tests', () => {
         }
         await clusterPage.clickClusterLink(clusterName);
         await clusterDetailsPage.assertComponentsHealth();
-        if (process.env.IS_AG !== 'true') {
+        // API-client creation is needed whenever this run will also exercise the
+        // full-suite pass: many of those specs call the Zeebe REST API directly
+        // and need CAMUNDA_OPTIMIZE_BASE_URL for Optimize session login, even
+        // though IS_AG=true skips this for the usual smoke-only case.
+        if (process.env.IS_AG !== 'true' || process.env.RUN_FULL_SUITE === 'true') {
             await clusterDetailsPage.clickAPITab();
-            await clusterDetailsPage.createAPIClientAndReturnVariables(apiClientName);
+            const zeebeCredentials = await clusterDetailsPage.createAPIClientAndReturnVariables(apiClientName);
             await clusterDetailsPage.clickCloseModalButton();
-            await clusterPage.clickConnectorSecretsTab(clusterName);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.awsSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.openAiSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.gcpSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.azureSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.connectorSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.bedrockKbSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.agentcoreSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.agentcoreLtmSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.bedrockCodeInterpreterSecretsValues);
-            await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.anthropicBedrockMantleSecretsValues);
+            if (process.env.RUN_FULL_SUITE === 'true') {
+                // Written to disk, not just process.env: the full-suite pass runs as
+                // a separate `npm run test` process (see run-tests-smoke-int's
+                // action.yml) that does not inherit this process's env mutations.
+                // mode 0o600: this file holds a live Zeebe API client secret.
+                fs_1.default.writeFileSync('cluster-credentials.json', JSON.stringify(zeebeCredentials), { mode: 0o600 });
+            }
+            // Deliberately NOT extended to RUN_FULL_SUITE like the API client
+            // above: clickConnectorSecretsTab hits the same pre-existing "click
+            // after 3 attempts" UI-contention flakiness seen throughout this
+            // suite, and here it's on test-setup's blocking critical path -- a
+            // transient failure takes down cluster provisioning entirely (and
+            // with it, both smoke and the full-suite pass), confirmed on a real
+            // run. connectors-user-flows.spec.ts's "Connector Secrets User Flow"
+            // tests keep failing in the full-suite pass without this, but that's
+            // a bounded, non-blocking loss -- not worth risking the whole run for.
+            if (process.env.IS_AG !== 'true') {
+                await clusterPage.clickConnectorSecretsTab(clusterName);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.awsSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.openAiSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.gcpSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.azureSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.connectorSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.bedrockKbSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.agentcoreSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.agentcoreLtmSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.bedrockCodeInterpreterSecretsValues);
+                await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.anthropicBedrockMantleSecretsValues);
+            }
         }
     });
     (0, _8_10_1.test)('Create Agentic Orchestration Cluster', async ({ page, loginPage, homePage, clusterPage, clusterDetailsPage, clusterSecretsPage, }, testInfo) => {
         _8_10_1.test.skip(process.env.IS_PROD === 'true', 'Skipping test because not required on PROD test org');
-        _8_10_1.test.skip(process.env.IS_AG === 'true', 'Skipping test because not required when IS_AG is true');
+        _8_10_1.test.skip(process.env.IS_AG === 'true' && process.env.RUN_FULL_SUITE !== 'true', 'Skipping test because not required when IS_AG is true -- unless ' +
+            'RUN_FULL_SUITE also needs this cluster for the MCP suite.');
         _8_10_1.test.slow();
         const clusterName = 'Agentic Orchestration Cluster';
         const apiClientName = 'Test_API_Client' + (await (0, _setup_1.generateRandomStringAsync)(3));
@@ -82,7 +109,14 @@ _8_10_1.test.describe('Cluster Setup Tests', () => {
         await clusterPage.assertClusterHealthyStatusWithRetry(clusterName);
         await clusterPage.clickClusterLink(clusterName);
         await clusterDetailsPage.clickAPITab();
-        await clusterDetailsPage.createAPIClientAndReturnVariables(apiClientName, true, 'agentic_cluster');
+        const agenticCredentials = await clusterDetailsPage.createAPIClientAndReturnVariables(apiClientName, true, 'agentic_cluster');
+        if (process.env.RUN_FULL_SUITE === 'true') {
+            // Written to disk, not just process.env: the full-suite pass runs as
+            // a separate `npm run test` process (see run-tests-smoke-int's
+            // action.yml) that does not inherit this process's env mutations.
+            // mode 0o600: this file holds a live Zeebe API client secret.
+            fs_1.default.writeFileSync('agentic-cluster-credentials.json', JSON.stringify(agenticCredentials), { mode: 0o600 });
+        }
         await clusterDetailsPage.clickCloseModalButton();
         const mcpCredentials = await clusterDetailsPage.createMCPClientAndReturnVariables(mcpClientName);
         await clusterDetailsPage.clickCloseModalButton();
@@ -109,10 +143,22 @@ _8_10_1.test.describe('Cluster Setup Tests', () => {
             await clusterSecretsPage.createSetOfSecrets(clusterName, connectorSecrets_1.connectorSecretsValues);
         }
     });
+    // Stagger 5x wider than the other setup tests above, capped at 60s: this
+    // loop logs in getTestUsers().length users (up to 27, one per worker) at
+    // once, so at 1000ms/worker the whole burst lands inside a ~20-30s window
+    // and trips Auth0's rate limiter (AUTH0_RATE_LIMIT, e.g. run 34099726060's
+    // User 18 failure) well before loginWithRetry's own 60-90s backoff gets a
+    // chance to help. An uncapped 5000ms/worker fixed the rate limit (run
+    // 34103380836 had none) but then burned up to 90s of the highest-indexed
+    // worker's fixed 12-minute test timeout on pure stagger sleep before its
+    // first login attempt, leaving too little budget for the retry loop to
+    // recover from an unrelated slow-page timeout (also run 34103380836,
+    // User 18 again). The 60s cap keeps most of the wider spread while
+    // bounding worst-case pre-login wait to 1/12 of the test budget.
     for (const [index, user] of (0, users_1.getTestUsers)().entries()) {
         (0, _8_10_1.test)(`Create Project Folder for User ${index + 1}`, async ({ page, loginPage, homePage, appsPage, modelerHomePage, }, testInfo) => {
             await _8_10_1.test.step('Log in', async () => {
-                await (0, UtilitiesPage_1.loginWithRetry)(page, loginPage, user, (testInfo.workerIndex + 1) * 1000);
+                await (0, UtilitiesPage_1.loginWithRetry)(page, loginPage, user, Math.min((testInfo.workerIndex + 1) * 5000, 60000));
             });
             await _8_10_1.test.step('Navigate to Web Modeler', async () => {
                 await (0, test_1.expect)(homePage.camundaComponentsButton).toBeVisible({
@@ -121,8 +167,23 @@ _8_10_1.test.describe('Cluster Setup Tests', () => {
                 await appsPage.openModelerWithRetry(modelerHomePage.modelerPageBanner, 180000);
             });
             await _8_10_1.test.step('Create Cross Component Project Folder', async () => {
-                await (0, UtilitiesPage_1.withSessionRecovery)(page, loginPage, appsPage, modelerHomePage.modelerPageBanner, user, (testInfo.workerIndex + 1) * 1000, async () => {
+                await (0, UtilitiesPage_1.withSessionRecovery)(page, loginPage, appsPage, modelerHomePage.modelerPageBanner, user, Math.min((testInfo.workerIndex + 1) * 5000, 60000), async () => {
                     await homePage.clickSkipCustomization();
+                    // Reverted the "only User 1 creates" leader-election attempt: it made
+                    // things worse, not better. Run 34191243820 showed followers still
+                    // timing out waiting for the folder even well after User 1 (2.9min)
+                    // had already finished creating it -- not a wait-budget problem, since
+                    // widening it wouldn't fix a folder that isn't becoming visible to
+                    // other sessions at all. Nightly (run-tests-saas) and on-demand never
+                    // hit this because both run test-setup.spec.ts as part of one
+                    // unfiltered `--project=X` invocation together with hundreds of other
+                    // tests, rather than isolating it into its own "Provision cluster"
+                    // step the way this PR's action.yml does -- that isolation is what
+                    // creates the every-worker-races-at-once window in the first place.
+                    // Back to every user independently checking-and-creating: still races
+                    // sometimes (3-7 of 18 users per run, per the create-organization
+                    // rate-limit findings above), but that's a smaller, known failure mode
+                    // than the 100% follower failure the leader-election attempt caused.
                     await modelerHomePage.createCrossComponentProjectFolder();
                     await modelerHomePage.clickHomeBreadcrumb();
                     await (0, test_1.expect)(modelerHomePage.crossComponentProjectFolder).toBeVisible({ timeout: 60000 });

package/dist/tests/8.10/web-modeler-user-flows.spec.js

@@ -34,7 +34,7 @@ _8_10_1.test.describe('Web Modeler User Flow Tests', () => {
             await (0, mailSlurpClient_1.deleteInbox)(inboxes[testInfo.title]);
         }
     });
-    (0, _8_10_1.test)('Form.js Integration with User Task and AI Generated Form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_10_1.test)('Form.js Integration with User Task and AI Generated Form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_10_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'User_Task_Process_With_AI_Form' + randomString;
@@ -92,7 +92,7 @@ _8_10_1.test.describe('Web Modeler User Flow Tests', () => {
             await operateTabTaskDetailsPage.clickCompleteTaskButton();
         });
     });
-    (0, _8_10_1.test)('Form.js Integration with AI generated Start form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_10_1.test)('Form.js Integration with AI generated Start form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_10_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'AI-generated-Start-form' + randomString;

package/dist/tests/8.7/web-modeler-user-flows.spec.js

@@ -33,7 +33,7 @@ _8_7_1.test.describe('Web Modeler User Flow Tests', () => {
             await (0, mailSlurpClient_1.deleteInbox)(inboxes[testInfo.title]);
         }
     });
-    (0, _8_7_1.test)('Form.js Integration with User Task and AI Generated Form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_7_1.test)('Form.js Integration with User Task and AI Generated Form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_7_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'User_Task_Process_With_AI_Form' + randomString;
@@ -280,7 +280,7 @@ _8_7_1.test.describe('Web Modeler User Flow Tests', () => {
             });
         });
     });
-    (0, _8_7_1.test)('Form.js Integration with AI generated Start form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_7_1.test)('Form.js Integration with AI generated Start form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_7_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'AI-generated-Start-form' + randomString;

package/dist/tests/8.8/web-modeler-user-flows.spec.js

@@ -31,7 +31,7 @@ _8_8_1.test.describe('Web Modeler User Flow Tests', () => {
             await (0, mailSlurpClient_1.deleteInbox)(inboxes[testInfo.title]);
         }
     });
-    (0, _8_8_1.test)('Form.js Integration with User Task and AI Generated Form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_8_1.test)('Form.js Integration with User Task and AI Generated Form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_8_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'User_Task_Process_With_AI_Form' + randomString;
@@ -88,7 +88,7 @@ _8_8_1.test.describe('Web Modeler User Flow Tests', () => {
             await operateTabTaskDetailsPage.clickCompleteTaskButton();
         });
     });
-    (0, _8_8_1.test)('Form.js Integration with AI generated Start form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_8_1.test)('Form.js Integration with AI generated Start form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_8_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'AI-generated-Start-form' + randomString;

package/dist/tests/8.9/web-modeler-user-flows.spec.js

@@ -34,7 +34,7 @@ _8_9_1.test.describe('Web Modeler User Flow Tests', () => {
             await (0, mailSlurpClient_1.deleteInbox)(inboxes[testInfo.title]);
         }
     });
-    (0, _8_9_1.test)('Form.js Integration with User Task and AI Generated Form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_9_1.test)('Form.js Integration with User Task and AI Generated Form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_9_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'User_Task_Process_With_AI_Form' + randomString;
@@ -92,7 +92,7 @@ _8_9_1.test.describe('Web Modeler User Flow Tests', () => {
             await operateTabTaskDetailsPage.clickCompleteTaskButton();
         });
     });
-    (0, _8_9_1.test)('Form.js Integration with AI generated Start form', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
+    (0, _8_9_1.test)('Form.js Integration with AI generated Start form @ai-external', async ({ page, homePage, modelerHomePage, appsPage, modelerCreatePage, formJsPage, }) => {
         _8_9_1.test.slow();
         const randomString = await (0, _setup_1.generateRandomStringAsync)(3);
         const processName = 'AI-generated-Start-form' + randomString;

package/dist/utils/users.js

@@ -98,7 +98,15 @@ function getTestUser(userKey) {
 }
 exports.getTestUser = getTestUser;
 function getTestUsers() {
-    if (process.env.IS_AG === 'true' && process.env.MULTI_USERS === 'true') {
+    // test-setup.spec.ts loops over this list to provision one Cross Component
+    // Project folder per user. Full-suite specs each log in as a different
+    // named user (getTestUser('eleventhUser') etc.) via getTestUser, not one
+    // of the 3 smokeTestUsers -- so under IS_AG=true, RUN_FULL_SUITE=true still
+    // needs the full list, or every one of those users hits a folder that was
+    // never created for them.
+    if (process.env.IS_AG === 'true' &&
+        process.env.MULTI_USERS === 'true' &&
+        process.env.RUN_FULL_SUITE !== 'true') {
         return Object.values(exports.smokeTestUsers);
     }
     else if (process.env.MULTI_USERS === 'true') {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@camunda/e2e-test-suite",
-  "version": "0.0.1129",
+  "version": "0.0.1131",
   "description": "End-to-end test helpers for Camunda 8",
   "repository": {
     "type": "git",

package/playwright.config.ts

@@ -10,17 +10,23 @@ if (!process.env.CAMUNDA_OPTIMIZE_BASE_URL && process.env.BASE_URL) {
 
 const projectName = process.env.PROJECT! || 'Chromium'; // Default to 'default-project' if PROJECT is not set
 
+// Set only by the concurrent full-suite pass in run-tests-smoke-int's action.yml,
+// so it doesn't clobber the smoke pass's report files when both `npm run test`
+// invocations run at the same time against the same working directory.
+// Empty by default -- every other caller is unaffected.
+const reportSuffix = process.env.REPORT_SUFFIX || '';
+
 const junitOptions = {
   embedAnnotationsAsProperties: true,
-  outputFile: './test-results/junit-report.xml',
+  outputFile: `./test-results${reportSuffix}/junit-report.xml`,
 };
 
 // Define reporters without SlackReporter
 const useReportersWithoutSlack: any[] = [
   ['list'],
   ['junit', junitOptions],
-  ['html', {outputFolder: 'html-report'}],
-  ['json', {outputFile: `json-report/results.json`}],
+  ['html', {outputFolder: `html-report${reportSuffix}`}],
+  ['json', {outputFile: `json-report${reportSuffix}/results.json`}],
 ];
 
 // Define reporters with SlackReporter
@@ -256,6 +262,7 @@ export default defineConfig({
   workers: workerCount(),
   retries: 1,
   fullyParallel: false,
+  outputDir: `test-results${reportSuffix}`,
   use: {
     baseURL: getBaseURL(),
     actionTimeout: 10000,