@sdods/cli 0.7.2 → 0.9.0

package/dist/commands/run.js

@@ -3,13 +3,13 @@ import { isAbsolute, join, relative, resolve as resolvePath } from 'node:path';
 import { execa } from 'execa';
 import pc from 'picocolors';
 import { RecordingModeSchema, newRunId, runFiles, } from '@sdods/contracts';
-import { SdodsError, CLI_OVERRIDES_ENV, VERSION, formatFindings, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
+import { SdodsError, CLI_OVERRIDES_ENV, VERSION, checkPoolCapacity, collectGateEvidence, effectiveWorkers, evaluateGates, formatFindings, hasGates, lintProject, listGeneratedProjects, moduleByName, moduleDir, normalizeTagExpr, parseTagExpr, poolAccountsFor, poolDemandByRole, poolTooSmallError, redactRunTraces, serializeCliOverrides, setupTierOf, } from '@sdods/core';
 import { analyzeChangeImpact } from '@sdods/mcp';
 import { createContext } from '../context.js';
 import { browserStatuses } from './browsers.js';
 import { maybeNotify, notifyRun } from '../notify.js';
 import { installedPlaywrightVersion, stepResultsWarning } from '../runner-compat.js';
-import { collect, json, out, parseIntFlag, table, warn } from '../ui.js';
+import { collect, json, out, parseIntFlag, renderError, table, warn } from '../ui.js';
 const RECORDING_MODES = RecordingModeSchema.options.join(' | ');
 /** `--trace`/`--video` take Playwright's modes; refuse anything else before specs are generated. */
 function recordingMode(flag, value) {
@@ -37,6 +37,7 @@ function addRunOptions(cmd) {
         .option('--device <name>', 'device name for mobile emulation, e.g. "iPhone 15"')
         .option('--headed', 'run headed')
         .option('-w, --workers <n>', 'parallel workers', parseIntFlag('workers'))
+        .option('--allow-pool-contention', 'run even when a @user: role has fewer pool accounts than the workers that need it')
         .option('--shard <i/n>', 'shard, e.g. 1/3')
         .option('--retries <n>', 'retries per test', parseIntFlag('retries'))
         .option('--trace <mode>', `Playwright trace: ${RECORDING_MODES} (default: evidence.trace)`)
@@ -54,7 +55,7 @@ function addRunOptions(cmd) {
         .option('--har-update', 'record/refresh HAR files for @har scenarios')
         .option('--har-replay', 'replay HAR files for @har scenarios')
         .option('--strict', 'with --har-replay: abort on any request not in the HAR (offline)')
-        .option('--update-snapshots', 'update visual baselines')
+        .option('--update-snapshots', 'overwrite visual baselines without review (prefer `sdods baselines diff` / `accept`)')
         .option('--ui', 'interactive UI mode')
         .option('--debug', 'step debugger')
         .option('--list', 'list the run targets and tests that would run')
@@ -150,6 +151,9 @@ export async function runCommand(flags, cmd) {
     if (stepResults)
         warn(stepResults);
     const runId = flags.runId ?? newRunId();
+    if (flags.updateSnapshots)
+        warn('--update-snapshots overwrites every compared baseline without showing the diff. ' +
+            `Prefer a normal run, then \`sdods baselines diff --run ${runId}\` and \`sdods baselines accept <name> --run ${runId}\`.`);
     const cli = {
         env: envName,
         headed: flags.headed,
@@ -206,6 +210,59 @@ export async function runCommand(flags, cmd) {
         if (result.warnings.length && ctx.opts.verbose)
             process.stderr.write(formatFindings(result) + '\n');
     }
+    // --since selects features by git range; resolved here so the pool check counts the same selection.
+    const impact = flags.since
+        ? analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since)
+        : undefined;
+    // #153: refuse, before generating anything, a run whose user pool cannot cover its workers.
+    // Listing runs nothing, and --ui / --debug run one scenario at a time by hand.
+    if (!flags.list && !flags.ui && !flags.debug) {
+        const poolCfg = cfg.project.data.userPool;
+        const accounts = poolCfg && poolCfg.mode !== 'shared' && poolCfg.leaseScope !== 'scenario'
+            ? poolAccountsFor(cfg)
+            : undefined;
+        if (accounts) {
+            const moduleRels = modules.map((m) => relative(cfg.project.root, moduleDir(cfg.project.root, moduleByName(projectCfg, m)))
+                .replace(/\\/g, '/')
+                .concat('/'));
+            const impactedRels = impact ? new Set(impact.impacted.map((r) => r.feature)) : undefined;
+            const grep = flags.grep ? safeRegExp(flags.grep) : undefined;
+            const features = cfg.env.vars?.features;
+            const demand = poolDemandByRole(cfg.project, cfg.env.name, {
+                layers: layers.length ? layers : projectCfg.layers,
+                browsers: browsers.length ? browsers : projectCfg.browsers,
+                tags,
+                setupTags: setup?.tags,
+                flags: typeof features === 'string'
+                    ? features
+                        .split(',')
+                        .map((f) => f.trim())
+                        .filter(Boolean)
+                    : undefined,
+                quarantine: process.env.SDODS_QUARANTINE === 'run' ? 'run' : 'skip',
+                fullyParallel: proc?.fullyParallel ?? projectCfg.fullyParallel,
+                includeFeature: (rel) => (!featureRel || rel === featureRel) &&
+                    (!moduleRels.length || moduleRels.some((m) => rel.startsWith(m))) &&
+                    (!impactedRels || impactedRels.has(rel)),
+                includeScenario: (name) => (!flags.scenario || name.includes(flags.scenario)) &&
+                    (!flags.grep || (grep?.test(name) ?? false)),
+                repeatEach: flags.repeatEach,
+                shardTotal: cfg.runtime.shard?.total,
+            });
+            const shortfalls = checkPoolCapacity({
+                pool: poolCfg,
+                accounts,
+                demand,
+                workers: effectiveWorkers(cfg.runtime.workers),
+            });
+            if (shortfalls.length) {
+                const error = poolTooSmallError(cfg, shortfalls, cfg.runtime.workers);
+                if (!flags.allowPoolContention)
+                    throw error;
+                warn(`${error.message} Running anyway (--allow-pool-contention).`);
+            }
+        }
+    }
     const selection = {
         project: entry.slug,
         env: cfg.env.name,
@@ -318,8 +375,7 @@ export async function runCommand(flags, cmd) {
     // `analyzeChangeImpact` has existed for a while and was reachable only
     // through MCP, so it could advise a human and could not select a run. This is
     // the seam that was missing; the mapping itself is unchanged.
-    if (flags.since) {
-        const impact = analyzeChangeImpact(cfg.project.root, ctx.rootDir, flags.since);
+    if (flags.since && impact) {
         if (impact.impacted.length === 0) {
             // Deliberately NOT "run everything" and deliberately not a silent empty
             // run. A run that registered zero scenarios exits 0 and looks identical
@@ -389,6 +445,29 @@ export async function runCommand(flags, cmd) {
             warn(`trace redaction failed for ${relative(ctx.rootDir, e.file)}: ${e.error}`);
     }
     const summary = readSummary(runDir, runId, exitCode, Date.now() - started);
+    // Process gates are judged here, from the run that just finished, before anything publishes it.
+    // One shard of several holds only part of the run, so its pass rate and its a11y/perf evidence
+    // are not the run's; a sharded process is gated where the shards are merged, not per shard.
+    const shardTotal = cfg.runtime.shard?.total ?? 1;
+    let gates;
+    if (proc && hasGates(proc.gates) && exitCode !== 130) {
+        if (shardTotal > 1) {
+            warn(`process "${proc.name}" gates are not evaluated on shard ${cfg.runtime.shard?.current}/${shardTotal}: one shard is not the whole run.`);
+        }
+        else {
+            gates = evaluateGates({
+                process: proc.name,
+                gates: proc.gates,
+                totals: summary?.totals,
+                evidence: collectGateEvidence(runDir),
+            });
+            writeFileSync(join(runDir, runFiles.gates), JSON.stringify(gates, null, 2));
+            if (summary)
+                summary.gates = gates;
+            manifest.gatesPassed = gates.passed;
+            writeManifest();
+        }
+    }
     if (summary)
         writeFileSync(join(runDir, runFiles.summary), JSON.stringify(summary, null, 2));
     const dbConfigured = process.env.DB_DRIVER === 'postgres'
@@ -428,6 +507,13 @@ export async function runCommand(flags, cmd) {
             writeManifest();
         }
     }
+    // A breached gate fails a run whose scenarios passed. Exit 1, as the exit-code reference says:
+    // a gate not met is a failure, not a configuration error.
+    if (gates && !gates.passed && finalExit === 0) {
+        finalExit = 1;
+        manifest.exitCode = finalExit;
+        writeManifest();
+    }
     // Enabled integrations (check runs, PR comment, issues) act on the run here, the same way
     // `sdods integrations notify --run-id` does. Failures are reported and never change the exit code.
     const notify = await maybeNotify({
@@ -461,9 +547,20 @@ export async function runCommand(flags, cmd) {
         out(pc.dim(`integrations: not notified (${notify.reason})`));
     }
     if (ctx.opts.json) {
-        json({ runId, runDir, exitCode: finalExit, summary, manifest, notify });
+        json({ runId, runDir, exitCode: finalExit, summary, manifest, notify, gates });
     }
     else {
+        if (gates) {
+            out('');
+            out(pc.bold(`gates · process ${gates.process}`));
+            table(gates.rows.map((r) => ({
+                gate: r.gate,
+                threshold: r.threshold,
+                actual: r.actual,
+                result: r.passed ? 'pass' : 'FAIL',
+                detail: r.detail ?? '',
+            })));
+        }
         const t = summary?.totals;
         out('');
         out(`${finalExit === 0 ? pc.green('✔ passed') : finalExit === 130 ? pc.yellow('■ cancelled') : finalExit === 2 ? pc.yellow('■ nothing ran') : pc.red('✖ failed')}  ${t ? `${t.passed} passed, ${t.failed} failed, ${t.skipped} skipped, ${t.flaky} flaky` : ''}  ${pc.dim(`(${Math.round((Date.now() - started) / 1000)}s)`)}`);
@@ -471,6 +568,13 @@ export async function runCommand(flags, cmd) {
         out(pc.dim(`html report: ${join(runDir, runFiles.htmlReport, 'index.html')}`));
         out(pc.dim(`dashboard:   ${join(runDir, runFiles.dashboard, 'index.html')}`));
     }
+    if (gates && !gates.passed) {
+        const failed = gates.rows.filter((r) => !r.passed);
+        renderError(new SdodsError('GATE_FAILED', `Process "${gates.process}" did not meet ${failed.length} gate(s): ${failed.map((r) => `${r.gate} (${r.actual}, needs ${r.threshold})`).join('; ')}.`, {
+            hint: `The verdict is in ${join(runDir, runFiles.gates)}. Gates are set under processes[].gates in sdods.project.yaml.`,
+            docsPath: '/docs/guides/processes-and-testing-types',
+        }), Boolean(ctx.opts.json));
+    }
     return finalExit;
 }
 /**
@@ -496,6 +600,15 @@ function projectFeaturePath(projectRoot, repoRoot, input) {
         });
     return rel;
 }
+/** `--grep` as the pool check reads it; a pattern Playwright would reject matches no scenario. */
+function safeRegExp(pattern) {
+    try {
+        return new RegExp(pattern);
+    }
+    catch {
+        return undefined;
+    }
+}
 function escapeRe(s) {
     return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
 }

package/dist/program.js

@@ -24,6 +24,7 @@ import { register as registerCoverage } from './commands/coverage.js';
 import { register as registerBrowsers } from './commands/browsers.js';
 import { register as registerWatch } from './commands/watch.js';
 import { register as registerTrace } from './commands/trace.js';
+import { register as registerBaselines } from './commands/baselines.js';
 import { register as registerAgent } from './commands/agent.js';
 import { register as registerProposals } from './commands/proposals.js';
 import { register as registerMcp } from './commands/mcp.js';
@@ -36,6 +37,8 @@ import { register as registerIntegrations } from './commands/integrations.js';
 import { register as registerCompletion } from './commands/completion.js';
 import { register as registerUpgrade } from './commands/upgrade.js';
 import { register as registerFeedback } from './commands/feedback.js';
+import { register as registerLoad } from './commands/load.js';
+import { register as registerMatrix } from './commands/matrix.js';
 /**
  * All commands are registered eagerly (cheap) but each module keeps heavy imports
  * (server, agents, DB, browser runner) inside its action via dynamic import.
@@ -78,6 +81,7 @@ export function buildProgram() {
     registerBrowsers(program);
     registerWatch(program);
     registerTrace(program);
+    registerBaselines(program);
     registerAgent(program);
     registerProposals(program);
     registerMcp(program);
@@ -90,6 +94,8 @@ export function buildProgram() {
     registerCompletion(program);
     registerUpgrade(program);
     registerFeedback(program);
+    registerLoad(program);
+    registerMatrix(program);
     return program;
 }
 //# sourceMappingURL=program.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sdods/cli",
-  "version": "0.7.2",
+  "version": "0.9.0",
   "description": "SDODS command line: run, record, analyze, agents, MCP server, scheduler and the web server.",
   "license": "Apache-2.0",
   "author": "SDODS <admin@sdods.com>",
@@ -24,8 +24,8 @@
     "node": ">=22"
   },
   "dependencies": {
-    "@sdods/contracts": "0.7.2",
-    "@sdods/core": "0.7.2",
+    "@sdods/contracts": "0.9.0",
+    "@sdods/core": "0.9.0",
     "commander": "^15.0.0",
     "execa": "^10.0.1",
     "picocolors": "^1.1.1",
@@ -34,12 +34,12 @@
     "zod": "^4.5.4",
     "@playwright/test": "^1.63.0",
     "playwright-core": "^1.63.0",
-    "@sdods/integrations": "0.7.2",
-    "@sdods/mcp": "0.7.2",
-    "@sdods/agents": "0.7.2",
-    "@sdods/db": "0.7.2",
+    "@sdods/integrations": "0.9.0",
+    "@sdods/mcp": "0.9.0",
+    "@sdods/agents": "0.9.0",
+    "@sdods/db": "0.9.0",
     "csv-parse": "^7.0.2",
-    "@sdods/server": "0.7.2"
+    "@sdods/server": "0.9.0"
   },
   "homepage": "https://sdods.com",
   "bugs": {

package/skills/sdods/SKILL.md

@@ -18,7 +18,7 @@ over shelling out.
 
 - Every scenario carries exactly one layer tag (`@ui`, `@api`, `@hybrid`) and exactly one suite tag
   (`@smoke`, `@regression`, `@sanity`). Optional: `@visual @a11y @perf @mock @data-driven @pool`,
-  and value tags `@user:<role> @data:<dataset> @har:<name> @jira:KEY @skip:<browser>`.
+  and value tags `@user:<role> @data:<dataset> @har:<name> @jira:KEY @skip:<browser> @matrix:<name> @req:<id>`, and per-scenario emulation `@locale:<bcp47> @theme:<light|dark|no-preference> @timezone:<IANA> @viewport:<W>x<H> @device:<name>`.
 - Reuse existing steps first (`sdods steps list -p <slug>`, or `step_find`). Add a step only when no
   phrasing fits, and name it in the same style.
 - Locator priority: role + name > label > test id > placeholder > text > CSS. Never XPath.
@@ -37,6 +37,7 @@ over shelling out.
 | Projects and hierarchy | `sdods project list`, `sdods workspace tree` |
 | Resolved config, with where each value came from | `sdods config show -p <slug> -e <env> --explain` |
 | Validate features | `sdods lint -p <slug>` |
+| Role × surface matrix (`roles.matrix.yaml`) | `sdods matrix expand -p <slug>` · `--check` in CI |
 | Run a slice | `sdods run -p <slug> -e <env> -l api` · `-l ui -b chromium -t @smoke` · `--process pr-check` |
 | Results | `sdods report --last`, `sdods heal report --last`, `sdods trace --last` |
 | Available steps | `sdods steps list -p <slug>` |

package/templates/projects/demo-shop/features/auth/user-pool.feature

@@ -1,6 +1,6 @@
 @ui @auth @pool
 Feature: User pool
-  Accounts are leased per worker from the pool dataset; login state is cached as storageState.
+  Accounts are leased per scenario from the pool dataset; login state is cached as storageState.
 
   @sanity @user:standard
   Scenario: A leased standard user starts logged in through cached storage state

package/templates/projects/demo-shop/features/cart/performance.feature

@@ -0,0 +1,14 @@
+@ui @cart @perf
+Feature: Cart performance
+  The @perf tag records the vitals of every document the scenario loads and judges them against
+  perf.budgets (pageLoadMs and lcpMs from the project, pageLoadMs overridden in envs/staging.yaml).
+
+  # WebKit does not implement largest-contentful-paint, and a vital a budget applies to that never
+  # fired fails the scenario rather than passing it, so this runs on Chromium and Firefox only.
+  @regression @skip:webkit
+  Scenario: The login and cart pages load within their budgets
+    Given I am on the login page
+    When I login with "standard_user" and "{{standardPassword}}"
+    And I add "{{defaultProduct}}" to the cart
+    And I navigate to the "cart" page
+    Then the cart should list "{{defaultProduct}}"

package/templates/projects/demo-shop/features/inventory/accessibility.feature

@@ -0,0 +1,12 @@
+@ui @inventory @a11y
+Feature: Inventory accessibility
+  The @a11y tag audits the page each scenario ends on with axe (WCAG 2.x A and AA) and fails on
+  anything at or above a11y.failOn (serious by default). SauceDemo's login, inventory and cart
+  pages have no WCAG A/AA violation axe can detect, so nothing is scoped out with include/exclude.
+
+  @regression
+  Scenario: The inventory page has no serious accessibility violations
+    Given I am on the login page
+    When I login with "standard_user" and "{{standardPassword}}"
+    Then I should be on the inventory page
+    And there should be 6 products listed

package/templates/projects/demo-shop/features/inventory/visual.feature

@@ -1,6 +1,7 @@
 @ui @inventory @visual
 Feature: Inventory visual baseline
-  Pixel comparison against a per-browser baseline (run once with --update-snapshots).
+  Pixel comparison against a per-browser, per-platform baseline. Review a failure with
+  `sdods baselines diff` and accept it with `sdods baselines accept inventory --run <id>`.
 
   @regression
   Scenario: Inventory page matches its baseline

package/templates/projects/demo-shop/load/posts.yaml

@@ -0,0 +1,17 @@
+# Read-only load profile for `sdods load`. The staging environment points at a public sandbox
+# (JSONPlaceholder) and deliberately does NOT opt in to load, so only `--dry-run` works against it.
+description: Read posts and one post's comments
+thinkTimeSeconds: 1
+vus: 2
+duration: 20s
+thresholds:
+  http_req_duration: ['p(95)<1500']
+  http_req_failed: ['rate<0.01']
+  checks: ['rate>0.99']
+requests:
+  - name: list posts
+    path: /posts
+    expect: { status: 200, bodyContains: userId }
+  - name: post comments
+    path: /posts/1/comments
+    expect: { status: 200 }

package/templates/projects/demo-shop/sdods.project.yaml

@@ -101,6 +101,9 @@ data:
     roleColumn: role
     leaseStore: file
     leaseTtlMs: 600000
+    # One account per role after poolSize, and `release:check:live` runs three browsers in one
+    # invocation: release each account when its scenario ends so the browsers queue for it (#153).
+    leaseScope: scenario
 
 auth:
   strategy: form