codeceptjs 4.2.0-beta.4 → 4.2.1-beta.1

package/lib/helper/Decision.js

@@ -0,0 +1,256 @@
+import fs from 'fs'
+import path from 'path'
+
+import Helper from '@codeceptjs/helper'
+import Container from '../container.js'
+import Config from '../config.js'
+import store from '../store.js'
+import output from '../output.js'
+import AssertionFailedError from '../assert/error.js'
+import { compactAriaSnapshot } from '../aria.js'
+import { minifyHtml } from '../html.js'
+import { pickActingHelper } from '../utils/trace.js'
+import { DecisionAI, DecisionConnectionError } from '../ai.js'
+
+/**
+ * Asserts statements about the current page with a decision model.
+ *
+ * Decision models (like [Jev](https://openrouter.ai/typesafe/jev-1.13) or [Clef](https://openrouter.ai/cloudflare/clef))
+ * do not generate text. They read the page state and return the probability that a statement is true.
+ * A statement passes when its probability reaches the configured `confidence`.
+ *
+ * ```js
+ * I.decide('top level navigation is available');
+ * I.decide([
+ *   'checkout form has all required fields',
+ *   'success message is shown',
+ *   'submit button enabled',
+ *   'cancel button present',
+ * ]);
+ * I.decideVisually('sidebar is shown');
+ * ```
+ *
+ * The model receives the page URL, title and ARIA snapshot.
+ * Helpers without ARIA snapshots (anything but Playwright) send minified HTML instead.
+ * `decideVisually` also sends a screenshot, so it requires a model with image input.
+ *
+ * This helper must be enabled together with a web helper (Playwright, Puppeteer, WebDriver).
+ *
+ * ## Configuration
+ *
+ * Decision model is created with the OpenRouter provider from `@openrouter/ai-sdk-provider`
+ * and configured in the `ai.decisionModel` section of the config:
+ *
+ * ```js
+ * import { createOpenRouter } from '@openrouter/ai-sdk-provider'
+ *
+ * const openrouter = createOpenRouter({
+ *   apiKey: process.env.OPENROUTER_API_KEY,
+ * })
+ *
+ * export default {
+ *   ai: {
+ *     decisionModel: {
+ *       model: openrouter.evaluationModel('typesafe/jev-1.13'),
+ *       confidence: 0.8,
+ *     },
+ *   },
+ *   helpers: {
+ *     Playwright: { url: 'http://localhost' },
+ *     Decision: {},
+ *   },
+ * }
+ * ```
+ *
+ * * `model` - decision model used by `decide`.
+ * * `confidence` (default: `0.7`) - minimal probability, between 0 and 1, for a statement to pass.
+ * * `timeout` (default: `15000`) - request timeout in ms.
+ * * `maxLength` (default: `12000`) - maximal length of ARIA snapshot or HTML sent to the model.
+ *
+ * `decideVisually` uses a separate decision model with image input, configured as `visualModel`:
+ *
+ * ```js
+ * decisionModel: {
+ *   model: openrouter.evaluationModel('typesafe/jev-1.13'),
+ *   visualModel: openrouter.evaluationModel('cloudflare/clef'),
+ * },
+ * ```
+ *
+ * The helper has one option:
+ *
+ * * `mode` (default: `assert`) - how decisions are executed:
+ *   * `assert` - request the model and fail the step when a statement is not confirmed.
+ *   * `report` - request the model but never fail, even on API errors. Results and errors are added to the step as a comment.
+ *   * `skip` - do not request the model, every decision passes.
+ *
+ * ```js
+ * Decision: { mode: process.env.CI ? 'assert' : 'skip' }
+ * ```
+ *
+ * Decisions work without the `--ai` flag.
+ * Browse all decision models at [OpenRouter](https://openrouter.ai/models?output_modalities=decisions).
+ *
+ * ## Methods
+ */
+class Decision extends Helper {
+  constructor(config = {}) {
+    super(config)
+    const modes = ['assert', 'report', 'skip']
+    this.mode = config.mode || 'assert'
+    if (!modes.includes(this.mode)) throw new Error(`Unknown Decision helper mode "${this.mode}", use one of: ${modes.join(', ')}`)
+    this.decisionAI = new DecisionAI(Config.get('ai', {}).decisionModel)
+    this.options = this.decisionAI.config
+  }
+
+  get isAsserting() {
+    return this.mode === 'assert'
+  }
+
+  get isReporting() {
+    return this.mode === 'report'
+  }
+
+  get isSkipping() {
+    return this.mode === 'skip'
+  }
+
+  /**
+   * Asserts that a statement (or each of statements) is true for the current page.
+   * The decision model receives page URL, title and ARIA snapshot.
+   * Multiple statements are checked in a single request and all of them must pass.
+   *
+   * ```js
+   * I.decide('user is logged in');
+   * I.decide(['submit button enabled', 'cancel button present']);
+   * const probability = await I.decide('cart is empty');
+   * ```
+   *
+   * @param {string|string[]} statements statement or list of statements to verify.
+   * @returns {Promise<number|number[]|undefined>} probability of each statement, `undefined` in `skip` mode.
+   */
+  async decide(statements) {
+    if (this.isSkipping) return this._skip()
+    return this._run(async () => {
+      const state = await this._grabState()
+      return this._assert(statements, 'model', state)
+    })
+  }
+
+  /**
+   * Asserts that a statement (or each of statements) is true for the current page using a screenshot.
+   * The decision model receives page URL, title, ARIA snapshot and a screenshot.
+   * Requires `visualModel` with image input. Experimental.
+   *
+   * ```js
+   * I.decideVisually('sidebar is shown');
+   * I.decideVisually(['logo is in the header', 'page uses dark theme']);
+   * ```
+   *
+   * @param {string|string[]} statements statement or list of statements to verify.
+   * @returns {Promise<number|number[]|undefined>} probability of each statement, `undefined` in `skip` mode.
+   */
+  async decideVisually(statements) {
+    if (this.isSkipping) return this._skip()
+    return this._run(async () => {
+      const state = await this._grabState()
+      const screenshot = await this._grabScreenshot()
+      const content = [
+        { type: 'text', text: JSON.stringify(state) },
+        { type: 'image_url', image_url: { url: `data:image/png;base64,${screenshot}` } },
+      ]
+      return this._assert(statements, 'visualModel', content)
+    })
+  }
+
+  _skip() {
+    if (store.currentStep) store.currentStep.comment = ' (skipped)'
+  }
+
+  async _run(fn) {
+    try {
+      return await fn()
+    } catch (err) {
+      if (this.isReporting) {
+        if (store.currentStep) store.currentStep.comment = ` (error: ${err.message})`
+        output.say(`  ✖ ${err.message}`, 'yellow')
+        return
+      }
+      if (!(err instanceof DecisionConnectionError)) err.isTerminal = true
+      throw err
+    }
+  }
+
+  async _assert(statements, option, state) {
+    const list = [statements].flat()
+    if (!list.length) throw new Error('No statements to decide')
+
+    const probabilities = await this.decisionAI.decide(option, state, list)
+
+    const failed = []
+    const results = list.map((statement, i) => {
+      const passed = probabilities[i] >= this.options.confidence
+      const result = `${passed ? '✔' : '✖'} ${statement} (${formatProbability(probabilities[i])})`
+      this.debugSection('Decision', result)
+      if (!passed) failed.push(`"${statement}" (${formatProbability(probabilities[i])})`)
+      return result
+    })
+
+    if (this.isReporting) {
+      if (store.currentStep) store.currentStep.comment = `\n${results.join('\n')}`
+      results.forEach(result => output.say(`  ${result}`, result.startsWith('✔') ? 'green' : 'yellow'))
+      return Array.isArray(statements) ? probabilities : probabilities[0]
+    }
+
+    if (failed.length) {
+      const err = new AssertionFailedError(
+        { statements: failed.join(', '), confidence: formatProbability(this.options.confidence) },
+        'expected page to satisfy {{statements}} with confidence of {{confidence}}',
+      )
+      err.showDiff = false
+      err.message = err.cliMessage()
+      throw err
+    }
+
+    return Array.isArray(statements) ? probabilities : probabilities[0]
+  }
+
+  async _grabState() {
+    const helper = this._actingHelper()
+    const state = {
+      url: await helper.grabCurrentUrl(),
+      title: await helper.grabTitle(),
+    }
+
+    if (helper.grabAriaSnapshot) {
+      state.aria = compactAriaSnapshot(await helper.grabAriaSnapshot()).slice(0, this.options.maxLength)
+      return state
+    }
+
+    state.html = (await minifyHtml(await helper.grabSource())).slice(0, this.options.maxLength)
+    return state
+  }
+
+  async _grabScreenshot() {
+    const helper = this._actingHelper()
+    const file = path.join(store.outputDir, `decision_${Date.now()}.png`)
+    await helper.saveScreenshot(file)
+    if (!fs.existsSync(file)) throw new Error('Could not take a screenshot for visual decision')
+    try {
+      return fs.readFileSync(file).toString('base64')
+    } finally {
+      fs.rmSync(file, { force: true })
+    }
+  }
+
+  _actingHelper() {
+    const helper = pickActingHelper(Container.helpers())
+    if (!helper) throw new Error('Decision helper requires a web helper: Playwright, Puppeteer or WebDriver')
+    return helper
+  }
+}
+
+function formatProbability(probability) {
+  return `${Math.round(probability * 100)}%`
+}
+
+export default Decision

package/lib/helper/Kitesurf.js

@@ -4,8 +4,7 @@ import CDPBrowser from './CDPBrowser.js'
 /**
  * ## Configuration
  *
- * This helper should be configured in codecept.conf.js. It accepts everything `CDPBrowser`
- * accepts (see its config table), plus:
+ * This helper should be configured in codecept.conf.js
  *
  * @typedef KitesurfConfig
  * @type {object}
@@ -14,8 +13,8 @@ import CDPBrowser from './CDPBrowser.js'
  * @prop {string} [apiToken] - Cloudflare API token; defaults to CF_API_TOKEN env var
  * @prop {number} [keepAlive=240000] - session keep-alive time in milliseconds
  * @prop {string} [apiBase=https://api.cloudflare.com/client/v4] - Cloudflare API base URL
- * @prop {string} [input=cdp] - input method for user actions; defaults to 'cdp' for Kitesurf's real layout engine, but can be overridden
- * @prop {object} [capabilities] - pre-configured capabilities; Kitesurf uses { layout: 'real', screenshot: true }
+ * @prop {string} [input=cdp] - how actions are dispatched: `cdp` (real mouse and keyboard events) or `synthetic` (DOM events).
+ * @prop {object} [capabilities] - browser capabilities; Kitesurf sets `{ layout: 'real', screenshot: true }`.
  */
 const config = {}
 
@@ -90,14 +89,6 @@ class Kitesurf extends CDPBrowser {
     this.cloudSessionId = null
   }
 
-  /**
-   * Acquires a Kitesurf browser session from the Cloudflare Browser Run API and resolves it to
-   * the `wss://` debugger URL `CDPConnection` connects to. Overrides `CDPBrowser._resolveEndpoint`,
-   * which resolves a fixed local endpoint instead of provisioning a cloud session per test.
-   *
-   * @returns {Promise<string>} a `wss://` debugger URL ready to be passed to `CDPConnection`.
-   * @protected
-   */
   async _resolveEndpoint() {
     if (!this.options.accountId || !this.options.apiToken) {
       throw new Error('Kitesurf requires accountId and apiToken (or CF_ACCOUNT_ID / CF_API_TOKEN env vars)')
@@ -113,15 +104,6 @@ class Kitesurf extends CDPBrowser {
     return data.webSocketDebuggerUrl
   }
 
-  /**
-   * Closes the target as `CDPBrowser._finishTest` does, then releases the cloud session acquired
-   * in `_resolveEndpoint` via the Cloudflare API so it does not linger for the full `keepAlive`
-   * window. The release runs in a `finally` so a rejection while closing the CDP connection still
-   * frees the cloud session instead of leaving the browser alive until `keepAlive` expires; the
-   * session id is cleared before the request, so a repeated call never releases it twice.
-   *
-   * @protected
-   */
   async _finishTest() {
     try {
       await super._finishTest()

package/lib/helper/Lightpanda.js

@@ -0,0 +1,137 @@
+import CDPBrowser from './CDPBrowser.js'
+
+/**
+ * ## Configuration
+ *
+ * This helper should be configured in codecept.conf.js
+ *
+ * @typedef LightpandaConfig
+ * @type {object}
+ * @prop {string} [endpoint] - explicit CDP endpoint. Setting this switches the helper to ATTACH
+ * mode: it only connects, and never spawns or kills a process, no matter what else is configured.
+ * Leave it unset for SELF-MANAGED mode (see below).
+ * @prop {string} [binaryPath] - path to the `lightpanda` executable, used in SELF-MANAGED mode
+ * (`endpoint` unset). Checked before `LIGHTPANDA_PATH` and `PATH`.
+ * @prop {number} [port] - port `lightpanda serve` listens on, in SELF-MANAGED mode. When unset, a
+ * free port is picked automatically, which is what makes `run-workers` collision-free — every
+ * worker gets its own instance on its own port with zero config.
+ * @prop {number} [serverStartTimeout=15000] - milliseconds to wait for a spawned `lightpanda serve`
+ * to answer `/json/version` before `_connect` gives up.
+ */
+const config = {}
+
+/**
+ * Lightpanda drives [Lightpanda](https://lightpanda.io), a headless browser written from scratch
+ * for automation, exposed over the Chrome DevTools Protocol. It executes real JavaScript in V8 and
+ * implements the DOM and Web APIs, but has no graphical rendering engine: nothing is ever painted,
+ * so there are no screenshots.
+ *
+ * This helper is a thin `CDPBrowser` subclass: it changes nothing about how locating or acting on
+ * elements works, it only pins the config presets Lightpanda requires and manages the
+ * `lightpanda serve` process lifecycle, the same way Playwright manages its own browser process.
+ *
+ * > Lightpanda support is experimental. Pin the browser version in CI and keep a
+ * > Playwright/WebDriver job for browser-compatibility coverage.
+ *
+ * ## Modes
+ *
+ * - **ATTACH** — `endpoint` is set explicitly in the config. The helper only connects to it; it
+ *   never spawns or kills anything, no matter what `binaryPath`/`port` are set to.
+ * - **SELF-LAUNCH** — `endpoint` is unset and a binary can be resolved, in order: `binaryPath` in
+ *   the config, then the `LIGHTPANDA_PATH` environment variable, then `lightpanda` on `PATH`. The
+ *   helper spawns `lightpanda serve --host 127.0.0.1 --port <port>` (`port` from the config, or a
+ *   free port picked automatically), waits for it to answer, connects, and kills it in
+ *   `_finishTest`.
+ * - **COURTESY-ATTACH** — `endpoint` is unset and no binary can be resolved, but something already
+ *   answers `http://127.0.0.1:9222/json/version` (e.g. `lightpanda serve` started by hand, or the
+ *   `lightpanda/browser` Docker image). The helper attaches to it and never kills it. If neither a
+ *   binary nor a running server on :9222 can be found, the helper throws a loud, actionable error.
+ *
+ * ## Install
+ *
+ * Follow the [official installation instructions](https://lightpanda.io/docs/open-source/installation),
+ * then put `lightpanda` on your `PATH`, or point `binaryPath`/`LIGHTPANDA_PATH` at it. The helper
+ * then launches and tears it down automatically.
+ *
+ * Lightpanda sends usage telemetry by default. This helper always spawns it with
+ * `LIGHTPANDA_DISABLE_TELEMETRY=true`; set that variable yourself when you start the server by hand.
+ *
+ * Lightpanda is licensed under AGPL-3.0. It runs as a separate process that CodeceptJS talks to
+ * over a WebSocket, so it does not change the license of CodeceptJS or of your tests.
+ *
+ * ## Config presets
+ *
+ * These are set automatically and only need overriding for unusual setups:
+ *
+ * | option | value | why |
+ * | --- | --- | --- |
+ * | `input` | `synthetic` | nothing is painted, so coordinates are not hit-tested; `click` always takes the `forceClick` path |
+ * | `xpathPolyfill` | `auto` | probed per binary/page |
+ * | `capabilities.layout` | `real` | Lightpanda computes visibility-related styles and element boxes without painting, which is enough for `seeElement` and the `waitForVisible` family; it is pinned because the runtime layout probe cannot run on Lightpanda's `about:blank` |
+ * | `capabilities.screenshot` | `false` | there is no rendering engine |
+ *
+ * ## Limitations
+ *
+ * - No screenshots and no screencast: `saveScreenshot` throws.
+ * - No frames or popups.
+ * - No scrolling: scroll positions always stay at 0.
+ * - Computed styles cover what visibility checks need (`display`, `visibility`, `opacity`,
+ *   `pointer-events`); `grabCssPropertyFrom` and `seeCssPropertiesOnElements` are unreliable for
+ *   other properties.
+ * - No coordinate-based input (`clickXY`).
+ * - No clipboard access.
+ * - Some rich text editors (TinyMCE, Trix, Monaco) never finish initialising.
+ *
+ * <!-- configuration -->
+ *
+ * ## Example
+ *
+ * ```js
+ * // inside codecept.conf.js — SELF-LAUNCH mode (recommended): the helper finds/starts/stops
+ * // lightpanda serve on its own, on a free port. Ideal for run-workers: every worker gets its
+ * // own instance with no config.
+ * {
+ *   helpers: {
+ *     Lightpanda: {
+ *       url: 'http://localhost',
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * ```js
+ * // ATTACH mode — connect to a Lightpanda instance you manage yourself (remote host, container, etc.)
+ * {
+ *   helpers: {
+ *     Lightpanda: {
+ *       url: 'http://localhost',
+ *       endpoint: 'http://127.0.0.1:9222',
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * ## Methods
+ */
+class Lightpanda extends CDPBrowser {
+  /**
+   * @param {LightpandaConfig} config
+   */
+  constructor(config) {
+    super({
+      input: 'synthetic',
+      xpathPolyfill: 'auto',
+      ...config,
+      capabilities: { layout: 'real', screenshot: false, ...((config && config.capabilities) || {}) },
+    })
+    this._manageServer(config, {
+      name: 'lightpanda',
+      envVar: 'LIGHTPANDA_PATH',
+      args: port => ['serve', '--host', '127.0.0.1', '--port', String(port)],
+      env: { LIGHTPANDA_DISABLE_TELEMETRY: 'true' },
+      releases: 'https://github.com/lightpanda-io/browser/releases',
+    })
+  }
+}
+
+export default Lightpanda