kaloko 0.17.0 → 0.18.0

package/README.md

@@ -162,6 +162,13 @@ jobs:
 Uploads go by content hash: captures that did not change since an earlier run
 are not sent again, and an interrupted `share` picks up where it stopped.
 
+With the Kaloko GitHub App (Team and up), each pull request also gets one
+`Kaloko` check on its newest commit. It turns green when the runs shared for
+that commit pass, whether they came from this workflow, a coding agent in the
+cloud or a laptop. Branch protection can require it, and labels and
+`/kaloko skip <reason>` control a single pull request. Details:
+`skills/kaloko/reference/ci.md`.
+
 ## When a screenshot is not enough
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -136,6 +136,21 @@
         "site_contacts": { "type": "boolean", "default": true, "description": "kaloko share takes the site's own contacts for not personal data: e-mails on the environment's domain, e-mails and phone numbers in the page footer, and those of an Organization or ContactPoint in JSON-LD. false checks them like any other value." }
       }
     },
+    "github": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "The Kaloko check on pull requests (#141). The service reads it from the base branch; it may only make the check stricter than Settings on Kaloko.",
+      "properties": {
+        "checks": { "enum": ["off", "manual", "auto", "strict"], "description": "manual: only on request (label kaloko:run, /kaloko walk); auto: every pull request, the scenarios its change reaches; strict: every pull request, all scenarios, review required." },
+        "scope": { "enum": ["affected", "all"] },
+        "require_review": { "type": "boolean", "description": "Also post Kaloko review, which turns green when the runs are accepted." },
+        "drafts": { "enum": ["wait", "run"], "description": "wait: no check on a draft pull request until it is ready for review." },
+        "skip_authors": { "type": "array", "items": { "type": "string" }, "description": "Only in Settings: skipping authors would loosen the check, so the file's list is ignored." },
+        "skip_by": { "enum": ["write", "maintain", "admin"], "description": "The access a person needs to skip a pull request." },
+        "author_may_skip": { "type": "boolean" },
+        "wait_hours": { "type": "integer", "minimum": 1, "maximum": 336, "description": "strict: hours to wait for a run before the check times out." }
+      }
+    },
     "share": {
       "type": "object",
       "additionalProperties": false,

package/schemas/run.v1.json

@@ -116,6 +116,23 @@
         "sha": {
           "type": "string",
           "description": "Full commit hash (GitHub check runs need it)."
+        },
+        "repo": {
+          "type": "string",
+          "description": "owner/name of the origin remote (kaloko 0.18+): the service finds the pull request by commit when pr is missing."
+        },
+        "host": {
+          "type": "string",
+          "enum": ["github", "gitlab", "other"],
+          "description": "Where the origin remote lives."
+        },
+        "prefix": {
+          "type": "string",
+          "description": "The project's folder inside the repository (monorepos), without a trailing slash."
+        },
+        "dirty": {
+          "type": "boolean",
+          "description": "Tracked files differed from the commit when the run started: the run does not count for the commit's GitHub check."
         }
       }
     },

package/skills/kaloko/SKILL.md

@@ -168,6 +168,10 @@ only when a person says that value may be uploaded.
   [reference/recording.md](reference/recording.md)
 - CI: a workflow that walks each pull request's preview deployment and
   comments on it (`kaloko ci github|gitlab`): [reference/ci.md](reference/ci.md)
+- The `Kaloko` check on a pull request, what makes it pass, `kaloko ci plan`,
+  labels and `/kaloko` commands, and walking from a coding agent in the cloud
+  (commit first, a dirty tree does not count; never skip the check yourself):
+  [reference/ci.md#the-kaloko-check-on-pull-requests](reference/ci.md#the-kaloko-check-on-pull-requests)
 - Environments, access (gates, accounts, sign-in), e-mails, readonly and
   production rules, and what is yours to arrange for the project's stack:
   [reference/environments.md](reference/environments.md)

package/skills/kaloko/reference/ci.md

@@ -30,6 +30,10 @@ the deployment):
    `walk --ephemeral --affected` and `share --pr` (one comment per scenario,
    updated on later pushes).
 
+Before the loop, the workflow runs `kaloko ci plan` (see
+[The Kaloko check on pull requests](#the-kaloko-check-on-pull-requests)), so
+the check on the pull request knows which scenarios to wait for.
+
 `kaloko ci` also adds the `preview` environment to `kaloko.config.yml` with
 `base_url: ${KALOKO_PREVIEW_URL}`. Scenarios that list `environments:` must
 include `preview`, or `start` refuses them; the command says which.
@@ -67,6 +71,85 @@ in the newest shared run of the scenario count too, so a fresh CI checkout
 maps routes as well as a laptop with earlier runs. See
 [execution.md](execution.md#only-what-changed).
 
+## The Kaloko check on pull requests
+
+With the Kaloko GitHub App installed (an organization admin: Settings →
+Integrations → GitHub), every pull request in a connected repository gets one
+check named `Kaloko` on its newest commit. It does not walk anything itself:
+it waits for the runs shared for that commit, from a coding agent in the
+cloud, a person's machine or this workflow, and decides from them.
+
+| The check says | When |
+|---|---|
+| success | every scenario the change reaches has a run on this commit without failing steps, or nothing reaches a scenario |
+| failure | a run on this commit has a failing step (in strict mode a partial one too) |
+| in progress | a reached scenario has no run on this commit yet; the summary says how to get one |
+| neutral | the repository has no scenario yet, or only code Kaloko cannot map without the repository changed (auto mode) |
+| skipped | someone with the right access skipped it; the summary names them and their reason |
+
+Runs of a dirty working tree (tracked files differ from the commit) are
+listed but do not count: commit first, then walk. A new push supersedes the
+old commit's check. The optional `Kaloko review` check follows the people's
+decision (accepted, returned) on those runs.
+
+Which scenarios a change reaches: the service maps the pull request's files
+onto the steps' paths and captured URLs, like `kaloko affected` without the
+repository. `kaloko ci plan` sends what `kaloko affected` found (it follows
+imports and `affects:` globs), which wins:
+
+```bash
+kaloko ci plan                         # every scenario of the config, for HEAD
+kaloko ci plan --scenario qa/flows/checkout.yml --none   # this commit reaches nothing there
+kaloko ci plan --dry-run --json        # show it, send nothing
+```
+
+Modes (Settings, per repository): `off`, `manual` (only after the label
+`kaloko:run` or the comment `/kaloko walk`), `auto` (the default) and
+`strict` (all scenarios, review required, unknown impact waits, the check
+times out after `wait_hours`). The `github:` section of `kaloko.config.yml`
+on the base branch may make it stricter, never looser:
+
+```yaml
+github:
+  checks: strict          # off | manual | auto | strict
+  scope: all              # affected | all
+  require_review: true
+  drafts: run             # wait (default) | run
+  skip_by: maintain       # write | maintain | admin
+  author_may_skip: false
+  wait_hours: 12
+```
+
+One pull request: the labels `kaloko:skip` (whole pull request), `kaloko:all`,
+`kaloko:review`, `kaloko:run`; the comments `/kaloko skip <reason>` (this
+commit only), `/kaloko walk`, `/kaloko all`; the buttons on the check. Only
+people with the access set in Settings skip, never apps or bots, and every
+skip is on the check and in the audit log. Do not skip a check yourself as an
+agent; walk, or tell the person why it fails.
+
+To make it block merging, the repository's branch protection (or ruleset)
+requires the status check `Kaloko`. Kaloko never changes branch protection.
+
+### Coding agents in the cloud
+
+An agent in a sandbox (Claude Code on the web, Codex, Cursor background
+agents, Copilot) runs the same commands as on a laptop:
+
+```bash
+git commit -am "…"                     # walk what is committed
+npx kaloko start --scenario qa/flows/checkout.yml --env preview
+npx kaloko walk --affected
+npx kaloko share                       # --pr also comments, if gh is signed in
+```
+
+- `KALOKO_TOKEN` goes into the sandbox's secrets or environment variables
+  (tester role, upload scope; a token per agent shows who shared what).
+- `npx kaloko browser install --with-deps` belongs in the environment's setup
+  script. When the sandbox cannot run a browser or reach the preview, let the
+  workflow above walk instead.
+- `gh` is not needed: the run carries the repository and the commit, and
+  Kaloko finds the pull request.
+
 ## Exit codes
 
 | Code | Meaning |

package/src/cli.mjs

@@ -127,6 +127,10 @@ Project
                                     In CI: the preview URL (KALOKO_PREVIEW_URL, the deployment event, provider
                                     variables or the commit's GitHub deployments), the pull request and its base;
                                     --wait polls that long for the deployment (seconds, or 10m)
+  ci plan [--scenario a.yml,b.yml] [--base ref] [--none] [--repo owner/name] [--sha commit] [--dry-run] [--json]
+                                    Tell the GitHub check which scenarios this commit reaches (kaloko affected, which
+                                    follows imports): it then waits for runs of those and passes for the rest;
+                                    --none marks the scenarios as not reached
   login [--org <slug>] [--url <service>] [--print] [--no-open]
                                     Sign this machine in through the browser: approve the code on kaloko.app, choose
                                     the organization and what the token may do; the token (at most 30 days) goes to
@@ -1776,7 +1780,31 @@ const commands = {
       console.log(preview.url)
       return
     }
-    if (!['github', 'gitlab'].includes(action)) throw new UsageError('Usage: kaloko ci github|gitlab [--scenario a.yml,b.yml] [--env preview] [--trigger deployment|pull_request] [--no-affected] [--out file] [--print] [--force]\n       kaloko ci github --release [--snippet] [--print] [--force]\n       kaloko ci preview-url [--wait 600] [--github-env] [--dotenv file]')
+    if (action === 'plan') {
+      // #141: what kaloko affected finds for this commit (it follows imports and affects: globs) decides the GitHub
+      // check over the service's guess from the file list; a scenario nothing reaches lets the check pass
+      const { config, root } = await context(args)
+      const g = await gitMeta(root)
+      const repo = typeof args.repo === 'string' ? args.repo : (g?.host === 'github' ? g.repo : null) || process.env.GITHUB_REPOSITORY || null
+      const sha = typeof args.sha === 'string' ? args.sha.toLowerCase() : g?.sha
+      if (!repo || !/^[\w.-]+\/[\w.-]+$/.test(repo)) throw new UserError('kaloko ci plan needs the GitHub repository: run it in a clone whose origin is on GitHub, or pass --repo owner/name.')
+      if (!sha || !/^[0-9a-f]{40}$/.test(sha)) throw new UserError('kaloko ci plan needs the full commit: run it inside the repository, or pass --sha.')
+      const targets = list(args.scenario)?.map((f) => resolve(f)) || await scenarioFiles(config, root)
+      if (!targets.length) throw new UserError('No scenario: pass --scenario <file> or list scenarios in the config.')
+      const scenarios = []
+      for (const file of targets) {
+        const scenario = await loadScenario(file)
+        if (args.none) { scenarios.push({ id: scenario.id, affected: false }); continue }
+        const r = await affectedSelection(args, { config, root, scenario, scenarioFile: file })
+        scenarios.push({ id: scenario.id, affected: !r.sel.none, steps: r.sel.none ? [] : selectedIds(scenario, r.sel) })
+      }
+      if (!args['dry-run']) await service(config).request('POST', `/orgs/${config.org}/github/plan`, { repo, sha, scenarios })
+      if (args.json) { say(JSON.stringify({ repo, sha, scenarios, sent: !args['dry-run'] }, null, 2)); return }
+      for (const x of scenarios) say(`${x.affected ? '→' : '·'} ${x.id}: ${x.affected ? `walk ${x.steps.length} step(s)` : 'nothing this change reaches'}`)
+      say(args['dry-run'] ? 'Not sent (--dry-run).' : `Sent to Kaloko for ${repo}@${sha.slice(0, 7)}: the GitHub check expects runs of the scenarios marked →.`)
+      return
+    }
+    if (!['github', 'gitlab'].includes(action)) throw new UsageError('Usage: kaloko ci github|gitlab [--scenario a.yml,b.yml] [--env preview] [--trigger deployment|pull_request] [--no-affected] [--out file] [--print] [--force]\n       kaloko ci github --release [--snippet] [--print] [--force]\n       kaloko ci preview-url [--wait 600] [--github-env] [--dotenv file]\n       kaloko ci plan [--scenario a.yml,b.yml] [--base ref] [--none] [--repo owner/name] [--sha commit] [--dry-run] [--json]')
     const { config, root, file } = await context(args)
     if (args.release) {
       // release-tool integrations: record each published release on Kaloko (docs/DOCS-CHANGELOG.md)

package/src/lib/changelog-core.mjs

@@ -60,11 +60,29 @@ export function formatStepRef ({ scenario = null, step, locale = null, viewport
 export function findStepRefs (markdown) {
   const out = []
   const re = /!\[([^\]]*)\]\((kaloko:[^)\s]+)\)/g
+  const text = withoutCode(String(markdown || ''))
   let m
-  while ((m = re.exec(String(markdown || '')))) out.push({ alt: m[1], ref: m[2], parsed: parseStepRef(m[2]), index: m.index })
+  while ((m = re.exec(text))) out.push({ alt: m[1], ref: m[2], parsed: parseStepRef(m[2]), index: m.index })
   return out
 }
 
+/** The Markdown with fenced code blocks and inline code blanked out (same length, newlines kept, so indexes still
+ *  point into the original): a reference shown as an example in a guide is text, not a picture to resolve. */
+export function withoutCode (md) {
+  const blank = (s) => s.replace(/[^\n]/g, ' ')
+  const lines = md.split('\n')
+  let fence = null
+  for (let i = 0; i < lines.length; i++) {
+    const f = lines[i].match(/^ {0,3}(?:[-*+]\s+|\d+[.)]\s+|>\s*)*(`{3,}|~{3,})/) || lines[i].match(/^\s*(`{3,}|~{3,})/)
+    if (fence) {
+      if (f && f[1][0] === fence[0] && f[1].length >= fence.length && !lines[i].trim().slice(f[1].length).trim()) fence = null
+      lines[i] = blank(lines[i])
+    } else if (f) { fence = f[1]; lines[i] = blank(lines[i]) }
+  }
+  // a code span stays inside its paragraph: a stray backtick never pairs across a blank line
+  return lines.join('\n').replace(/(?<!`)(`+)(?!`)(?:(?!\n[ \t]*\n)[\s\S])+?(?<!`)\1(?!`)/g, blank)
+}
+
 // ---------- versions ----------
 
 /** "v2.4.0" → "2.4.0"; anything else stays as it is (trimmed). */

package/src/lib/ci.mjs

@@ -99,9 +99,11 @@ export function runner ({ pm = 'npm', dependency = true, version = 'latest' }) {
 }
 
 /** The loop over scenarios both workflows run: walk what the change reaches, share, comment on the PR. */
-function walkScript ({ scenarios, env, affected, k }) {
+function walkScript ({ scenarios, env, affected, k, plan = false }) {
   return [
     'status=0',
+    // the Kaloko check on the pull request (GitHub App): which scenarios this commit reaches, followed through imports
+    ...(plan && affected ? [`${k} ci plan --scenario ${scenarios.map(q).join(',')} || echo "kaloko ci plan did not reach Kaloko; the GitHub check guesses from the changed files."`] : []),
     `for s in ${scenarios.map(q).join(' ')}; do`,
     ...(affected ? [`  if ! ${k} affected --scenario "$s" --fail-if-none; then echo "Nothing in $s is affected; skipped."; continue; fi`] : []),
     `  ${k} start --scenario "$s" --env ${env}`,
@@ -159,7 +161,7 @@ ${install ? `${['pnpm', 'yarn'].includes(pm) ? '      - run: corepack enable\n'
         run: ${k} ci preview-url --wait 600 --github-env
       - name: Walk, share, comment
         run: |
-${indent(walkScript({ scenarios, env, affected, k }), 10)}
+${indent(walkScript({ scenarios, env, affected, k, plan: true }), 10)}
 `
 }
 

package/src/lib/git.mjs

@@ -90,5 +90,18 @@ export async function gitMeta (cwd) {
       pr = { number: p.number, url: p.url, state: String(p.state).toLowerCase(), ...(p.title ? { title: String(p.title).slice(0, 300) } : {}) }
     } catch {}
   }
-  return { branch: branch === 'HEAD' && ciBranch ? ciBranch : branch, commit, sha, message, author, pr }
+  // #141: the repository from the origin remote (a sandbox without gh has no PR; the service finds it by commit), the
+  // project's folder inside it (monorepos) and whether tracked files differ from the commit (such a walk does not
+  // count for the commit's GitHub check)
+  const [origin, prefix, changes] = await Promise.all([
+    repoSlug(cwd),
+    out('git', ['rev-parse', '--show-prefix'], cwd),
+    out('git', ['status', '--porcelain', '--untracked-files=no'], cwd)
+  ])
+  return {
+    branch: branch === 'HEAD' && ciBranch ? ciBranch : branch, commit, sha, message, author, pr,
+    ...(origin ? { repo: origin.slug, host: origin.host } : {}),
+    ...(prefix ? { prefix: prefix.replace(/\/$/, '') } : {}),
+    dirty: changes === null ? false : changes.length > 0
+  }
 }

package/viewer/index.html

@@ -3973,7 +3973,13 @@ function applyHash () {
 }
 
 /* ---------- replay: guided walkthrough with branch choice ---------- */
-const outEdges = (id) => RUN.edges.filter((e) => e.from === id && byId(e.to)).sort((a, b) => (a.kind === 'alt') - (b.kind === 'alt'));
+// the ways on from a step in the order of the map: the main path before alternatives, forward before back, the same
+// row before another, then the nearest column and row, so → in replay follows the diagram, not the order of the file
+const outEdges = (id) => {
+  const from = byId(id); const c = (n) => n?.col || 0, l = (n) => n?.lane || 0;
+  const k = (e) => { const t = byId(e.to); return [e.kind === 'alt' ? 1 : 0, c(t) > c(from) ? 0 : 1, l(t) === l(from) ? 0 : 1, Math.abs(c(t) - c(from)), Math.abs(l(t) - l(from))]; };
+  return RUN.edges.filter((e) => e.from === id && byId(e.to)).map((e, i) => [e, k(e), i]).sort(([, a, i], [, b, j]) => { for (let x = 0; x < a.length; x++) if (a[x] !== b[x]) return a[x] - b[x]; return i - j; }).map(([e]) => e);
+};
 function startNode () { const order = RUN.nodes.slice().sort((a, b) => a.col - b.col || a.lane - b.lane); return order.find((n) => !RUN.edges.some((e) => e.to === n.id)) || order[0]; }
 function rpOpen (path) {
   const ids = (path || []).filter((id) => byId(id));
@@ -4035,7 +4041,8 @@ function renderReplay () {
   $('rpClose').onclick = rpClose; $('rpPrev').onclick = rpBack;
   $('rp').querySelectorAll('[data-go]').forEach((b) => { b.onclick = () => rpGo(b.dataset.go); });
   $('rp').querySelectorAll('[data-jump]').forEach((b) => { b.onclick = () => rpJump(Number(b.dataset.jump)); });
-  $('rp').querySelectorAll('[data-stack-open]').forEach((b) => { b.onclick = () => openStack(b.dataset.stackOpen); });
+  // the stack's index opens over the canvas, not under the replay (which would hide it and take its keys)
+  $('rp').querySelectorAll('[data-stack-open]').forEach((b) => { b.onclick = () => { const id = b.dataset.stackOpen; rpClose(); openStack(id); }; });
   $('rp').querySelector('[data-restart]')?.addEventListener('click', () => rpOpen(null));
   if ($('rpVid')) $('rpVid').onclick = () => { S.rpVideo = !S.rpVideo; renderReplay(); };
   if (S.rpVideo) {