kaloko 0.17.0 → 0.19.0

package/README.md

@@ -162,6 +162,13 @@ jobs:
 Uploads go by content hash: captures that did not change since an earlier run
 are not sent again, and an interrupted `share` picks up where it stopped.
 
+With the Kaloko GitHub App (Team and up), each pull request also gets one
+`Kaloko` check on its newest commit. It turns green when the runs shared for
+that commit pass, whether they came from this workflow, a coding agent in the
+cloud or a laptop. Branch protection can require it, and labels and
+`/kaloko skip <reason>` control a single pull request. Details:
+`skills/kaloko/reference/ci.md`.
+
 ## When a screenshot is not enough
 
 ```bash

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.17.0",
+  "version": "0.19.0",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -33,10 +33,28 @@
       "minProperties": 1,
       "additionalProperties": { "$ref": "#/$defs/environment" }
     },
-    "evaluators": {
+    "evaluate": {
       "type": "object",
       "additionalProperties": false,
+      "description": "Settings of every evaluation (kaloko evaluate and the end of kaloko walk).",
+      "properties": {
+        "budget": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Upper bound of one evaluation (#158). Steps go first, then the semantic sample of each stack, chosen by priority (pages failing a deterministic acceptance check, then those checked longest ago) unless the step says semantic_order: walk; what is past the budget stays not_evaluated with a note.",
+          "properties": {
+            "calls": { "type": "integer", "minimum": 0, "description": "Uncached requests to JEV (your key or through Kaloko) per evaluation, escalations included; requests to your own openai_compatible evaluator are not counted. With a budget, screens without semantic criteria get no quick signals unless already cached. kaloko evaluate --budget N overrides it." }
+          }
+        }
+      }
+    },
+    "evaluators": {
+      "type": "object",
+      "description": "Text evaluators (jev, and your own of kind openai_compatible under any other name), the vision evaluator and triage.",
+      "additionalProperties": { "$ref": "#/$defs/openaiEvaluator" },
       "properties": {
+        "text": { "type": "string", "default": "jev", "description": "The evaluator that answers the semantic questions first: jev, or the name of an openai_compatible evaluator here (a local model in Ollama or LM Studio). Its verdicts are marked via: own-model on the canvas, in the PR comment and in acceptance records." },
+        "escalate_to": { "const": "jev", "description": "Answers of your own evaluator (evaluators.text) that are not clear enough for a verdict (between the fail and pass thresholds, or missing) are asked again of JEV (your key or through Kaloko, counted in evaluate.budget). Only your model escalates to JEV, never the other way. Quick signals are not escalated." },
         "jev": {
           "type": "object",
           "additionalProperties": false,
@@ -136,6 +154,21 @@
         "site_contacts": { "type": "boolean", "default": true, "description": "kaloko share takes the site's own contacts for not personal data: e-mails on the environment's domain, e-mails and phone numbers in the page footer, and those of an Organization or ContactPoint in JSON-LD. false checks them like any other value." }
       }
     },
+    "github": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "The Kaloko check on pull requests (#141). The service reads it from the base branch; it may only make the check stricter than Settings on Kaloko.",
+      "properties": {
+        "checks": { "enum": ["off", "manual", "auto", "strict"], "description": "manual: only on request (label kaloko:run, /kaloko walk); auto: every pull request, the scenarios its change reaches; strict: every pull request, all scenarios, review required." },
+        "scope": { "enum": ["affected", "all"] },
+        "require_review": { "type": "boolean", "description": "Also post Kaloko review, which turns green when the runs are accepted." },
+        "drafts": { "enum": ["wait", "run"], "description": "wait: no check on a draft pull request until it is ready for review." },
+        "skip_authors": { "type": "array", "items": { "type": "string" }, "description": "Only in Settings: skipping authors would loosen the check, so the file's list is ignored." },
+        "skip_by": { "enum": ["write", "maintain", "admin"], "description": "The access a person needs to skip a pull request." },
+        "author_may_skip": { "type": "boolean" },
+        "wait_hours": { "type": "integer", "minimum": 1, "maximum": 336, "description": "strict: hours to wait for a run before the check times out." }
+      }
+    },
     "share": {
       "type": "object",
       "additionalProperties": false,
@@ -186,6 +219,23 @@
         "images": { "type": "object", "additionalProperties": false, "properties": { "viewport": { "type": "string" }, "locale": { "type": "string", "description": "reader (the reader's language) or a locale." } } }
       }
     },
+    "openaiEvaluator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "base_url", "model"],
+      "description": "Your own text evaluator behind an OpenAI-compatible chat completions API (#158): Ollama (http://localhost:11434/v1), LM Studio (http://localhost:1234/v1), vLLM or a gateway. Used only when evaluators.text or evaluators.escalate_to names it; the page state goes only to base_url.",
+      "properties": {
+        "kind": { "const": "openai_compatible" },
+        "base_url": { "type": "string", "description": "The API root with /v1; requests go to {base_url}/chat/completions." },
+        "model": { "type": "string", "description": "The model as the server names it (qwen3:8b, gemma3:12b)." },
+        "api_key_env": { "type": "string", "description": "Variable with a key sent as a bearer token; a local server needs none." },
+        "concurrency": { "type": "integer", "minimum": 1, "maximum": 16, "default": 2, "description": "Requests in flight at once; a local model usually answers one at a time." },
+        "max_state_chars": { "type": "integer", "minimum": 1000, "default": 24000, "description": "The page text is cut to this length (the top of the page): small models have small context windows." },
+        "timeout_ms": { "type": "integer", "minimum": 1000, "default": 120000 },
+        "temperature": { "type": "number", "minimum": 0, "maximum": 2, "default": 0 },
+        "response_format": { "enum": ["json_schema", "json_object", "none"], "default": "json_schema", "description": "How the answer is held to JSON; json_object or none for servers without structured output." }
+      }
+    },
     "record": {
       "description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
       "oneOf": [

package/schemas/run.v1.json

@@ -116,6 +116,23 @@
         "sha": {
           "type": "string",
           "description": "Full commit hash (GitHub check runs need it)."
+        },
+        "repo": {
+          "type": "string",
+          "description": "owner/name of the origin remote (kaloko 0.18+): the service finds the pull request by commit when pr is missing."
+        },
+        "host": {
+          "type": "string",
+          "enum": ["github", "gitlab", "other"],
+          "description": "Where the origin remote lives."
+        },
+        "prefix": {
+          "type": "string",
+          "description": "The project's folder inside the repository (monorepos), without a trailing slash."
+        },
+        "dirty": {
+          "type": "boolean",
+          "description": "Tracked files differed from the commit when the run started: the run does not count for the commit's GitHub check."
         }
       }
     },
@@ -424,6 +441,40 @@
         "null"
       ],
       "description": "Version of the application under test (git tag on HEAD or --app-version); documentation captures carry it."
+    },
+    "site": {
+      "type": "object",
+      "description": "Site checks (#152, scenario health.site), once per site and run: robots.txt, sitemaps, llms.txt, http→https and www redirects. Advisory: health shows a finding per failed check (pack site, step site:<host>); they never decide a verdict.",
+      "properties": {
+        "origins": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["origin", "checks"],
+            "properties": {
+              "origin": { "type": "string" },
+              "checked_at": { "type": "string" },
+              "checks": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "required": ["id", "verdict"],
+                  "properties": {
+                    "id": { "type": "string", "description": "HC1101 … HC1107." },
+                    "text": { "type": "string" },
+                    "verdict": { "enum": ["pass", "fail", "not_evaluated"] },
+                    "level": { "enum": ["error", "warning"] },
+                    "evidence": { "type": ["string", "null"] },
+                    "na": { "type": "boolean", "description": "Nothing to check here (no robots.txt to read, no other host name)." },
+                    "details": { "type": "array", "items": { "type": "string" } },
+                    "help": { "type": "object" }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "$defs": {
@@ -648,7 +699,9 @@
               "counts": { "type": "object", "description": "Stack steps: pages per verdict." },
               "evidence": { "type": ["string", "null"] },
               "check": { "type": "object" },
-              "help": { "type": "object" }
+              "help": { "type": "object" },
+              "cross_page": { "type": "boolean", "description": "Stack steps (#151): a check across the stack's pages (HC121–HC126: duplicates, canonicals, hreflang pairs, sitemap pages). counts says how many pages are affected; it cannot be adopted as a criterion." },
+              "urls": { "type": "array", "items": { "type": "string" }, "description": "Cross-page checks: the first affected addresses (at most 20); the stack's items.json has every group." }
             }
           }
         },
@@ -707,6 +760,16 @@
                 "string",
                 "null"
               ]
+            },
+            "rotation": {
+              "type": "object",
+              "description": "Rotating sample (items.rotate), per locale: mode (week or run), week or run, offset, limit, total, rounds (runs to cover every page), covered, never (pages not checked yet) and oldest_checked.",
+              "additionalProperties": { "type": "object" }
+            },
+            "cross_page": {
+              "type": "array",
+              "description": "Cross-page checks of a stack with the seo pack (#151): id, verdict, pages compared and pages affected. items.json has them in full under cross_page (groups with the affected addresses) and marks each affected page with cross_fails.",
+              "items": { "type": "object" }
             }
           }
         }

package/schemas/scenario.v1.json

@@ -94,7 +94,23 @@
       "additionalProperties": false,
       "description": "Health (CS Kondice): the checks of packs are advisory, shown apart from acceptance and never part of a verdict. acceptance lists the packs whose checks still count for acceptance in this scenario, as they did before health was split off (the design pack always does). To make one check count, adopt it: kaloko health adopt <step>:<check> (e.g. home:HC103) adds the criterion AC103 with from: HC103.",
       "properties": {
-        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design"] } }
+        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design", "aeo", "locale"] } },
+        "site": {
+          "description": "Site checks (#152), once per site and run: robots.txt, the sitemaps it lists, llms.txt, and http→https and www redirects in one hop. Advisory (health, pack site, HC11xx); off unless set, because they make requests of their own. true for the defaults, or an object.",
+          "oneOf": [
+            { "type": "boolean" },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "paths": { "type": "array", "maxItems": 50, "items": { "type": "string", "pattern": "^/" }, "description": "Paths robots.txt must not block for search engines (default: / and the start path of each walked language)." },
+                "ai_crawlers": { "type": "boolean", "description": "false skips HC1103 (AI crawlers blocked in robots.txt) when blocking them is the site's decision. Default true." },
+                "sitemaps": { "type": "integer", "minimum": 1, "maximum": 50, "description": "Most sitemap files to fetch per site, index children included (default and most 50)." },
+                "environments": { "type": "array", "minItems": 1, "items": { "type": "string" }, "description": "Run the site checks only on these environments (e.g. [production]). Default: every environment; off production HC1102 is a warning and HC1103 is not evaluated." }
+              }
+            }
+          ]
+        }
       }
     },
     "tags": {
@@ -344,6 +360,10 @@
           "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,39}$", "not": { "enum": ["index"] } },
           "description": "States this step is captured in besides the default one, e.g. [empty, error, loading]; any name works. Each state is one more capture per viewport (keys <viewport>.state-<name>) and a value of the State switcher on the canvas. Design steps render <step>/<state>.html (or <step>/<locale>/<state>.html), else <step>/index.html?state=<name>; walks of web and app steps run the step script once more per state with the argument state. `default` may be listed; it is always captured."
         },
+        "expect_status": {
+          "oneOf": [{ "type": "integer", "minimum": 100, "maximum": 599 }, { "type": "array", "minItems": 1, "items": { "type": "integer", "minimum": 100, "maximum": 599 } }],
+          "description": "The HTTP status this step is expected to answer with, e.g. 404 for a missing page. A status of 400 or more marks an error page: the errors pack checks HC601 and HC605 only on error pages (this, or a 4xx/5xx answer)."
+        },
         "page": {
           "enum": ["landing", "work", "list", "settings", "form", "doc"],
           "description": "What kind of page this step is (see the scenario's page); the usability pack asks per type, e.g. one primary call to action only on landing pages."
@@ -359,10 +379,12 @@
               "console",
               "errors",
               "app",
-              "design"
+              "design",
+              "aeo",
+              "locale"
             ]
           },
-          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
+          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app, HC9xx aeo, HC10xx locale. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
         },
         "capture": {
           "$ref": "#/$defs/capture"
@@ -394,7 +416,7 @@
         "items": {
           "type": "object",
           "additionalProperties": false,
-          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap (with match/exclude), crawl or script.",
+          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap or robots (with match/exclude), crawl or script.",
           "properties": {
             "urls": {
               "type": "array",
@@ -405,19 +427,23 @@
             },
             "file": {
               "type": "string",
-              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path,title}; relative to the scenario"
+              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path, title, key, purpose, edit_url, content_ref}; relative to the scenario"
             },
             "sitemap": {
               "type": "string",
-              "description": "Sitemap path or URL (sitemap indexes are followed one level)"
+              "description": "Sitemap path or URL, gzipped (.gz) or not; sitemap indexes are followed two levels. `robots` reads the Sitemap: lines of /robots.txt (see robots)."
+            },
+            "robots": {
+              "type": "boolean",
+              "description": "The sitemaps listed in /robots.txt (Sitemap: lines); when their URLs name the walked locale (/sitemaps/de/…), only those of that locale"
             },
             "match": {
-              "type": "string",
-              "description": "Regex on the path: keep matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: keep matching pages. A map by locale ({ default: '^/eshop/', fi: '^/fi/tuotteet/' }) gives each locale its own regex (the locale, its language, then default). Applies to every source before rotate and limit."
             },
             "exclude": {
-              "type": "string",
-              "description": "Regex on the path: drop matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: drop matching pages. A string or a map by locale like match."
             },
             "crawl": {
               "type": "object",
@@ -449,7 +475,15 @@
             },
             "script": {
               "type": "string",
-              "description": "Module relative to the scenario: default export async ({ page, env, locale, url }) => [{ path | url, title?, key? }]"
+              "description": "Module relative to the scenario: default export async ({ page, env, locale, url, node, params }) => [{ path | url, title?, key?, purpose?, edit_url?, content_ref? }]. node: { id, title, lane, lane_title, session }; params: items.params. edit_url (the page in the CMS) and content_ref (its record id) show in the stack index and in kaloko export findings."
+            },
+            "resolve": {
+              "type": "string",
+              "description": "Module relative to the scenario that links each walked page to its source: default export async (item, { env, locale, node, params, url }) => ({ edit_url?, content_ref? }). Called for the walked pages only (after match, rotate, limit and walk --items), 15 s per page; a page it fails on is walked without the link."
+            },
+            "params": {
+              "type": "object",
+              "description": "Values passed to items.script as params, so one script serves several stacks (for example { family: magazine })."
             },
             "limit": {
               "type": "integer",
@@ -457,11 +491,19 @@
               "maximum": 10000,
               "description": "At most this many pages (default 500)"
             },
+            "rotate": {
+              "enum": ["week", "run"],
+              "description": "Rotating sample: each run takes limit pages starting at a stable offset, the ISO week (week) or the number of runs so far in this output folder (run), so the whole list is covered over several runs."
+            },
             "semantic_sample": {
               "type": "integer",
               "minimum": 0,
               "maximum": 10000,
-              "description": "Semantic criteria are evaluated on the first N pages only (default 20); deterministic ones on every page"
+              "description": "Semantic criteria are evaluated on N pages only (default 20), in the order of semantic_order; deterministic ones on every page"
+            },
+            "semantic_order": {
+              "enum": ["walk", "priority"],
+              "description": "Which pages make the semantic sample. walk (default without an evaluator budget): the first N in walk order. priority (default with evaluate.budget): pages failing a deterministic acceptance criterion first, then those never or least recently checked (last_checked of a rotating sample), then walk order."
             },
             "viewports": {
               "type": "array",
@@ -492,7 +534,7 @@
               },
               "from": {
                 "type": "string",
-                "description": "title | h1 | description | canonical | lang | words | http | url | path | a CSS selector (text), selector@attribute, or count:selector"
+                "description": "title | h1 | description | canonical | lang | words | http | url | path | jsonld_types (the @type values of the JSON-LD blocks) | og:title, twitter:card or meta:<name> (a meta tag's content) | a CSS selector (text), selector@attribute, or count:selector"
               }
             }
           }
@@ -649,6 +691,16 @@
             "link_home",
             "links_ok",
             "hreflang_pairs",
+            "llms_txt",
+            "nojs_parity",
+            "jsonld_present",
+            "jsonld_valid",
+            "aeo_faq_schema",
+            "aeo_answer_first",
+            "locale_lang",
+            "locale_hreflang",
+            "locale_text",
+            "locale_prices",
             "app_touch_targets",
             "app_no_truncation",
             "app_no_crash",
@@ -687,6 +739,11 @@
         "attr": {
           "type": "string"
         },
+        "types": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "jsonld_present: schema.org types the page's structured data must contain (Product, BreadcrumbList, FAQPage…)."
+        },
         "property": {
           "type": "string",
           "minLength": 1,
@@ -759,7 +816,13 @@
         "ratio": {
           "type": "number",
           "minimum": 0,
-          "description": "design_pixels: maximum share of differing pixels (default 0.03)."
+          "description": "design_pixels: maximum share of differing pixels (default 0.03); nojs_parity: the least share of the rendered body text the raw HTML must hold (default 0.3)."
+        },
+        "user_agent": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 300,
+          "description": "nojs_parity: fetch the raw HTML as this crawler, without cookies, and respect its robots.txt rules: gptbot, oai-searchbot, claudebot, perplexitybot, googlebot, bingbot, or a whole user agent string. Without it, the walker's own browser user agent and session, and robots.txt is not read."
         },
         "follow": {
           "type": "boolean",
@@ -918,6 +981,10 @@
           "type": "boolean",
           "description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
         },
+        "screenshot": {
+          "enum": ["full", "thumb"],
+          "description": "thumb keeps only the small card thumbnail (640 px wide) as the step's image: for stacks of hundreds of pages where the checks read the page, not the picture (SEO crawls). Default full."
+        },
         "record": { "$ref": "#/$defs/record" },
         "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
       }

package/skills/kaloko/SKILL.md

@@ -168,6 +168,10 @@ only when a person says that value may be uploaded.
   [reference/recording.md](reference/recording.md)
 - CI: a workflow that walks each pull request's preview deployment and
   comments on it (`kaloko ci github|gitlab`): [reference/ci.md](reference/ci.md)
+- The `Kaloko` check on a pull request, what makes it pass, `kaloko ci plan`,
+  labels and `/kaloko` commands, and walking from a coding agent in the cloud
+  (commit first, a dirty tree does not count; never skip the check yourself):
+  [reference/ci.md#the-kaloko-check-on-pull-requests](reference/ci.md#the-kaloko-check-on-pull-requests)
 - Environments, access (gates, accounts, sign-in), e-mails, readonly and
   production rules, and what is yours to arrange for the project's stack:
   [reference/environments.md](reference/environments.md)

package/skills/kaloko/reference/acceptance.md

@@ -43,8 +43,15 @@ health checks (ids `HC1xx` for `seo`, `HC2xx` for `usability`; acceptance
 criteria are `AC`, health checks `HC`, and the number is the check's: the
 same check kept in acceptance or adopted is `AC103` for `HC103`) at load time:
 `seo` checks one H1, title and description length, canonical, hreflang for
-every locale, Open Graph, indexing, viewport, `html lang`, overflow and
-console; `usability` checks unlabeled inputs and link texts and asks the
+every locale, Open Graph, indexing, viewport, `html lang`, overflow,
+console and structured data (HC112 JSON-LD present, HC113 every block valid);
+`aeo` (HC9xx) adds answer-engine signals: FAQPage/QAPage where the page asks
+and answers, an answer-first paragraph under the H1, `/llms.txt`, a meta
+description, the raw HTML as GPTBot gets it without JavaScript (HC905,
+`nojs_parity`; `user_agent: claudebot|googlebot|…` and `ratio` on a criterion
+of your own); `locale` (HC10xx, scenarios with `locales`) checks html lang,
+the own hreflang, the text's language and price decimals against the locale;
+`usability` checks unlabeled inputs and link texts and asks the
 evaluator questions that fit the step's page type. Pack checks are health,
 not acceptance: they are evaluated on every walk but never decide a step's
 verdict (see [evaluation.md#health](evaluation.md#health)). The `design` pack
@@ -122,13 +129,20 @@ English words; one plain word such as a brand name is left alone).
   - a plain-language message with a way forward (semantic);
   - no console errors. The document's own error status is not counted.
 
+  HC601 and HC605 apply only to error pages: a step with `expect_status`
+  (400 or more, or an own `http_status` criterion of 400 or more) or a page
+  that answered 4xx/5xx. On other pages they are not applicable, so keep the
+  pack off normal pages and stacks (`kaloko validate` warns on a stack).
+
 ## Error pages
 
 Every site has them and they are easy to forget. Give each one its own
 step with `packs: [errors]`:
 
-- **404:** use `path: /no-such-page-{random}`. `{random}` becomes a new
-  token on every walk, so no cache or an accidentally created page makes it
+- **404:** use `path: /no-such-page-{random}` and `expect_status: 404`, so a
+  soft 404 (HTTP 200) fails HC601 instead of being not applicable.
+  `{random}` becomes a new token on every walk, so no cache or an
+  accidentally created page makes it
   pass. `kaloko draft` adds this step on its own.
 - **Exact status:** override HC601 with your own AC601, for example
   `{ id: AC601, text: Answers 410, check: { type: deterministic, assert: http_status, value: 410 } }`.

package/skills/kaloko/reference/ci.md

@@ -30,6 +30,10 @@ the deployment):
    `walk --ephemeral --affected` and `share --pr` (one comment per scenario,
    updated on later pushes).
 
+Before the loop, the workflow runs `kaloko ci plan` (see
+[The Kaloko check on pull requests](#the-kaloko-check-on-pull-requests)), so
+the check on the pull request knows which scenarios to wait for.
+
 `kaloko ci` also adds the `preview` environment to `kaloko.config.yml` with
 `base_url: ${KALOKO_PREVIEW_URL}`. Scenarios that list `environments:` must
 include `preview`, or `start` refuses them; the command says which.
@@ -67,6 +71,85 @@ in the newest shared run of the scenario count too, so a fresh CI checkout
 maps routes as well as a laptop with earlier runs. See
 [execution.md](execution.md#only-what-changed).
 
+## The Kaloko check on pull requests
+
+With the Kaloko GitHub App installed (an organization admin: Settings →
+Integrations → GitHub), every pull request in a connected repository gets one
+check named `Kaloko` on its newest commit. It does not walk anything itself:
+it waits for the runs shared for that commit, from a coding agent in the
+cloud, a person's machine or this workflow, and decides from them.
+
+| The check says | When |
+|---|---|
+| success | every scenario the change reaches has a run on this commit without failing steps, or nothing reaches a scenario |
+| failure | a run on this commit has a failing step (in strict mode a partial one too) |
+| in progress | a reached scenario has no run on this commit yet; the summary says how to get one |
+| neutral | the repository has no scenario yet, or only code Kaloko cannot map without the repository changed (auto mode) |
+| skipped | someone with the right access skipped it; the summary names them and their reason |
+
+Runs of a dirty working tree (tracked files differ from the commit) are
+listed but do not count: commit first, then walk. A new push supersedes the
+old commit's check. The optional `Kaloko review` check follows the people's
+decision (accepted, returned) on those runs.
+
+Which scenarios a change reaches: the service maps the pull request's files
+onto the steps' paths and captured URLs, like `kaloko affected` without the
+repository. `kaloko ci plan` sends what `kaloko affected` found (it follows
+imports and `affects:` globs), which wins:
+
+```bash
+kaloko ci plan                         # every scenario of the config, for HEAD
+kaloko ci plan --scenario qa/flows/checkout.yml --none   # this commit reaches nothing there
+kaloko ci plan --dry-run --json        # show it, send nothing
+```
+
+Modes (Settings, per repository): `off`, `manual` (only after the label
+`kaloko:run` or the comment `/kaloko walk`), `auto` (the default) and
+`strict` (all scenarios, review required, unknown impact waits, the check
+times out after `wait_hours`). The `github:` section of `kaloko.config.yml`
+on the base branch may make it stricter, never looser:
+
+```yaml
+github:
+  checks: strict          # off | manual | auto | strict
+  scope: all              # affected | all
+  require_review: true
+  drafts: run             # wait (default) | run
+  skip_by: maintain       # write | maintain | admin
+  author_may_skip: false
+  wait_hours: 12
+```
+
+One pull request: the labels `kaloko:skip` (whole pull request), `kaloko:all`,
+`kaloko:review`, `kaloko:run`; the comments `/kaloko skip <reason>` (this
+commit only), `/kaloko walk`, `/kaloko all`; the buttons on the check. Only
+people with the access set in Settings skip, never apps or bots, and every
+skip is on the check and in the audit log. Do not skip a check yourself as an
+agent; walk, or tell the person why it fails.
+
+To make it block merging, the repository's branch protection (or ruleset)
+requires the status check `Kaloko`. Kaloko never changes branch protection.
+
+### Coding agents in the cloud
+
+An agent in a sandbox (Claude Code on the web, Codex, Cursor background
+agents, Copilot) runs the same commands as on a laptop:
+
+```bash
+git commit -am "…"                     # walk what is committed
+npx kaloko start --scenario qa/flows/checkout.yml --env preview
+npx kaloko walk --affected
+npx kaloko share                       # --pr also comments, if gh is signed in
+```
+
+- `KALOKO_TOKEN` goes into the sandbox's secrets or environment variables
+  (tester role, upload scope; a token per agent shows who shared what).
+- `npx kaloko browser install --with-deps` belongs in the environment's setup
+  script. When the sandbox cannot run a browser or reach the preview, let the
+  workflow above walk instead.
+- `gh` is not needed: the run carries the repository and the commit, and
+  Kaloko finds the pull request.
+
 ## Exit codes
 
 | Code | Meaning |

package/skills/kaloko/reference/evaluation.md

@@ -27,6 +27,48 @@ report with the reason `kaloko evaluate` gives.
 The default evaluator is text-only and most accurate in English. It is weak
 at numbers, which is why those are checked deterministically.
 
+## Budget and your own model
+
+On large sites the semantic questions are what costs money. Three settings keep it in hand:
+
+- **Budget.** `evaluate.budget.calls` in `kaloko.config.yml` (or `kaloko evaluate --budget N`, `kaloko walk
+  --budget N`) caps the uncached JEV calls of one evaluation, the walk's own questions included. Ordinary steps
+  go first, then each stack's sample. What does not fit stays `not_evaluated` with the note "the evaluator
+  budget of N call(s) is used up", and `kaloko evaluate` prints how many captures it left out. Cached answers
+  are free, so the next evaluation continues where this one stopped. With a budget, screens without semantic
+  criteria get no new quick signals. Report the skipped count; never call those criteria passed.
+- **Sample order.** A stack asks the semantic questions on `semantic_sample` pages (default 20). With a budget,
+  or `items.semantic_order: priority` on the step, they are chosen by priority: pages failing a deterministic
+  acceptance criterion first (health checks do not count), then pages never or least recently checked
+  (`last_checked` of a rotating sample), then the rest in walk order. Otherwise, and with `semantic_order: walk`,
+  the first pages in walk order. The same captures always give the same sample and spend the budget on the same
+  pages.
+- **Your own model.** An evaluator of `kind: openai_compatible` (Ollama, LM Studio, vLLM) answers the same
+  questions from the same page text, for free and without the page leaving the machine. It is opt-in:
+  `evaluators.text: <name>` makes it answer first. With `evaluators.escalate_to: jev` its answers that are not
+  clear enough for a verdict (between the thresholds, a missing answer, or anything but a number from 0 to 1)
+  are asked again of JEV, counted in the budget; the quick signals are not escalated. Escalation goes only that
+  way, never from JEV to your model. Small models are rougher judges, so treat their verdicts as a first pass and
+  keep JEV for escalation. After three failed connections in a row an evaluation stops asking your model.
+
+```yaml
+evaluate:
+  budget: { calls: 300 }
+evaluators:
+  ollama:
+    kind: openai_compatible
+    base_url: http://localhost:11434/v1   # LM Studio: http://localhost:1234/v1
+    model: qwen3:8b
+  text: ollama
+  escalate_to: jev
+```
+
+Every text verdict then names who decided it, with `via`, `model` and `prompt_version`: `openai-compatible`
+via `own-model` for your model, `typesafe-jev` for an escalated answer with the local score kept as
+`first_score`. The canvas, acceptance records and the PR comment show your model's verdicts as such; say so in the
+report too, and never present them as JEV's. Pages are not batched into one
+request: each page state is one request, cached by its content, which is what makes re-evaluation free.
+
 ## Vision and reasons
 
 Every semantic verdict carries one sentence, `reason`, shown as "Why" next to
@@ -71,7 +113,8 @@ evaluator saw the screenshot, say so.
 ## Health
 
 The checks of packs (`seo`, `a11y`, `perf`, `usability`, `console`,
-`errors`, `app`) are the run's **health** (CS *kondice*): advisory, stored in
+`errors`, `app`, `aeo`, `locale`), the cross-page checks of stacks with
+`seo` and the site checks of `health.site` are the run's **health** (CS *kondice*): advisory, stored in
 `node.health` and `summary.health` of `run.json`, never in `criteria`. They
 do not change a step's status, the run verdict, the review queue, carried
 approvals, required approvers, the acceptance record or exit codes. The

package/skills/kaloko/reference/integrations.md

@@ -77,6 +77,12 @@ kaloko schedule hook <id>       # URL + secret that start the run after a deploy
   and the `health.worse` webhook on a new error-level finding. Each schedule
   sets it to `error` (default), `warning` or `off` on kaloko.app or with
   `kaloko schedule add --health-alert`; changing it is the person's call.
+- Health for dashboards: every shared run with health checks gets counts per
+  locale, step, check and pack (`GET …/runs/<id>/health/aggregate`,
+  `…/scenarios/<id>/latest/health/aggregate`) and a diff against the previous
+  run per page (`…/health/diff`: new, fixed, still open, first seen). A
+  `read:health` token reads these and nothing else; the `health.changed`
+  webhook says when something is new or fixed. help/reference/health-api.md.
 - Does not work for: native apps, local or private addresses (localhost, IPs,
   `.local`), header credentials, base URLs per locale.
 - For those, or for the full walk on a schedule, write a GitHub Actions

package/skills/kaloko/reference/mcp.md

@@ -52,6 +52,8 @@ is null. Projects and products come complete, without pages.
 | `add_sample_run` | adds the sample run (or returns the one there is); needs an upload or full token and the Developer / Tester role |
 | `search_screens` | the screen library: newest screenshot of every step across the organization's flows (`q` words matched against titles, ids, paths, URLs, page title and H1; `project`, `product`, `module`, `scenario`, `locale`, `viewport`, `device`, `status`, `environment`; `source: design` or `all` for design versions; `group: none` for every locale and viewport; `cursor` from `next`), each with a signed image and the canvas link |
 | `get_health` | the advisory health of a run (`run_id`), the newest run of a scenario (`scenario`) or the trend of a project (`project`): findings by severity, pack and step with evidence and a fix, and what is new or fixed since the previous run; never a verdict, see [evaluation.md#health](evaluation.md#health) |
+| `get_health_aggregate` | health counted for a dashboard (`run_id`, or `scenario` and `environment` for its newest run; `by` locale, step, check, pack; filters; `top` pages per group): pass / fail / unsure / not evaluated per group, the most affected pages, and new / fixed / still open since the previous run; group ids are stable across runs |
+| `get_health_diff` | per page, locale and check what is new, fixed or still open since the previous run (`state` picks one list), each with `first_seen` (run and date); ids `step\|locale\|check\|page` are stable across runs |
 | `adopt_health_check` | turns a health check into an acceptance criterion of the scenario (`run_id`, `step`, `check` such as `HC103`, `text` to reword it; it becomes `AC103` with `from: HC103`); Developer / Tester role. Only when the person asks; `kaloko health pull` then writes it into the scenario file |
 | `get_affected_steps` | which steps of a scenario changed files reach, with the reason per step (`scenario`, `changed_files`, `run_id`, `upstream`); route files are matched against the step paths and URLs of the newest run. Imports and `affects:` globs need the repository: `kaloko affected` follows them |