kaloko 0.18.0 → 0.20.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.18.0",
+  "version": "0.20.0",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -33,10 +33,28 @@
       "minProperties": 1,
       "additionalProperties": { "$ref": "#/$defs/environment" }
     },
-    "evaluators": {
+    "evaluate": {
       "type": "object",
       "additionalProperties": false,
+      "description": "Settings of every evaluation (kaloko evaluate and the end of kaloko walk).",
+      "properties": {
+        "budget": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Upper bound of one evaluation (#158). Steps go first, then the semantic sample of each stack, chosen by priority (pages failing a deterministic acceptance check, then those checked longest ago) unless the step says semantic_order: walk; what is past the budget stays not_evaluated with a note.",
+          "properties": {
+            "calls": { "type": "integer", "minimum": 0, "description": "Uncached requests to JEV (your key or through Kaloko) per evaluation, escalations included; requests to your own openai_compatible evaluator are not counted. With a budget, screens without semantic criteria get no quick signals unless already cached. kaloko evaluate --budget N overrides it." }
+          }
+        }
+      }
+    },
+    "evaluators": {
+      "type": "object",
+      "description": "Text evaluators (jev, and your own of kind openai_compatible under any other name), the vision evaluator and triage.",
+      "additionalProperties": { "$ref": "#/$defs/openaiEvaluator" },
       "properties": {
+        "text": { "type": "string", "default": "jev", "description": "The evaluator that answers the semantic questions first: jev, or the name of an openai_compatible evaluator here (a local model in Ollama or LM Studio). Its verdicts are marked via: own-model on the canvas, in the PR comment and in acceptance records." },
+        "escalate_to": { "const": "jev", "description": "Answers of your own evaluator (evaluators.text) that are not clear enough for a verdict (between the fail and pass thresholds, or missing) are asked again of JEV (your key or through Kaloko, counted in evaluate.budget). Only your model escalates to JEV, never the other way. Quick signals are not escalated." },
         "jev": {
           "type": "object",
           "additionalProperties": false,
@@ -106,6 +124,17 @@
         }
       }
     },
+    "traffic": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Traffic data per page (#157), opt-in: Search Console clicks, impressions, CTR and position and GA4 sessions and conversions, joined by URL onto stack pages (items.json traffic) and the rows of kaloko export findings (impact, sorted by it); with the seo pack a stack also gets HC127 (high impressions, low CTR). Nothing is requested unless gsc or ga4 is set. A scenario's traffic block replaces the config's.",
+      "properties": {
+        "file": { "oneOf": [{ "type": "string" }, { "type": "array", "minItems": 1, "items": { "type": "string" } }], "description": "CSV exports of Search Console (Pages) or GA4 (landing pages), English or Czech headers; a path relative to the file that names it." },
+        "gsc": { "type": "object", "additionalProperties": false, "required": ["property"], "properties": { "property": { "type": "string", "minLength": 1, "description": "The Search Console property: sc-domain:example.com or https://www.example.com/." }, "credentials_env": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", "description": "Variable with the path of the service account JSON key (or the JSON itself). Default GOOGLE_APPLICATION_CREDENTIALS." }, "days": { "type": "integer", "minimum": 1, "maximum": 480, "description": "Days of data, default 28 (ending two days ago; Search Console data comes in late)." } }, "description": "Search Console API (searchanalytics.query by page) with a service account added to the property as a user. Requested once per run and cached in the run folder .cache." },
+        "ga4": { "type": "object", "additionalProperties": false, "required": ["property"], "properties": { "property": { "type": "string", "minLength": 1, "pattern": "^(properties/)?[0-9]+$", "description": "The GA4 property id (123456789 or properties/123456789)." }, "credentials_env": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", "description": "Variable with the path of the service account JSON key (or the JSON itself). Default GOOGLE_APPLICATION_CREDENTIALS." }, "days": { "type": "integer", "minimum": 1, "maximum": 480, "description": "Days of data, default 28 (ending yesterday)." } }, "description": "GA4 Data API (runReport: sessions and key events by landing page) with a service account added to the property as a viewer. Requested once per run and cached in the run folder .cache." },
+        "low_ctr": { "type": "object", "additionalProperties": false, "description": "HC127 thresholds: a page with at least min_impressions impressions fails when its CTR is below ratio x the site's median CTR at similar positions (1-3, 4-10, 11-20).", "properties": { "min_impressions": { "type": "integer", "minimum": 1, "default": 100 }, "ratio": { "type": "number", "exclusiveMinimum": 0, "maximum": 1, "default": 0.5 } } }
+      }
+    },
     "walk": {
       "type": "object",
       "additionalProperties": false,
@@ -201,6 +230,23 @@
         "images": { "type": "object", "additionalProperties": false, "properties": { "viewport": { "type": "string" }, "locale": { "type": "string", "description": "reader (the reader's language) or a locale." } } }
       }
     },
+    "openaiEvaluator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "base_url", "model"],
+      "description": "Your own text evaluator behind an OpenAI-compatible chat completions API (#158): Ollama (http://localhost:11434/v1), LM Studio (http://localhost:1234/v1), vLLM or a gateway. Used only when evaluators.text or evaluators.escalate_to names it; the page state goes only to base_url.",
+      "properties": {
+        "kind": { "const": "openai_compatible" },
+        "base_url": { "type": "string", "description": "The API root with /v1; requests go to {base_url}/chat/completions." },
+        "model": { "type": "string", "description": "The model as the server names it (qwen3:8b, gemma3:12b)." },
+        "api_key_env": { "type": "string", "description": "Variable with a key sent as a bearer token; a local server needs none." },
+        "concurrency": { "type": "integer", "minimum": 1, "maximum": 16, "default": 2, "description": "Requests in flight at once; a local model usually answers one at a time." },
+        "max_state_chars": { "type": "integer", "minimum": 1000, "default": 24000, "description": "The page text is cut to this length (the top of the page): small models have small context windows." },
+        "timeout_ms": { "type": "integer", "minimum": 1000, "default": 120000 },
+        "temperature": { "type": "number", "minimum": 0, "maximum": 2, "default": 0 },
+        "response_format": { "enum": ["json_schema", "json_object", "none"], "default": "json_schema", "description": "How the answer is held to JSON; json_object or none for servers without structured output." }
+      }
+    },
     "record": {
       "description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
       "oneOf": [

package/schemas/run.v1.json

@@ -441,6 +441,40 @@
         "null"
       ],
       "description": "Version of the application under test (git tag on HEAD or --app-version); documentation captures carry it."
+    },
+    "site": {
+      "type": "object",
+      "description": "Site checks (#152, scenario health.site), once per site and run: robots.txt, sitemaps, llms.txt, http→https and www redirects. Advisory: health shows a finding per failed check (pack site, step site:<host>); they never decide a verdict.",
+      "properties": {
+        "origins": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["origin", "checks"],
+            "properties": {
+              "origin": { "type": "string" },
+              "checked_at": { "type": "string" },
+              "checks": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "required": ["id", "verdict"],
+                  "properties": {
+                    "id": { "type": "string", "description": "HC1101 … HC1107." },
+                    "text": { "type": "string" },
+                    "verdict": { "enum": ["pass", "fail", "not_evaluated"] },
+                    "level": { "enum": ["error", "warning"] },
+                    "evidence": { "type": ["string", "null"] },
+                    "na": { "type": "boolean", "description": "Nothing to check here (no robots.txt to read, no other host name)." },
+                    "details": { "type": "array", "items": { "type": "string" } },
+                    "help": { "type": "object" }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "$defs": {
@@ -665,7 +699,9 @@
               "counts": { "type": "object", "description": "Stack steps: pages per verdict." },
               "evidence": { "type": ["string", "null"] },
               "check": { "type": "object" },
-              "help": { "type": "object" }
+              "help": { "type": "object" },
+              "cross_page": { "type": "boolean", "description": "Stack steps (#151): a check across the stack's pages (HC121–HC126: duplicates, canonicals, hreflang pairs, sitemap pages; HC1005: pages without their own version in a walked locale, with items.variants). counts says how many pages are affected; it cannot be adopted as a criterion." },
+              "urls": { "type": "array", "items": { "type": "string" }, "description": "Cross-page checks: the first affected addresses (at most 20); the stack's items.json has every group." }
             }
           }
         },
@@ -724,6 +760,26 @@
                 "string",
                 "null"
               ]
+            },
+            "rotation": {
+              "type": "object",
+              "description": "Rotating sample (items.rotate), per locale: mode (week or run), week or run, offset, limit, total, rounds (runs to cover every page), covered, never (pages not checked yet) and oldest_checked.",
+              "additionalProperties": { "type": "object" }
+            },
+            "variants": {
+              "type": "object",
+              "description": "Locale variants (items.variants: hreflang): mode, source_locale, and per other locale the number of source pages without their own version there (missing) and of versions several source pages share (shared). items.json lists them under variants.locales.",
+              "properties": {
+                "mode": { "type": "string" },
+                "source_locale": { "type": ["string", "null"] },
+                "missing": { "type": "object", "additionalProperties": { "type": "integer" } },
+                "shared": { "type": "object", "additionalProperties": { "type": "integer" } }
+              }
+            },
+            "cross_page": {
+              "type": "array",
+              "description": "Cross-page checks of a stack with the seo pack (#151), and HC1005 with locale variants: id, verdict, pages compared and pages affected. items.json has them in full under cross_page (groups with the affected addresses) and marks each affected page with cross_fails.",
+              "items": { "type": "object" }
             }
           }
         }

package/schemas/scenario.v1.json

@@ -94,7 +94,34 @@
       "additionalProperties": false,
       "description": "Health (CS Kondice): the checks of packs are advisory, shown apart from acceptance and never part of a verdict. acceptance lists the packs whose checks still count for acceptance in this scenario, as they did before health was split off (the design pack always does). To make one check count, adopt it: kaloko health adopt <step>:<check> (e.g. home:HC103) adds the criterion AC103 with from: HC103.",
       "properties": {
-        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design"] } }
+        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design", "aeo", "locale"] } },
+        "site": {
+          "description": "Site checks (#152), once per site and run: robots.txt, the sitemaps it lists, llms.txt, and http→https and www redirects in one hop. Advisory (health, pack site, HC11xx); off unless set, because they make requests of their own. true for the defaults, or an object.",
+          "oneOf": [
+            { "type": "boolean" },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "paths": { "type": "array", "maxItems": 50, "items": { "type": "string", "pattern": "^/" }, "description": "Paths robots.txt must not block for search engines (default: / and the start path of each walked language)." },
+                "ai_crawlers": { "type": "boolean", "description": "false skips HC1103 (AI crawlers blocked in robots.txt) when blocking them is the site's decision. Default true." },
+                "sitemaps": { "type": "integer", "minimum": 1, "maximum": 50, "description": "Most sitemap files to fetch per site, index children included (default and most 50)." },
+                "environments": { "type": "array", "minItems": 1, "items": { "type": "string" }, "description": "Run the site checks only on these environments (e.g. [production]). Default: every environment; off production HC1102 is a warning and HC1103 is not evaluated." }
+              }
+            }
+          ]
+        }
+      }
+    },
+    "traffic": {
+      "type": "object",
+      "additionalProperties": false,
+      "description": "Traffic data per page (#157), opt-in: Search Console clicks, impressions, CTR and position and GA4 sessions and conversions, joined by URL onto stack pages (items.json traffic) and the rows of kaloko export findings (impact, sorted by it); with the seo pack a stack also gets HC127 (high impressions, low CTR). Nothing is requested unless gsc or ga4 is set. A scenario's traffic block replaces the config's.",
+      "properties": {
+        "file": { "oneOf": [{ "type": "string" }, { "type": "array", "minItems": 1, "items": { "type": "string" } }], "description": "CSV exports of Search Console (Pages) or GA4 (landing pages), English or Czech headers; a path relative to the file that names it." },
+        "gsc": { "type": "object", "additionalProperties": false, "required": ["property"], "properties": { "property": { "type": "string", "minLength": 1, "description": "The Search Console property: sc-domain:example.com or https://www.example.com/." }, "credentials_env": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", "description": "Variable with the path of the service account JSON key (or the JSON itself). Default GOOGLE_APPLICATION_CREDENTIALS." }, "days": { "type": "integer", "minimum": 1, "maximum": 480, "description": "Days of data, default 28 (ending two days ago; Search Console data comes in late)." } }, "description": "Search Console API (searchanalytics.query by page) with a service account added to the property as a user. Requested once per run and cached in the run folder .cache." },
+        "ga4": { "type": "object", "additionalProperties": false, "required": ["property"], "properties": { "property": { "type": "string", "minLength": 1, "pattern": "^(properties/)?[0-9]+$", "description": "The GA4 property id (123456789 or properties/123456789)." }, "credentials_env": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$", "description": "Variable with the path of the service account JSON key (or the JSON itself). Default GOOGLE_APPLICATION_CREDENTIALS." }, "days": { "type": "integer", "minimum": 1, "maximum": 480, "description": "Days of data, default 28 (ending yesterday)." } }, "description": "GA4 Data API (runReport: sessions and key events by landing page) with a service account added to the property as a viewer. Requested once per run and cached in the run folder .cache." },
+        "low_ctr": { "type": "object", "additionalProperties": false, "description": "HC127 thresholds: a page with at least min_impressions impressions fails when its CTR is below ratio x the site's median CTR at similar positions (1-3, 4-10, 11-20).", "properties": { "min_impressions": { "type": "integer", "minimum": 1, "default": 100 }, "ratio": { "type": "number", "exclusiveMinimum": 0, "maximum": 1, "default": 0.5 } } }
       }
     },
     "tags": {
@@ -344,6 +371,10 @@
           "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,39}$", "not": { "enum": ["index"] } },
           "description": "States this step is captured in besides the default one, e.g. [empty, error, loading]; any name works. Each state is one more capture per viewport (keys <viewport>.state-<name>) and a value of the State switcher on the canvas. Design steps render <step>/<state>.html (or <step>/<locale>/<state>.html), else <step>/index.html?state=<name>; walks of web and app steps run the step script once more per state with the argument state. `default` may be listed; it is always captured."
         },
+        "expect_status": {
+          "oneOf": [{ "type": "integer", "minimum": 100, "maximum": 599 }, { "type": "array", "minItems": 1, "items": { "type": "integer", "minimum": 100, "maximum": 599 } }],
+          "description": "The HTTP status this step is expected to answer with, e.g. 404 for a missing page. A status of 400 or more marks an error page: the errors pack checks HC601 and HC605 only on error pages (this, or a 4xx/5xx answer)."
+        },
         "page": {
           "enum": ["landing", "work", "list", "settings", "form", "doc"],
           "description": "What kind of page this step is (see the scenario's page); the usability pack asks per type, e.g. one primary call to action only on landing pages."
@@ -359,10 +390,12 @@
               "console",
               "errors",
               "app",
-              "design"
+              "design",
+              "aeo",
+              "locale"
             ]
           },
-          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
+          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app, HC9xx aeo, HC10xx locale. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
         },
         "capture": {
           "$ref": "#/$defs/capture"
@@ -394,7 +427,7 @@
         "items": {
           "type": "object",
           "additionalProperties": false,
-          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap (with match/exclude), crawl or script.",
+          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap or robots (with match/exclude), crawl or script.",
           "properties": {
             "urls": {
               "type": "array",
@@ -405,19 +438,23 @@
             },
             "file": {
               "type": "string",
-              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path,title}; relative to the scenario"
+              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path, title, key, purpose, edit_url, content_ref}; relative to the scenario"
             },
             "sitemap": {
               "type": "string",
-              "description": "Sitemap path or URL (sitemap indexes are followed one level)"
+              "description": "Sitemap path or URL, gzipped (.gz) or not; sitemap indexes are followed two levels. `robots` reads the Sitemap: lines of /robots.txt (see robots)."
+            },
+            "robots": {
+              "type": "boolean",
+              "description": "The sitemaps listed in /robots.txt (Sitemap: lines); when their URLs name the walked locale (/sitemaps/de/…), only those of that locale"
             },
             "match": {
-              "type": "string",
-              "description": "Regex on the path: keep matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: keep matching pages. A map by locale ({ default: '^/eshop/', fi: '^/fi/tuotteet/' }) gives each locale its own regex (the locale, its language, then default). Applies to every source before rotate and limit."
             },
             "exclude": {
-              "type": "string",
-              "description": "Regex on the path: drop matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: drop matching pages. A string or a map by locale like match."
             },
             "crawl": {
               "type": "object",
@@ -449,7 +486,15 @@
             },
             "script": {
               "type": "string",
-              "description": "Module relative to the scenario: default export async ({ page, env, locale, url }) => [{ path | url, title?, key? }]"
+              "description": "Module relative to the scenario: default export async ({ page, env, locale, url, node, params }) => [{ path | url, title?, key?, purpose?, edit_url?, content_ref? }]. node: { id, title, lane, lane_title, session }; params: items.params. edit_url (the page in the CMS) and content_ref (its record id) show in the stack index and in kaloko export findings."
+            },
+            "resolve": {
+              "type": "string",
+              "description": "Module relative to the scenario that links each walked page to its source: default export async (item, { env, locale, node, params, url }) => ({ edit_url?, content_ref? }). Called for the walked pages only (after match, rotate, limit and walk --items), 15 s per page; a page it fails on is walked without the link."
+            },
+            "params": {
+              "type": "object",
+              "description": "Values passed to items.script as params, so one script serves several stacks (for example { family: magazine })."
             },
             "limit": {
               "type": "integer",
@@ -457,11 +502,27 @@
               "maximum": 10000,
               "description": "At most this many pages (default 500)"
             },
+            "variants": {
+              "enum": ["hreflang"],
+              "description": "Locale variants of one page: the list, limit and rotation come from items.source_locale, and every other locale walks each page's own hreflang alternate for that locale (from the page's HTML head, its Link header or the sitemap's xhtml:link). The item id comes from the source page, so a page and its translations are one item; a page without a version in a locale is reported (HC1005) instead of dropped silently. x-default never stands for a locale."
+            },
+            "source_locale": {
+              "type": "string",
+              "description": "With variants: the locale whose list is walked (default: the scenario's first locale). One of the scenario's locales."
+            },
+            "rotate": {
+              "enum": ["week", "run"],
+              "description": "Rotating sample: each run takes limit pages starting at a stable offset, the ISO week (week) or the number of runs so far in this output folder (run), so the whole list is covered over several runs."
+            },
             "semantic_sample": {
               "type": "integer",
               "minimum": 0,
               "maximum": 10000,
-              "description": "Semantic criteria are evaluated on the first N pages only (default 20); deterministic ones on every page"
+              "description": "Semantic criteria are evaluated on N pages only (default 20), in the order of semantic_order; deterministic ones on every page"
+            },
+            "semantic_order": {
+              "enum": ["walk", "priority"],
+              "description": "Which pages make the semantic sample. walk (default without an evaluator budget): the first N in walk order. priority (default with evaluate.budget): pages failing a deterministic acceptance criterion first, then those never or least recently checked (last_checked of a rotating sample), then walk order."
             },
             "viewports": {
               "type": "array",
@@ -492,7 +553,7 @@
               },
               "from": {
                 "type": "string",
-                "description": "title | h1 | description | canonical | lang | words | http | url | path | a CSS selector (text), selector@attribute, or count:selector"
+                "description": "title | h1 | description | canonical | lang | words | http | url | path | jsonld_types (the @type values of the JSON-LD blocks) | og:title, twitter:card or meta:<name> (a meta tag's content) | a CSS selector (text), selector@attribute, or count:selector"
               }
             }
           }
@@ -649,6 +710,16 @@
             "link_home",
             "links_ok",
             "hreflang_pairs",
+            "llms_txt",
+            "nojs_parity",
+            "jsonld_present",
+            "jsonld_valid",
+            "aeo_faq_schema",
+            "aeo_answer_first",
+            "locale_lang",
+            "locale_hreflang",
+            "locale_text",
+            "locale_prices",
             "app_touch_targets",
             "app_no_truncation",
             "app_no_crash",
@@ -687,6 +758,11 @@
         "attr": {
           "type": "string"
         },
+        "types": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "jsonld_present: schema.org types the page's structured data must contain (Product, BreadcrumbList, FAQPage…)."
+        },
         "property": {
           "type": "string",
           "minLength": 1,
@@ -759,7 +835,13 @@
         "ratio": {
           "type": "number",
           "minimum": 0,
-          "description": "design_pixels: maximum share of differing pixels (default 0.03)."
+          "description": "design_pixels: maximum share of differing pixels (default 0.03); nojs_parity: the least share of the rendered body text the raw HTML must hold (default 0.3)."
+        },
+        "user_agent": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 300,
+          "description": "nojs_parity: fetch the raw HTML as this crawler, without cookies, and respect its robots.txt rules: gptbot, oai-searchbot, claudebot, perplexitybot, googlebot, bingbot, or a whole user agent string. Without it, the walker's own browser user agent and session, and robots.txt is not read."
         },
         "follow": {
           "type": "boolean",
@@ -918,6 +1000,10 @@
           "type": "boolean",
           "description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
         },
+        "screenshot": {
+          "enum": ["full", "thumb"],
+          "description": "thumb keeps only the small card thumbnail (640 px wide) as the step's image: for stacks of hundreds of pages where the checks read the page, not the picture (SEO crawls). Default full."
+        },
         "record": { "$ref": "#/$defs/record" },
         "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
       }

package/skills/kaloko/reference/acceptance.md

@@ -43,8 +43,15 @@ health checks (ids `HC1xx` for `seo`, `HC2xx` for `usability`; acceptance
 criteria are `AC`, health checks `HC`, and the number is the check's: the
 same check kept in acceptance or adopted is `AC103` for `HC103`) at load time:
 `seo` checks one H1, title and description length, canonical, hreflang for
-every locale, Open Graph, indexing, viewport, `html lang`, overflow and
-console; `usability` checks unlabeled inputs and link texts and asks the
+every locale, Open Graph, indexing, viewport, `html lang`, overflow,
+console and structured data (HC112 JSON-LD present, HC113 every block valid);
+`aeo` (HC9xx) adds answer-engine signals: FAQPage/QAPage where the page asks
+and answers, an answer-first paragraph under the H1, `/llms.txt`, a meta
+description, the raw HTML as GPTBot gets it without JavaScript (HC905,
+`nojs_parity`; `user_agent: claudebot|googlebot|…` and `ratio` on a criterion
+of your own); `locale` (HC10xx, scenarios with `locales`) checks html lang,
+the own hreflang, the text's language and price decimals against the locale;
+`usability` checks unlabeled inputs and link texts and asks the
 evaluator questions that fit the step's page type. Pack checks are health,
 not acceptance: they are evaluated on every walk but never decide a step's
 verdict (see [evaluation.md#health](evaluation.md#health)). The `design` pack
@@ -122,13 +129,20 @@ English words; one plain word such as a brand name is left alone).
   - a plain-language message with a way forward (semantic);
   - no console errors. The document's own error status is not counted.
 
+  HC601 and HC605 apply only to error pages: a step with `expect_status`
+  (400 or more, or an own `http_status` criterion of 400 or more) or a page
+  that answered 4xx/5xx. On other pages they are not applicable, so keep the
+  pack off normal pages and stacks (`kaloko validate` warns on a stack).
+
 ## Error pages
 
 Every site has them and they are easy to forget. Give each one its own
 step with `packs: [errors]`:
 
-- **404:** use `path: /no-such-page-{random}`. `{random}` becomes a new
-  token on every walk, so no cache or an accidentally created page makes it
+- **404:** use `path: /no-such-page-{random}` and `expect_status: 404`, so a
+  soft 404 (HTTP 200) fails HC601 instead of being not applicable.
+  `{random}` becomes a new token on every walk, so no cache or an
+  accidentally created page makes it
   pass. `kaloko draft` adds this step on its own.
 - **Exact status:** override HC601 with your own AC601, for example
   `{ id: AC601, text: Answers 410, check: { type: deterministic, assert: http_status, value: 410 } }`.

package/skills/kaloko/reference/evaluation.md

@@ -27,6 +27,48 @@ report with the reason `kaloko evaluate` gives.
 The default evaluator is text-only and most accurate in English. It is weak
 at numbers, which is why those are checked deterministically.
 
+## Budget and your own model
+
+On large sites the semantic questions are what costs money. Three settings keep it in hand:
+
+- **Budget.** `evaluate.budget.calls` in `kaloko.config.yml` (or `kaloko evaluate --budget N`, `kaloko walk
+  --budget N`) caps the uncached JEV calls of one evaluation, the walk's own questions included. Ordinary steps
+  go first, then each stack's sample. What does not fit stays `not_evaluated` with the note "the evaluator
+  budget of N call(s) is used up", and `kaloko evaluate` prints how many captures it left out. Cached answers
+  are free, so the next evaluation continues where this one stopped. With a budget, screens without semantic
+  criteria get no new quick signals. Report the skipped count; never call those criteria passed.
+- **Sample order.** A stack asks the semantic questions on `semantic_sample` pages (default 20). With a budget,
+  or `items.semantic_order: priority` on the step, they are chosen by priority: pages failing a deterministic
+  acceptance criterion first (health checks do not count), then pages never or least recently checked
+  (`last_checked` of a rotating sample), then the rest in walk order. Otherwise, and with `semantic_order: walk`,
+  the first pages in walk order. The same captures always give the same sample and spend the budget on the same
+  pages.
+- **Your own model.** An evaluator of `kind: openai_compatible` (Ollama, LM Studio, vLLM) answers the same
+  questions from the same page text, for free and without the page leaving the machine. It is opt-in:
+  `evaluators.text: <name>` makes it answer first. With `evaluators.escalate_to: jev` its answers that are not
+  clear enough for a verdict (between the thresholds, a missing answer, or anything but a number from 0 to 1)
+  are asked again of JEV, counted in the budget; the quick signals are not escalated. Escalation goes only that
+  way, never from JEV to your model. Small models are rougher judges, so treat their verdicts as a first pass and
+  keep JEV for escalation. After three failed connections in a row an evaluation stops asking your model.
+
+```yaml
+evaluate:
+  budget: { calls: 300 }
+evaluators:
+  ollama:
+    kind: openai_compatible
+    base_url: http://localhost:11434/v1   # LM Studio: http://localhost:1234/v1
+    model: qwen3:8b
+  text: ollama
+  escalate_to: jev
+```
+
+Every text verdict then names who decided it, with `via`, `model` and `prompt_version`: `openai-compatible`
+via `own-model` for your model, `typesafe-jev` for an escalated answer with the local score kept as
+`first_score`. The canvas, acceptance records and the PR comment show your model's verdicts as such; say so in the
+report too, and never present them as JEV's. Pages are not batched into one
+request: each page state is one request, cached by its content, which is what makes re-evaluation free.
+
 ## Vision and reasons
 
 Every semantic verdict carries one sentence, `reason`, shown as "Why" next to
@@ -71,7 +113,8 @@ evaluator saw the screenshot, say so.
 ## Health
 
 The checks of packs (`seo`, `a11y`, `perf`, `usability`, `console`,
-`errors`, `app`) are the run's **health** (CS *kondice*): advisory, stored in
+`errors`, `app`, `aeo`, `locale`), the cross-page checks of stacks with
+`seo` and the site checks of `health.site` are the run's **health** (CS *kondice*): advisory, stored in
 `node.health` and `summary.health` of `run.json`, never in `criteria`. They
 do not change a step's status, the run verdict, the review queue, carried
 approvals, required approvers, the acceptance record or exit codes. The

package/skills/kaloko/reference/integrations.md

@@ -72,13 +72,42 @@ kaloko schedule hook <id>       # URL + secret that start the run after a deploy
   script that only calls `page.goto`). Steps that click or type are left out and
   named in the answer. Criteria decided: `exists`, `absent`, `count`, `text_*`,
   `attr_*`, `title_matches`, `meta_matches`; the rest shows "not evaluated".
+- Stack steps: pages from `items.urls`, `items.sitemap` or `items.robots`
+  (gzip and indexes too) with `match`/`exclude` (string or per locale), `limit`
+  and `rotate: week`, picked like `kaloko walk` picks them. Read without
+  screenshots (one `/scrape` per page in the step's first viewport): the
+  criteria above, index fields (title, h1, description, words, jsonld_types,
+  meta and selector fields; `http` stays empty) and `items.json` on the canvas.
+  Read over several 15-minute ticks (150 pages each), at most 1,000 stack pages
+  per run and 5 × limit sitemap addresses per locale; each page counts against
+  the monthly budget, and a spent budget waits up to a day, then shares with a
+  note. Run now, hooks and edits queue behind a run under way. Patterns: ≤ 300
+  characters, no backreferences or lookaround. `crawl`, `script` and `file` sources, item scripts and
+  `items.resolve` edit links need the full walk.
+- Base URL per locale: an environment with `base_url: { default: …, cs: … }`
+  is scheduled as it is; each locale opens its own address, else `default`.
+  `kaloko schedule list` shows stack progress and the last run's new and fixed
+  health results.
 - Health alerts: a hosted run (and a run shared from CI) compares its health
   with the previous run of the scenario and alerts the followers, the channels
   and the `health.worse` webhook on a new error-level finding. Each schedule
   sets it to `error` (default), `warning` or `off` on kaloko.app or with
   `kaloko schedule add --health-alert`; changing it is the person's call.
+- Health for dashboards: every shared run with health checks gets counts per
+  locale, step, check and pack (`GET …/runs/<id>/health/aggregate`,
+  `…/scenarios/<id>/latest/health/aggregate`) and a diff against the previous
+  run per page (`…/health/diff`: new, fixed, still open, first seen). A
+  `read:health` token reads these and nothing else; the `health.changed`
+  webhook says when something is new or fixed. help/reference/health-api.md.
+- Findings: `GET …/scenarios/<id>/findings` (MCP `list_findings`) groups a
+  scenario's open results by check, site, locale and step, with a status
+  people set (acknowledged, ignored with a reason) and fixed when no page
+  fails. A scenario may open one Jira / Linear issue per new finding
+  (`PUT …/findings/sync { tracker, min_severity }`, Developer / Tester): at
+  most 20 per run, a comment on big changes, closed when fixed. Switching it
+  on or ignoring findings is the team's call, not an agent's.
 - Does not work for: native apps, local or private addresses (localhost, IPs,
-  `.local`), header credentials, base URLs per locale.
+  `.local`), header credentials, stacks listed by crawl, script or file.
 - For those, or for the full walk on a schedule, write a GitHub Actions
   workflow instead: `kaloko schedule export github --scenario <file> --env <name>
   [--every week] [--at 5]`. It names the repository secrets it needs.

package/skills/kaloko/reference/mcp.md

@@ -52,6 +52,9 @@ is null. Projects and products come complete, without pages.
 | `add_sample_run` | adds the sample run (or returns the one there is); needs an upload or full token and the Developer / Tester role |
 | `search_screens` | the screen library: newest screenshot of every step across the organization's flows (`q` words matched against titles, ids, paths, URLs, page title and H1; `project`, `product`, `module`, `scenario`, `locale`, `viewport`, `device`, `status`, `environment`; `source: design` or `all` for design versions; `group: none` for every locale and viewport; `cursor` from `next`), each with a signed image and the canvas link |
 | `get_health` | the advisory health of a run (`run_id`), the newest run of a scenario (`scenario`) or the trend of a project (`project`): findings by severity, pack and step with evidence and a fix, and what is new or fixed since the previous run; never a verdict, see [evaluation.md#health](evaluation.md#health) |
+| `get_health_aggregate` | health counted for a dashboard (`run_id`, or `scenario` and `environment` for its newest run; `by` locale, step, check, pack; filters; `top` pages per group): pass / fail / unsure / not evaluated per group, the most affected pages, and new / fixed / still open since the previous run; group ids are stable across runs |
+| `get_health_diff` | per page, locale and check what is new, fixed or still open since the previous run (`state` picks one list), each with `first_seen` (run and date); ids `step\|locale\|check\|page` are stable across runs |
+| `list_findings` | a scenario's open results grouped into findings (`scenario`, `environment`; filters `locale`, `step`, `check`, `status`): fingerprint `check\|site\|locale\|step`, severity, pages affected, up to 10 sample pages with `edit_url`, first / last seen, status (open, acknowledged, ignored with a reason, fixed) and the tracker issue. Read-only: fix the worst first; acknowledging or ignoring is the team's call |
 | `adopt_health_check` | turns a health check into an acceptance criterion of the scenario (`run_id`, `step`, `check` such as `HC103`, `text` to reword it; it becomes `AC103` with `from: HC103`); Developer / Tester role. Only when the person asks; `kaloko health pull` then writes it into the scenario file |
 | `get_affected_steps` | which steps of a scenario changed files reach, with the reason per step (`scenario`, `changed_files`, `run_id`, `upstream`); route files are matched against the step paths and URLs of the newest run. Imports and `affects:` globs need the repository: `kaloko affected` follows them |