kaloko 0.18.0 → 0.19.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kaloko",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "Your AI agents build; Kaloko walks every screen, puts it on one canvas and records who approved what.",
   "keywords": [
     "qa",

package/schemas/config.v1.json

@@ -33,10 +33,28 @@
       "minProperties": 1,
       "additionalProperties": { "$ref": "#/$defs/environment" }
     },
-    "evaluators": {
+    "evaluate": {
       "type": "object",
       "additionalProperties": false,
+      "description": "Settings of every evaluation (kaloko evaluate and the end of kaloko walk).",
+      "properties": {
+        "budget": {
+          "type": "object",
+          "additionalProperties": false,
+          "description": "Upper bound of one evaluation (#158). Steps go first, then the semantic sample of each stack, chosen by priority (pages failing a deterministic acceptance check, then those checked longest ago) unless the step says semantic_order: walk; what is past the budget stays not_evaluated with a note.",
+          "properties": {
+            "calls": { "type": "integer", "minimum": 0, "description": "Uncached requests to JEV (your key or through Kaloko) per evaluation, escalations included; requests to your own openai_compatible evaluator are not counted. With a budget, screens without semantic criteria get no quick signals unless already cached. kaloko evaluate --budget N overrides it." }
+          }
+        }
+      }
+    },
+    "evaluators": {
+      "type": "object",
+      "description": "Text evaluators (jev, and your own of kind openai_compatible under any other name), the vision evaluator and triage.",
+      "additionalProperties": { "$ref": "#/$defs/openaiEvaluator" },
       "properties": {
+        "text": { "type": "string", "default": "jev", "description": "The evaluator that answers the semantic questions first: jev, or the name of an openai_compatible evaluator here (a local model in Ollama or LM Studio). Its verdicts are marked via: own-model on the canvas, in the PR comment and in acceptance records." },
+        "escalate_to": { "const": "jev", "description": "Answers of your own evaluator (evaluators.text) that are not clear enough for a verdict (between the fail and pass thresholds, or missing) are asked again of JEV (your key or through Kaloko, counted in evaluate.budget). Only your model escalates to JEV, never the other way. Quick signals are not escalated." },
         "jev": {
           "type": "object",
           "additionalProperties": false,
@@ -201,6 +219,23 @@
         "images": { "type": "object", "additionalProperties": false, "properties": { "viewport": { "type": "string" }, "locale": { "type": "string", "description": "reader (the reader's language) or a locale." } } }
       }
     },
+    "openaiEvaluator": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "base_url", "model"],
+      "description": "Your own text evaluator behind an OpenAI-compatible chat completions API (#158): Ollama (http://localhost:11434/v1), LM Studio (http://localhost:1234/v1), vLLM or a gateway. Used only when evaluators.text or evaluators.escalate_to names it; the page state goes only to base_url.",
+      "properties": {
+        "kind": { "const": "openai_compatible" },
+        "base_url": { "type": "string", "description": "The API root with /v1; requests go to {base_url}/chat/completions." },
+        "model": { "type": "string", "description": "The model as the server names it (qwen3:8b, gemma3:12b)." },
+        "api_key_env": { "type": "string", "description": "Variable with a key sent as a bearer token; a local server needs none." },
+        "concurrency": { "type": "integer", "minimum": 1, "maximum": 16, "default": 2, "description": "Requests in flight at once; a local model usually answers one at a time." },
+        "max_state_chars": { "type": "integer", "minimum": 1000, "default": 24000, "description": "The page text is cut to this length (the top of the page): small models have small context windows." },
+        "timeout_ms": { "type": "integer", "minimum": 1000, "default": 120000 },
+        "temperature": { "type": "number", "minimum": 0, "maximum": 2, "default": 0 },
+        "response_format": { "enum": ["json_schema", "json_object", "none"], "default": "json_schema", "description": "How the answer is held to JSON; json_object or none for servers without structured output." }
+      }
+    },
     "record": {
       "description": "Recordings of walked steps (opt-in): true or step records each step from the previous screen to its capture; flow records the whole walk per locale with a chapter per step; failed keeps only steps that fail; false or off turns it off. Uploaded on Business and Enterprise.",
       "oneOf": [

package/schemas/run.v1.json

@@ -441,6 +441,40 @@
         "null"
       ],
       "description": "Version of the application under test (git tag on HEAD or --app-version); documentation captures carry it."
+    },
+    "site": {
+      "type": "object",
+      "description": "Site checks (#152, scenario health.site), once per site and run: robots.txt, sitemaps, llms.txt, http→https and www redirects. Advisory: health shows a finding per failed check (pack site, step site:<host>); they never decide a verdict.",
+      "properties": {
+        "origins": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["origin", "checks"],
+            "properties": {
+              "origin": { "type": "string" },
+              "checked_at": { "type": "string" },
+              "checks": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "required": ["id", "verdict"],
+                  "properties": {
+                    "id": { "type": "string", "description": "HC1101 … HC1107." },
+                    "text": { "type": "string" },
+                    "verdict": { "enum": ["pass", "fail", "not_evaluated"] },
+                    "level": { "enum": ["error", "warning"] },
+                    "evidence": { "type": ["string", "null"] },
+                    "na": { "type": "boolean", "description": "Nothing to check here (no robots.txt to read, no other host name)." },
+                    "details": { "type": "array", "items": { "type": "string" } },
+                    "help": { "type": "object" }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "$defs": {
@@ -665,7 +699,9 @@
               "counts": { "type": "object", "description": "Stack steps: pages per verdict." },
               "evidence": { "type": ["string", "null"] },
               "check": { "type": "object" },
-              "help": { "type": "object" }
+              "help": { "type": "object" },
+              "cross_page": { "type": "boolean", "description": "Stack steps (#151): a check across the stack's pages (HC121–HC126: duplicates, canonicals, hreflang pairs, sitemap pages). counts says how many pages are affected; it cannot be adopted as a criterion." },
+              "urls": { "type": "array", "items": { "type": "string" }, "description": "Cross-page checks: the first affected addresses (at most 20); the stack's items.json has every group." }
             }
           }
         },
@@ -724,6 +760,16 @@
                 "string",
                 "null"
               ]
+            },
+            "rotation": {
+              "type": "object",
+              "description": "Rotating sample (items.rotate), per locale: mode (week or run), week or run, offset, limit, total, rounds (runs to cover every page), covered, never (pages not checked yet) and oldest_checked.",
+              "additionalProperties": { "type": "object" }
+            },
+            "cross_page": {
+              "type": "array",
+              "description": "Cross-page checks of a stack with the seo pack (#151): id, verdict, pages compared and pages affected. items.json has them in full under cross_page (groups with the affected addresses) and marks each affected page with cross_fails.",
+              "items": { "type": "object" }
             }
           }
         }

package/schemas/scenario.v1.json

@@ -94,7 +94,23 @@
       "additionalProperties": false,
       "description": "Health (CS Kondice): the checks of packs are advisory, shown apart from acceptance and never part of a verdict. acceptance lists the packs whose checks still count for acceptance in this scenario, as they did before health was split off (the design pack always does). To make one check count, adopt it: kaloko health adopt <step>:<check> (e.g. home:HC103) adds the criterion AC103 with from: HC103.",
       "properties": {
-        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design"] } }
+        "acceptance": { "type": "array", "uniqueItems": true, "items": { "enum": ["seo", "usability", "a11y", "perf", "console", "errors", "app", "design", "aeo", "locale"] } },
+        "site": {
+          "description": "Site checks (#152), once per site and run: robots.txt, the sitemaps it lists, llms.txt, and http→https and www redirects in one hop. Advisory (health, pack site, HC11xx); off unless set, because they make requests of their own. true for the defaults, or an object.",
+          "oneOf": [
+            { "type": "boolean" },
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "paths": { "type": "array", "maxItems": 50, "items": { "type": "string", "pattern": "^/" }, "description": "Paths robots.txt must not block for search engines (default: / and the start path of each walked language)." },
+                "ai_crawlers": { "type": "boolean", "description": "false skips HC1103 (AI crawlers blocked in robots.txt) when blocking them is the site's decision. Default true." },
+                "sitemaps": { "type": "integer", "minimum": 1, "maximum": 50, "description": "Most sitemap files to fetch per site, index children included (default and most 50)." },
+                "environments": { "type": "array", "minItems": 1, "items": { "type": "string" }, "description": "Run the site checks only on these environments (e.g. [production]). Default: every environment; off production HC1102 is a warning and HC1103 is not evaluated." }
+              }
+            }
+          ]
+        }
       }
     },
     "tags": {
@@ -344,6 +360,10 @@
           "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_-]{0,39}$", "not": { "enum": ["index"] } },
           "description": "States this step is captured in besides the default one, e.g. [empty, error, loading]; any name works. Each state is one more capture per viewport (keys <viewport>.state-<name>) and a value of the State switcher on the canvas. Design steps render <step>/<state>.html (or <step>/<locale>/<state>.html), else <step>/index.html?state=<name>; walks of web and app steps run the step script once more per state with the argument state. `default` may be listed; it is always captured."
         },
+        "expect_status": {
+          "oneOf": [{ "type": "integer", "minimum": 100, "maximum": 599 }, { "type": "array", "minItems": 1, "items": { "type": "integer", "minimum": 100, "maximum": 599 } }],
+          "description": "The HTTP status this step is expected to answer with, e.g. 404 for a missing page. A status of 400 or more marks an error page: the errors pack checks HC601 and HC605 only on error pages (this, or a 4xx/5xx answer)."
+        },
         "page": {
           "enum": ["landing", "work", "list", "settings", "form", "doc"],
           "description": "What kind of page this step is (see the scenario's page); the usability pack asks per type, e.g. one primary call to action only on landing pages."
@@ -359,10 +379,12 @@
               "console",
               "errors",
               "app",
-              "design"
+              "design",
+              "aeo",
+              "locale"
             ]
           },
-          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
+          "description": "Check packs whose checks are added to this node. Health checks (advisory): HC1xx seo, HC2xx usability, HC3xx a11y, HC4xx perf, HC5xx console, HC6xx errors, HC7xx app, HC9xx aeo, HC10xx locale. AC8xx design: checks against the design reference, always acceptance. A pack listed in health.acceptance adds its checks as acceptance criteria with the same number (HC302 → AC302)."
         },
         "capture": {
           "$ref": "#/$defs/capture"
@@ -394,7 +416,7 @@
         "items": {
           "type": "object",
           "additionalProperties": false,
-          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap (with match/exclude), crawl or script.",
+          "description": "Stack steps: which pages of one template the step covers. One source: urls, file, sitemap or robots (with match/exclude), crawl or script.",
           "properties": {
             "urls": {
               "type": "array",
@@ -405,19 +427,23 @@
             },
             "file": {
               "type": "string",
-              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path,title}; relative to the scenario"
+              "description": "Text (one path or URL per line), CSV (first column) or JSON array of paths/URLs or {path, title, key, purpose, edit_url, content_ref}; relative to the scenario"
             },
             "sitemap": {
               "type": "string",
-              "description": "Sitemap path or URL (sitemap indexes are followed one level)"
+              "description": "Sitemap path or URL, gzipped (.gz) or not; sitemap indexes are followed two levels. `robots` reads the Sitemap: lines of /robots.txt (see robots)."
+            },
+            "robots": {
+              "type": "boolean",
+              "description": "The sitemaps listed in /robots.txt (Sitemap: lines); when their URLs name the walked locale (/sitemaps/de/…), only those of that locale"
             },
             "match": {
-              "type": "string",
-              "description": "Regex on the path: keep matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: keep matching pages. A map by locale ({ default: '^/eshop/', fi: '^/fi/tuotteet/' }) gives each locale its own regex (the locale, its language, then default). Applies to every source before rotate and limit."
             },
             "exclude": {
-              "type": "string",
-              "description": "Regex on the path: drop matching pages"
+              "oneOf": [{ "type": "string" }, { "type": "object", "minProperties": 1, "additionalProperties": { "type": "string" } }],
+              "description": "Regex on the path: drop matching pages. A string or a map by locale like match."
             },
             "crawl": {
               "type": "object",
@@ -449,7 +475,15 @@
             },
             "script": {
               "type": "string",
-              "description": "Module relative to the scenario: default export async ({ page, env, locale, url }) => [{ path | url, title?, key? }]"
+              "description": "Module relative to the scenario: default export async ({ page, env, locale, url, node, params }) => [{ path | url, title?, key?, purpose?, edit_url?, content_ref? }]. node: { id, title, lane, lane_title, session }; params: items.params. edit_url (the page in the CMS) and content_ref (its record id) show in the stack index and in kaloko export findings."
+            },
+            "resolve": {
+              "type": "string",
+              "description": "Module relative to the scenario that links each walked page to its source: default export async (item, { env, locale, node, params, url }) => ({ edit_url?, content_ref? }). Called for the walked pages only (after match, rotate, limit and walk --items), 15 s per page; a page it fails on is walked without the link."
+            },
+            "params": {
+              "type": "object",
+              "description": "Values passed to items.script as params, so one script serves several stacks (for example { family: magazine })."
             },
             "limit": {
               "type": "integer",
@@ -457,11 +491,19 @@
               "maximum": 10000,
               "description": "At most this many pages (default 500)"
             },
+            "rotate": {
+              "enum": ["week", "run"],
+              "description": "Rotating sample: each run takes limit pages starting at a stable offset, the ISO week (week) or the number of runs so far in this output folder (run), so the whole list is covered over several runs."
+            },
             "semantic_sample": {
               "type": "integer",
               "minimum": 0,
               "maximum": 10000,
-              "description": "Semantic criteria are evaluated on the first N pages only (default 20); deterministic ones on every page"
+              "description": "Semantic criteria are evaluated on N pages only (default 20), in the order of semantic_order; deterministic ones on every page"
+            },
+            "semantic_order": {
+              "enum": ["walk", "priority"],
+              "description": "Which pages make the semantic sample. walk (default without an evaluator budget): the first N in walk order. priority (default with evaluate.budget): pages failing a deterministic acceptance criterion first, then those never or least recently checked (last_checked of a rotating sample), then walk order."
             },
             "viewports": {
               "type": "array",
@@ -492,7 +534,7 @@
               },
               "from": {
                 "type": "string",
-                "description": "title | h1 | description | canonical | lang | words | http | url | path | a CSS selector (text), selector@attribute, or count:selector"
+                "description": "title | h1 | description | canonical | lang | words | http | url | path | jsonld_types (the @type values of the JSON-LD blocks) | og:title, twitter:card or meta:<name> (a meta tag's content) | a CSS selector (text), selector@attribute, or count:selector"
               }
             }
           }
@@ -649,6 +691,16 @@
             "link_home",
             "links_ok",
             "hreflang_pairs",
+            "llms_txt",
+            "nojs_parity",
+            "jsonld_present",
+            "jsonld_valid",
+            "aeo_faq_schema",
+            "aeo_answer_first",
+            "locale_lang",
+            "locale_hreflang",
+            "locale_text",
+            "locale_prices",
             "app_touch_targets",
             "app_no_truncation",
             "app_no_crash",
@@ -687,6 +739,11 @@
         "attr": {
           "type": "string"
         },
+        "types": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "description": "jsonld_present: schema.org types the page's structured data must contain (Product, BreadcrumbList, FAQPage…)."
+        },
         "property": {
           "type": "string",
           "minLength": 1,
@@ -759,7 +816,13 @@
         "ratio": {
           "type": "number",
           "minimum": 0,
-          "description": "design_pixels: maximum share of differing pixels (default 0.03)."
+          "description": "design_pixels: maximum share of differing pixels (default 0.03); nojs_parity: the least share of the rendered body text the raw HTML must hold (default 0.3)."
+        },
+        "user_agent": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 300,
+          "description": "nojs_parity: fetch the raw HTML as this crawler, without cookies, and respect its robots.txt rules: gptbot, oai-searchbot, claudebot, perplexitybot, googlebot, bingbot, or a whole user agent string. Without it, the walker's own browser user agent and session, and robots.txt is not read."
         },
         "follow": {
           "type": "boolean",
@@ -918,6 +981,10 @@
           "type": "boolean",
           "description": "Screenshot of the whole page (default true; false for stack pages: the first screen only)"
         },
+        "screenshot": {
+          "enum": ["full", "thumb"],
+          "description": "thumb keeps only the small card thumbnail (640 px wide) as the step's image: for stacks of hundreds of pages where the checks read the page, not the picture (SEO crawls). Default full."
+        },
         "record": { "$ref": "#/$defs/record" },
         "trace": { "type": "boolean", "default": true, "description": "Keep a trace per walked step (actions, network requests, console) shown under Technical; false or walk --no-trace turns it off." }
       }

package/skills/kaloko/reference/acceptance.md

@@ -43,8 +43,15 @@ health checks (ids `HC1xx` for `seo`, `HC2xx` for `usability`; acceptance
 criteria are `AC`, health checks `HC`, and the number is the check's: the
 same check kept in acceptance or adopted is `AC103` for `HC103`) at load time:
 `seo` checks one H1, title and description length, canonical, hreflang for
-every locale, Open Graph, indexing, viewport, `html lang`, overflow and
-console; `usability` checks unlabeled inputs and link texts and asks the
+every locale, Open Graph, indexing, viewport, `html lang`, overflow,
+console and structured data (HC112 JSON-LD present, HC113 every block valid);
+`aeo` (HC9xx) adds answer-engine signals: FAQPage/QAPage where the page asks
+and answers, an answer-first paragraph under the H1, `/llms.txt`, a meta
+description, the raw HTML as GPTBot gets it without JavaScript (HC905,
+`nojs_parity`; `user_agent: claudebot|googlebot|…` and `ratio` on a criterion
+of your own); `locale` (HC10xx, scenarios with `locales`) checks html lang,
+the own hreflang, the text's language and price decimals against the locale;
+`usability` checks unlabeled inputs and link texts and asks the
 evaluator questions that fit the step's page type. Pack checks are health,
 not acceptance: they are evaluated on every walk but never decide a step's
 verdict (see [evaluation.md#health](evaluation.md#health)). The `design` pack
@@ -122,13 +129,20 @@ English words; one plain word such as a brand name is left alone).
   - a plain-language message with a way forward (semantic);
   - no console errors. The document's own error status is not counted.
 
+  HC601 and HC605 apply only to error pages: a step with `expect_status`
+  (400 or more, or an own `http_status` criterion of 400 or more) or a page
+  that answered 4xx/5xx. On other pages they are not applicable, so keep the
+  pack off normal pages and stacks (`kaloko validate` warns on a stack).
+
 ## Error pages
 
 Every site has them and they are easy to forget. Give each one its own
 step with `packs: [errors]`:
 
-- **404:** use `path: /no-such-page-{random}`. `{random}` becomes a new
-  token on every walk, so no cache or an accidentally created page makes it
+- **404:** use `path: /no-such-page-{random}` and `expect_status: 404`, so a
+  soft 404 (HTTP 200) fails HC601 instead of being not applicable.
+  `{random}` becomes a new token on every walk, so no cache or an
+  accidentally created page makes it
   pass. `kaloko draft` adds this step on its own.
 - **Exact status:** override HC601 with your own AC601, for example
   `{ id: AC601, text: Answers 410, check: { type: deterministic, assert: http_status, value: 410 } }`.

package/skills/kaloko/reference/evaluation.md

@@ -27,6 +27,48 @@ report with the reason `kaloko evaluate` gives.
 The default evaluator is text-only and most accurate in English. It is weak
 at numbers, which is why those are checked deterministically.
 
+## Budget and your own model
+
+On large sites the semantic questions are what costs money. Three settings keep it in hand:
+
+- **Budget.** `evaluate.budget.calls` in `kaloko.config.yml` (or `kaloko evaluate --budget N`, `kaloko walk
+  --budget N`) caps the uncached JEV calls of one evaluation, the walk's own questions included. Ordinary steps
+  go first, then each stack's sample. What does not fit stays `not_evaluated` with the note "the evaluator
+  budget of N call(s) is used up", and `kaloko evaluate` prints how many captures it left out. Cached answers
+  are free, so the next evaluation continues where this one stopped. With a budget, screens without semantic
+  criteria get no new quick signals. Report the skipped count; never call those criteria passed.
+- **Sample order.** A stack asks the semantic questions on `semantic_sample` pages (default 20). With a budget,
+  or `items.semantic_order: priority` on the step, they are chosen by priority: pages failing a deterministic
+  acceptance criterion first (health checks do not count), then pages never or least recently checked
+  (`last_checked` of a rotating sample), then the rest in walk order. Otherwise, and with `semantic_order: walk`,
+  the first pages in walk order. The same captures always give the same sample and spend the budget on the same
+  pages.
+- **Your own model.** An evaluator of `kind: openai_compatible` (Ollama, LM Studio, vLLM) answers the same
+  questions from the same page text, for free and without the page leaving the machine. It is opt-in:
+  `evaluators.text: <name>` makes it answer first. With `evaluators.escalate_to: jev` its answers that are not
+  clear enough for a verdict (between the thresholds, a missing answer, or anything but a number from 0 to 1)
+  are asked again of JEV, counted in the budget; the quick signals are not escalated. Escalation goes only that
+  way, never from JEV to your model. Small models are rougher judges, so treat their verdicts as a first pass and
+  keep JEV for escalation. After three failed connections in a row an evaluation stops asking your model.
+
+```yaml
+evaluate:
+  budget: { calls: 300 }
+evaluators:
+  ollama:
+    kind: openai_compatible
+    base_url: http://localhost:11434/v1   # LM Studio: http://localhost:1234/v1
+    model: qwen3:8b
+  text: ollama
+  escalate_to: jev
+```
+
+Every text verdict then names who decided it, with `via`, `model` and `prompt_version`: `openai-compatible`
+via `own-model` for your model, `typesafe-jev` for an escalated answer with the local score kept as
+`first_score`. The canvas, acceptance records and the PR comment show your model's verdicts as such; say so in the
+report too, and never present them as JEV's. Pages are not batched into one
+request: each page state is one request, cached by its content, which is what makes re-evaluation free.
+
 ## Vision and reasons
 
 Every semantic verdict carries one sentence, `reason`, shown as "Why" next to
@@ -71,7 +113,8 @@ evaluator saw the screenshot, say so.
 ## Health
 
 The checks of packs (`seo`, `a11y`, `perf`, `usability`, `console`,
-`errors`, `app`) are the run's **health** (CS *kondice*): advisory, stored in
+`errors`, `app`, `aeo`, `locale`), the cross-page checks of stacks with
+`seo` and the site checks of `health.site` are the run's **health** (CS *kondice*): advisory, stored in
 `node.health` and `summary.health` of `run.json`, never in `criteria`. They
 do not change a step's status, the run verdict, the review queue, carried
 approvals, required approvers, the acceptance record or exit codes. The

package/skills/kaloko/reference/integrations.md

@@ -77,6 +77,12 @@ kaloko schedule hook <id>       # URL + secret that start the run after a deploy
   and the `health.worse` webhook on a new error-level finding. Each schedule
   sets it to `error` (default), `warning` or `off` on kaloko.app or with
   `kaloko schedule add --health-alert`; changing it is the person's call.
+- Health for dashboards: every shared run with health checks gets counts per
+  locale, step, check and pack (`GET …/runs/<id>/health/aggregate`,
+  `…/scenarios/<id>/latest/health/aggregate`) and a diff against the previous
+  run per page (`…/health/diff`: new, fixed, still open, first seen). A
+  `read:health` token reads these and nothing else; the `health.changed`
+  webhook says when something is new or fixed. help/reference/health-api.md.
 - Does not work for: native apps, local or private addresses (localhost, IPs,
   `.local`), header credentials, base URLs per locale.
 - For those, or for the full walk on a schedule, write a GitHub Actions

package/skills/kaloko/reference/mcp.md

@@ -52,6 +52,8 @@ is null. Projects and products come complete, without pages.
 | `add_sample_run` | adds the sample run (or returns the one there is); needs an upload or full token and the Developer / Tester role |
 | `search_screens` | the screen library: newest screenshot of every step across the organization's flows (`q` words matched against titles, ids, paths, URLs, page title and H1; `project`, `product`, `module`, `scenario`, `locale`, `viewport`, `device`, `status`, `environment`; `source: design` or `all` for design versions; `group: none` for every locale and viewport; `cursor` from `next`), each with a signed image and the canvas link |
 | `get_health` | the advisory health of a run (`run_id`), the newest run of a scenario (`scenario`) or the trend of a project (`project`): findings by severity, pack and step with evidence and a fix, and what is new or fixed since the previous run; never a verdict, see [evaluation.md#health](evaluation.md#health) |
+| `get_health_aggregate` | health counted for a dashboard (`run_id`, or `scenario` and `environment` for its newest run; `by` locale, step, check, pack; filters; `top` pages per group): pass / fail / unsure / not evaluated per group, the most affected pages, and new / fixed / still open since the previous run; group ids are stable across runs |
+| `get_health_diff` | per page, locale and check what is new, fixed or still open since the previous run (`state` picks one list), each with `first_seen` (run and date); ids `step\|locale\|check\|page` are stable across runs |
 | `adopt_health_check` | turns a health check into an acceptance criterion of the scenario (`run_id`, `step`, `check` such as `HC103`, `text` to reword it; it becomes `AC103` with `from: HC103`); Developer / Tester role. Only when the person asks; `kaloko health pull` then writes it into the scenario file |
 | `get_affected_steps` | which steps of a scenario changed files reach, with the reason per step (`scenario`, `changed_files`, `run_id`, `upstream`); route files are matched against the step paths and URLs of the newest run. Imports and `affects:` globs need the repository: `kaloko affected` follows them |
 

package/skills/kaloko/reference/stacks.md

@@ -17,6 +17,15 @@ sorting, thumbnails and CSV export, and each page opens in the lightbox.
   per page, and export the failing ones for the content team.
 - A redesign or migration: walk the old URL list against the new site.
 
+For a weekly SEO and answer-engine monitor of a public site, start with
+`npx kaloko init --seo <site>`: it reads robots.txt and the sitemaps, finds
+the languages and the biggest content families and writes
+`qa/flows/seo-<site>.yml` with one rotating stack per family (`match` per
+language, packs seo and aeo). Read the comment at the top of the file:
+families it paired across languages by size are guesses to check. Then
+`kaloko validate`, `kaloko walk --limit 20`, and
+`kaloko schedule export github`. See help/how-to/seo-monitor.md.
+
 Not for a flow of different screens: those stay separate steps. A stack
 step does not click through anything. It opens each page by URL. Use a
 step script only to prepare each page, for example to dismiss a dialog
@@ -31,10 +40,13 @@ step script only to prepare each page, for example to dismiss a dialog
   purpose: Every article reads well and is findable.
   packs: [seo]                              # health checks of every page (advisory)
   items:
-    sitemap: /sitemap.xml                   # or urls / file / crawl / script (exactly one)
+    sitemap: /sitemap.xml                   # or robots: true / urls / file / crawl / script (exactly one)
     match: '^/clanky/'                      # regex on the path with its query; exclude: '…' drops pages
+                                            # or a map by locale: { default: '^/blog/', fi: '^/fi/blogi/' }
     limit: 800                              # default 500, at most 10000
-    semantic_sample: 20                     # semantic criteria on the first 20 pages only
+    # rotate: week                          # each ISO week the next `limit` pages (run: each run)
+    semantic_sample: 20                     # semantic criteria on 20 pages only
+    # semantic_order: priority              # failing and least recently checked pages first
     # viewports: [desktop]                  # default: the first viewport of the scenario
   fields:                                   # index columns (max 12); default title, h1, words, http
     - { id: title, label: Title, from: title }
@@ -55,12 +67,43 @@ Item sources:
 - `file: qa/data/products.csv` is a list next to the scenario. Accepted
   formats are text with one path per line, CSV (first column), or JSON:
   `["/a"]` or `[{ path, title, key, purpose }]`, where `key` gives a stable id.
-- `sitemap: /sitemap.xml` also follows a sitemap index (up to 50 sitemaps).
+- `sitemap: /sitemap.xml` also follows a sitemap index, two levels deep
+  (up to 50 sitemaps each). Gzipped sitemaps work: a `.gz` URL or gzip
+  bytes behind any URL are unpacked.
+- `sitemap: robots` (or `robots: true`) reads the `Sitemap:` lines of
+  `/robots.txt` on the environment's origin. When their URLs name the
+  walked locale as a segment (`/sitemaps/de/sitemap.xml.gz`,
+  `sitemap-de.xml`; `en_US` finds `en-us`, `en` does not), only that
+  locale's sitemaps are read; otherwise all of them. Use it when a site
+  lists one sitemap per language, so the scenario does not hard-code paths.
 - `crawl: { from: /clanky, links: "a.card", next: "a.next", pages: 20 }`
   collects links from a listing and follows its pagination.
 - `script: qa/steps/items.mjs` uses the default export
-  `({ page, env, locale, url }) => [paths | { path, title, key, purpose }]` for
-  anything else, such as an API or a search.
+  `({ page, env, locale, url, node, params }) => [paths | { path, title, key, purpose }]`
+  for anything else, such as an API or a search. `node` is
+  `{ id, title, lane, lane_title, session }` of the step and `params` is
+  `items.params`, so one script can serve several stacks.
+
+Fix handoff: a script or a JSON list may give each page `edit_url` (the
+page in the CMS, http(s) only) and `content_ref` (its record id for a
+content API). `resolve: steps/resolve.mjs` (default export
+`async (item, { env, locale, node, params, url }) => ({ edit_url, content_ref })`)
+looks them up for the walked pages only, 15 s per page; a page it fails on is walked
+without them and the walk says how many. They show in the page detail
+("Edit in CMS"), the stack CSV, `get_stack_items` and
+`kaloko export findings`.
+
+`match` and `exclude` apply to every source before `limit` (and before
+`rotate`). Either is a regex or a map by locale; the walked locale, then its
+language (`en` for `en_US`), then `default` is used. A `match` map without
+the locale and without `default` is an error.
+
+`rotate: week` (or `rotate: run`) walks a big list in turns: each ISO week
+(or each run) takes the next `limit` pages, wrapping around, so the list is
+covered in `pages ÷ limit` rounds. `run` keeps its count and each page's last
+check in `.cache/stack/` of the output folder, so use `week` on CI with a
+fresh checkout. The stack index gets `rotation` per locale (window, covered,
+never checked) and each page `last_checked` (its previous check, or null).
 
 An item's `purpose` (for example its title and meta description) is what the
 semantic evaluator reads as `screen_purpose` for that page instead of the
@@ -73,11 +116,44 @@ in the wrong language.
 Pages are deduplicated. External URLs are allowed (e.g. an old domain).
 `{locale}` in any source string is replaced by the walked locale. This is
 how one sitemap with every language is split:
-`match: '^/guides/.*lang={locale}'`, or `sitemap: /{locale}/sitemap.xml`.
+`match: '^/guides/.*lang={locale}'`, or `sitemap: /{locale}/sitemap.xml`,
+or `sitemap: /sitemaps/{locale}/sitemap.xml.gz`. One content family per
+stack: the same per-locale sitemap with `match: '/eshop/'` in one step and
+`match: '/(magazin|magazine|revista)/'` in another.
 
 Field sources: `title`, `h1`, `description`, `canonical`, `lang`, `url`,
 `path`, `words` (words in main/article/body), `http` (the navigation
-status), `count:<selector>`, `<selector>` (text) and `<selector>@<attr>`.
+status), `jsonld_types` (the schema.org types of the page's JSON-LD, for
+example `Product, Offer, BreadcrumbList`; `invalid×1` when a block does not
+parse), a meta tag by name (`og:title`, `og:image`, `twitter:card`,
+`meta:robots`), `count:<selector>`, `<selector>` (text) and
+`<selector>@<attr>`.
+
+Structured data is also summarised without a field: every capture records
+`structured_data` (blocks, invalid blocks, types), and `items.json` carries
+`structured_data` per page and, at the top, pages per type, pages without
+structured data and pages with invalid JSON-LD. The `seo` pack checks it
+(HC112 present, HC113 valid); `packs: [aeo]` adds the answer-engine signals
+(HC901–HC905; HC905 `nojs_parity` fetches each page's raw HTML as GPTBot,
+respecting robots.txt, and fails when the title, canonical, H1, JSON-LD or
+most of the text appear only after JavaScript). See
+help/reference/checks-and-packs.md.
+
+With the `seo` pack a stack also compares its pages with each other after the
+walk (HC121–HC126, advisory): shared titles, descriptions and H1s within a
+language, canonicals pointing elsewhere or to error/noindex pages, hreflang
+that the other walked version does not answer, and (sitemap or robots
+sources) listed pages that answer non-200, redirect or say noindex. They are
+health entries of the stack node (`cross_page: true`, `counts` in pages,
+`urls`); `items.json` has them under `cross_page` with the groups, and each
+affected page lists the ids in `cross_fails`. They cannot be adopted.
+`packs: [locale]` (scenarios with `locales`) checks html lang, the own
+hreflang, the text's language and price decimals against the walked locale
+(HC1001–HC1004). `health: { site: true }` on the scenario adds site checks
+once per run (robots.txt, sitemaps, llms.txt, http→https and www redirects;
+HC1101–HC1107, `site.json` in the run, `site` in the manifest). Off the
+`production` environment HC1102 is a warning and HC1103 is skipped;
+`health.site.environments: [production]` runs them there only.
 
 A step `execution.script` still works. It is called with `item` in the
 context, opens `item.url` itself and may call `capture()`. Without a script,
@@ -86,19 +162,28 @@ the walk opens each page and captures it.
 ## Cost and size: choose deliberately
 
 - Deterministic criteria and packs run on every page. They are cheap.
-- Semantic criteria go to the evaluator. On a stack they run only on the
-  first `semantic_sample` pages (default 20). The other pages show the
-  criterion as not evaluated. Raise the sample only when the question
+- Semantic criteria go to the evaluator. On a stack they run only on
+  `semantic_sample` pages (default 20), the first in walk order. With
+  `semantic_order: priority`, or an evaluator budget, pages failing a
+  deterministic acceptance criterion come first, then those never or least
+  recently checked. The other pages show the criterion as not evaluated. `evaluate.budget.calls`
+  caps the calls of a whole evaluation (evaluation.md). Raise the sample only when the question
   really differs between pages, and tell the person what it costs
   (one call per page and viewport).
 - Stacks capture the first screen of each page (fast, small). Set
   `capture: { full_page: true }` when the criteria are about the whole page.
 - Walking takes about 1–2 s per page. For a first look, use
   `kaloko walk --limit 50`. Re-walk chosen pages with
-  `--items clanek-7,clanek-12` (matches the id or the path).
-- A shared run holds at most 20000 files and 300 MB. Each page stores about
-  four files, so a stack of a few thousand first-screen pages fits. Split
-  larger catalogues by section (`match`), one stack per section.
+  `--items clanek-7,clanek-12` (matches the id or the path; picked from
+  every matching page, also outside this week's rotation, and such a
+  partial walk leaves the rotation state alone).
+- A shared run holds at most 20000 files and 300 MB as stored (HTML and JSON
+  travel gzip-compressed, about a tenth of their size on disk). Each page
+  stores about four files, so a stack of a few thousand first-screen pages
+  fits. For SEO crawls where the checks read the page, not the picture, set
+  `capture: { screenshot: thumb }`: only the 640 px thumbnail is kept. Split
+  larger catalogues by section (`match`) or walk a rotating sample
+  (`rotate: week`).
 
 ## Reading the results
 
@@ -118,6 +203,11 @@ the walk opens each page and captures it.
   `get_stack_items {run_id, step, status?, criterion?, query?, offset, limit}`
   lists pages with fields, failing criteria, evidence and screenshot URLs.
   This is the way to hand an agent "fix these 37 articles".
+- `kaloko export findings [--format csv|json] [--locale l] [--step s] [--out file]`
+  writes the fix list of the local run: one row per failing criterion or
+  health check, page and locale, with URL, evidence, `edit_url`,
+  `content_ref` and `recheck` (`kaloko walk --steps s --items <page> --locales l`,
+  which re-walks just that page after the fix).
 
 ## Report