kaloko 0.4.8 → 0.4.9

package/src/lib/vision-core.mjs

@@ -0,0 +1,132 @@
+// Vision evaluator (#17), shared by the CLI and the service: which criteria go to it, the request for a
+// vision-capable model (Anthropic Messages API with image input and a strict JSON answer), the parsed answer, and the
+// one-sentence reason every semantic verdict carries. Pure: no Node APIs (the worker imports it).
+//
+// The default text evaluator (JEV) reads a reduced page state and returns a probability only; it takes no images
+// and gives no reasons. This second provider sees the screenshot and explains itself. It is asked only for criteria
+// marked `vision: true` and for text answers in the uncertain band, so the cost stays bounded.
+
+export const VISION_PROMPT_VERSION = 'kaloko-vision-v1'
+export const VISION_DEFAULTS = {
+  provider: 'anthropic', model: 'claude-opus-5-5', api_key_env: 'ANTHROPIC_API_KEY', base_url: 'https://api.anthropic.com',
+  when: 'uncertain', uncertain: [0.15, 0.85], max_calls: 40, max_width: 1000, tile_height: 1400, max_tiles: 2,
+  effort: 'low', reason_language: 'en', hosted: true, concurrency: 3
+}
+/** Models the service accepts for hosted vision questions. */
+export const VISION_MODEL = /^claude-[a-z0-9-]{3,40}$/
+/** One hosted vision question counts as this many questions of the monthly allowance. */
+export const VISION_WEIGHT = 5
+export const MAX_VISION_IMAGES = 5
+// models that take the server-side refusal fallback ("default" routing)
+const FALLBACK_MODELS = new Set(['claude-opus-5-5', 'claude-opus-5', 'claude-fable-5-1', 'claude-sonnet-5-5'])
+
+/** Vision settings from the config (`evaluators.vision`; false or when: off turns it off). */
+export function visionSettings (config) {
+  const v = config?.evaluators?.vision
+  if (v === false || v === null) return null
+  const s = { ...VISION_DEFAULTS, ...(v || {}) }
+  return s.when === 'off' ? null : s
+}
+
+/**
+ * The semantic criteria of one capture that go to the vision evaluator: flagged ones (check.vision) always,
+ * every one with when: always, and with when: uncertain those whose text score lies strictly inside the band.
+ * Console triage stays with the text evaluator.
+ */
+export function visionCriteria ({ criteria, answers = {}, settings }) {
+  if (!settings) return []
+  const [lo, hi] = settings.uncertain || VISION_DEFAULTS.uncertain
+  return criteria.filter((c) => {
+    if (c.check?.type !== 'semantic' || c.check.about === 'console') return false
+    if (c.check.vision === true) return true
+    if (settings.when === 'always') return true
+    if (settings.when !== 'uncertain') return false
+    const s = answers?.[c.id]?.noul
+    return s != null && s > lo && s < hi
+  })
+}
+
+const SYSTEM = 'You check web pages captured during acceptance tests. You see screenshots of one page and answer yes/no questions about it. Judge only what is visible in the images and the stated facts; do not guess about content you cannot see. For each question give the probability (0 to 1) that the answer is yes, and one short sentence that names what in the screenshot decided it.'
+
+/**
+ * The Messages API request body for one capture.
+ * questions: [{ id, question, true?, false?, image? }] (image: label of the crop meant for this question)
+ * images: [{ label, media_type, data }] (base64, at most MAX_VISION_IMAGES)
+ */
+export function buildVisionRequest ({ model, effort = 'low', context = {}, questions, images, reasonLanguage = 'en' }) {
+  const content = []
+  for (const img of images.slice(0, MAX_VISION_IMAGES)) {
+    content.push({ type: 'text', text: `Image "${img.label}"` })
+    content.push({ type: 'image', source: { type: 'base64', media_type: img.media_type, data: img.data } })
+  }
+  const facts = [
+    context.purpose ? `What this screen must achieve: ${context.purpose}` : null,
+    context.url ? `URL: ${context.url}` : null,
+    context.locale ? `Expected language: ${context.locale}` : null,
+    context.viewport ? `Viewport: ${context.viewport}` : null
+  ].filter(Boolean)
+  const list = questions.map((q) => ({
+    id: q.id,
+    question: q.question,
+    ...(q.true ? { counts_as_yes: q.true } : {}),
+    ...(q.false ? { counts_as_no: q.false } : {}),
+    look_at: q.image ? `image "${q.image}" (the part of the page the question is about)` : 'the full-page images'
+  }))
+  content.push({ type: 'text', text: `${facts.join('\n')}\n\nQuestions (JSON):\n${JSON.stringify(list, null, 1)}\n\nAnswer every question once, by id. "reason": one sentence of at most 25 words${reasonLanguage && reasonLanguage !== 'en' ? ` in the language with code "${reasonLanguage}"` : ''}, saying what you saw.` })
+  const body = {
+    model,
+    max_tokens: 4096,
+    system: SYSTEM,
+    messages: [{ role: 'user', content }],
+    output_config: {
+      effort,
+      format: {
+        type: 'json_schema',
+        schema: {
+          type: 'object',
+          properties: { answers: { type: 'array', items: { type: 'object', properties: { id: { type: 'string' }, probability: { type: 'number' }, reason: { type: 'string' } }, required: ['id', 'probability', 'reason'], additionalProperties: false } } },
+          required: ['answers'],
+          additionalProperties: false
+        }
+      }
+    }
+  }
+  // a declined request is re-run on a fallback model in the same call instead of ending without answers
+  if (FALLBACK_MODELS.has(model)) body.fallbacks = 'default'
+  return body
+}
+
+/** Headers of the Messages API call (beta header only with the refusal fallback). */
+export function visionHeaders (apiKey, body) {
+  return { 'x-api-key': apiKey, 'anthropic-version': '2023-06-01', 'content-type': 'application/json', ...(body.fallbacks ? { 'anthropic-beta': 'server-side-fallback-2026-07-01' } : {}) }
+}
+
+const sentence = (s) => { const t = String(s || '').replace(/\s+/g, ' ').trim(); const m = t.match(/^.{10,300}?[.!?](?=\s|$)/); return (m ? m[0] : t).slice(0, 300) }
+
+/** { model, answers: { id: { probability, reason } }, usage } from a Messages API response; throws on refusal. */
+export function parseVisionResponse (resp, ids = null) {
+  if (resp?.stop_reason === 'refusal') throw new Error(`the vision evaluator declined (${resp.stop_details?.category || 'refusal'})`)
+  const text = (resp?.content || []).filter((b) => b.type === 'text').map((b) => b.text).join('')
+  let parsed
+  try { parsed = JSON.parse(text) } catch { throw new Error(`the vision evaluator did not answer in JSON${resp?.stop_reason === 'max_tokens' ? ' (cut off)' : ''}`) }
+  const answers = {}
+  for (const a of parsed?.answers || []) {
+    if (!a || typeof a.id !== 'string' || (ids && !ids.includes(a.id))) continue
+    const p = Number(a.probability)
+    if (!Number.isFinite(p)) continue
+    answers[a.id] = { probability: Math.round(Math.min(1, Math.max(0, p)) * 1000) / 1000, reason: sentence(a.reason) }
+  }
+  return { model: resp.model || null, answers, usage: resp.usage ? { input_tokens: resp.usage.input_tokens ?? null, output_tokens: resp.usage.output_tokens ?? null } : null }
+}
+
+/**
+ * The one sentence stored with a semantic verdict when no model explained it (the text evaluator returns a
+ * probability only): what was asked of what, and how sure the answer was.
+ */
+export function templateReason ({ score, scope = null, about = null, verdict = null }) {
+  if (score == null) return null
+  const p = Math.round(score * 100)
+  const what = about === 'console' ? 'the console errors' : scope ? `the text of ${scope}` : 'the page text'
+  const sure = verdict === 'pass' || verdict === 'fail' ? 'clear enough for a verdict' : 'not sure either way'
+  return `Judged from ${what} without the screenshot: ${p} % yes, ${sure}.`
+}

package/src/lib/vision.mjs

@@ -0,0 +1,127 @@
+// Vision evaluator calls from the CLI (#17): screenshots scaled and cut in a browser page (WebP and PNG decode
+// there, no native dependency), one request per capture with the criteria that need it, cached by content like
+// the text evaluator. With ANTHROPIC_API_KEY (or the configured env) the call goes to the provider directly;
+// without it, through the organization's service, which counts it against the monthly allowance.
+import { join } from 'node:path'
+import { readFile } from 'node:fs/promises'
+import { readJson, writeJson, sha256 } from './util.mjs'
+import { buildVisionRequest, parseVisionResponse, visionHeaders, VISION_PROMPT_VERSION, MAX_VISION_IMAGES } from './vision-core.mjs'
+
+/**
+ * Images for one capture: tiles of the full page from the top (when a question needs the whole page) and a crop per
+ * scoped criterion (its highlight box, padded). Runs in `page` (a blank browser page).
+ * crops: [{ id, box: { x, y, w, h } }] in CSS px of the captured page; cssWidth: the page width in CSS px.
+ * @returns {Promise<{ images: Array<{ label, media_type, data, id? }>, failed: string[] }>} failed: crops that could not be cut (their
+ *   questions look at the full page)
+ */
+export async function prepareImages ({ page, file, cssWidth, crops = [], full = true, settings }) {
+  const src = `data:image/${file.endsWith('.png') ? 'png' : 'webp'};base64,${(await readFile(file)).toString('base64')}`
+  return page.evaluate(async ({ src, cssWidth, crops, full, maxW, tileH, maxTiles, maxImages }) => {
+    const img = await new Promise((ok, no) => { const i = new Image(); i.onload = () => ok(i); i.onerror = () => no(new Error('the screenshot could not be decoded')); i.src = src })
+    const k = img.naturalWidth / (cssWidth || img.naturalWidth)
+    const out = []
+    const encode = (c) => c.toDataURL('image/webp', 0.82).split(',')[1]
+    const draw = (sx, sy, sw, sh, dw, dh) => { const c = document.createElement('canvas'); c.width = Math.max(1, Math.round(dw)); c.height = Math.max(1, Math.round(dh)); const g = c.getContext('2d'); g.fillStyle = '#fff'; g.fillRect(0, 0, c.width, c.height); g.drawImage(img, sx, sy, sw, sh, 0, 0, c.width, c.height); return encode(c) }
+    const failed = []
+    for (const { id, box } of crops) {
+      if (out.length >= maxImages - (full ? maxTiles : 0)) { failed.push(id); continue }
+      const pad = 24
+      const sx = Math.max(0, (box.x - pad) * k); const sy = Math.max(0, (box.y - pad) * k)
+      const sw = Math.min(img.naturalWidth - sx, (box.w + pad * 2) * k); let sh = Math.min(img.naturalHeight - sy, (box.h + pad * 2) * k)
+      if (sw < 4 || sh < 4) { failed.push(id); continue }
+      const s = Math.min(1, maxW / sw)
+      if (sh * s > tileH * 1.2) sh = (tileH * 1.2) / s // a very tall region: its top part
+      out.push({ id, label: `crop ${id}`, media_type: 'image/webp', data: draw(sx, sy, sw, sh, sw * s, sh * s) })
+    }
+    if (full || failed.length) {
+      const s = Math.min(1, maxW / img.naturalWidth)
+      const H = img.naturalHeight * s
+      const parts = Math.ceil(H / tileH)
+      const tiles = Math.min(maxTiles, parts, maxImages - out.length)
+      const list = []
+      for (let t = 0; t < tiles; t++) {
+        const h = Math.min(tileH, H - t * tileH)
+        const label = parts === 1 ? 'page' : `page, part ${t + 1} of ${parts} from the top${t === tiles - 1 && tiles < parts ? ' (the rest of the page is not shown)' : ''}`
+        list.push({ label, media_type: 'image/webp', data: draw(0, (t * tileH) / s, img.naturalWidth, h / s, img.naturalWidth * s, h) })
+      }
+      out.unshift(...list)
+    }
+    return { images: out, failed }
+  }, { src, cssWidth, crops, full, maxW: settings.max_width, tileH: settings.tile_height, maxTiles: settings.max_tiles, maxImages: MAX_VISION_IMAGES })
+}
+
+/**
+ * The crops and questions for one capture: a scoped criterion (or one with a highlight) gets the crop of its first
+ * box unless that box is most of the page; the rest look at the full page.
+ */
+export function visionPlan ({ criteria, capture }) {
+  const page = capture.size ? capture.size.width * capture.size.height : 0
+  const crops = []; const questions = []
+  for (const c of criteria) {
+    const box = (c.check.scope || c.highlight) ? capture.boxes?.[c.id]?.[0] : null
+    const useCrop = box && box.w > 0 && box.h > 0 && (!page || (box.w * box.h) / page < 0.6)
+    if (useCrop) crops.push({ id: c.id, box })
+    questions.push({ id: c.id, question: c.check.question, ...(c.check.true ? { true: c.check.true } : {}), ...(c.check.false ? { false: c.check.false } : {}), ...(useCrop ? { image: `crop ${c.id}` } : {}) })
+  }
+  return { crops, questions, full: questions.some((q) => !q.image) }
+}
+
+/**
+ * Asks the vision evaluator with a content-addressed cache. Returns { answers, model, usage, cached, via } or
+ * { skipped } / { error }. `hosted(payload)` sends through the service when no local key is set; `allowCall()` is asked
+ * before every uncached request (false: skipped).
+ */
+export async function askVision ({ payload, settings, cacheDir, fetchImpl = fetch, hosted = null, allowCall = () => true }) {
+  const body = buildVisionRequest({ model: settings.model, effort: settings.effort, context: payload.context, questions: payload.questions, images: payload.images, reasonLanguage: settings.reason_language })
+  const key = sha256(JSON.stringify(body) + VISION_PROMPT_VERSION)
+  const cacheFile = join(cacheDir, `${key}.json`)
+  const cached = await readJson(cacheFile, null)
+  if (cached) return { ...cached, cached: true, key }
+  const apiKey = process.env[settings.api_key_env]
+  // uncached requests are bounded per evaluation (max_calls); the rest keep the text answer
+  if ((apiKey || (hosted && settings.hosted !== false)) && !allowCall()) return { skipped: `vision call limit reached (max_calls ${settings.max_calls})` }
+  if (!apiKey && hosted && settings.hosted !== false) {
+    const r = await hosted({ model: settings.model, effort: settings.effort, reason_language: settings.reason_language, ...payload })
+    if (r.skipped || r.error) return r
+    const out = { answers: r.response.answers || {}, model: r.response.model || settings.model, usage: r.response.usage || null, via: 'qawalk', hosted: r.response.qawalk || null }
+    await writeJson(cacheFile, out)
+    return { ...out, cached: !!r.response.qawalk?.cached, key }
+  }
+  if (!apiKey) return { skipped: `${settings.api_key_env} is not set${hosted ? '' : ' and the service is not configured (KALOKO_TOKEN)'}` }
+  const ids = payload.questions.map((q) => q.id)
+  for (let attempt = 0; attempt < 4; attempt++) {
+    let res
+    try {
+      res = await fetchImpl(`${settings.base_url.replace(/\/$/, '')}/v1/messages`, { method: 'POST', headers: visionHeaders(apiKey, body), body: JSON.stringify(body) })
+    } catch (error) { return { error: `vision evaluator unreachable: ${error.message}` } }
+    if (res.status === 429 || res.status === 529 || res.status === 503) {
+      const wait = Number(res.headers.get('retry-after')) * 1000 || 1000 * 2 ** attempt
+      await new Promise((r) => setTimeout(r, Math.min(wait, 20000)))
+      continue
+    }
+    if (!res.ok) return { error: `vision evaluator HTTP ${res.status}: ${(await res.text()).slice(0, 300)}` }
+    let parsed
+    try { parsed = parseVisionResponse(await res.json(), ids) } catch (error) { return { error: error.message } }
+    const out = { ...parsed, model: parsed.model || settings.model, via: 'own-key' }
+    await writeJson(cacheFile, out)
+    return { ...out, cached: false, key }
+  }
+  return { error: 'vision evaluator rate limited after 4 attempts' }
+}
+
+/**
+ * Sends a vision payload through the organization's service. An allowance used up or a service without hosted
+ * vision stops further hosted vision calls for this evaluation with one message.
+ */
+export function hostedVisionSender (config, report) {
+  let client = null
+  return async (payload) => {
+    if (report.visionStop) return { skipped: report.visionStop }
+    try { if (!client) { const { service } = await import('./service.mjs'); client = service(config) } } catch { return { skipped: 'no vision key (ANTHROPIC_API_KEY) and no service token (KALOKO_TOKEN) for hosted evaluation' } }
+    try { return { response: await client.request('POST', `/orgs/${config.org}/semantic/vision`, payload) } } catch (e) {
+      const m = String(e.message || e)
+      if (/HTTP (402|403|404|503)/.test(m)) { report.visionStop = m.replace(/^.*HTTP \d+: /, ''); return { skipped: report.visionStop } }
+      return { error: m }
+    }
+  }
+}

package/templates/kaloko.config.yml

@@ -69,6 +69,12 @@ evaluators:
     model: jev-1.13.0
     api_key_env: TYPESAFE_API_KEY
     thresholds: { pass: 0.85, fail: 0.15 }
+  # second look with the screenshot for criteria with vision: true and unsure text answers (false: off)
+  vision:
+    model: claude-opus-5-5
+    api_key_env: ANTHROPIC_API_KEY
+    when: uncertain
+    max_calls: 40
 
 share:
   url: https://kaloko.app