@uibot/vlm-scorer 0.0.0-stage → 0.2.1

package/dist/handoff-visual-critique.d.ts

@@ -0,0 +1,63 @@
+import type { ModelGenerateRequest, ModelGenerateResult, PrototypeSpec, VisualEvaluationEvidence } from "@uibot/contracts";
+import { type RenderScreenshotOptions } from "./render-screenshot.js";
+/**
+ * Generate adapter. The optional `context` argument carries non-serializable
+ * runtime concerns (currently just `signal` for AbortController-based
+ * cancellation). Honoring it is best-effort: providers that ignore the signal
+ * still work, but the in-flight HTTP request will keep running after timeout.
+ */
+export interface ModelGenerateFnContext {
+    signal?: AbortSignal;
+}
+export type ModelGenerateFn = (req: ModelGenerateRequest, context?: ModelGenerateFnContext) => Promise<ModelGenerateResult>;
+export type HandoffVisualCritiqueSource = {
+    kind: "prototype";
+    prototypeId: string;
+    prototype: PrototypeSpec;
+} | {
+    kind: "html" | "deck";
+    html: string;
+    title?: string;
+    designMdId?: string;
+};
+export interface HandoffVisualCritiqueInput {
+    intent: string;
+    draftYaml: string;
+    source: HandoffVisualCritiqueSource;
+}
+export type HandoffVisualCritiqueResult = {
+    status: "scored";
+    evidence?: VisualEvaluationEvidence;
+    score: number;
+    reasoning?: string;
+    raw?: unknown;
+} | {
+    status: "unavailable";
+    evidence?: VisualEvaluationEvidence;
+    reason?: string;
+};
+export type HandoffRenderHtmlFn = (input: HandoffVisualCritiqueInput) => Promise<string> | string;
+export interface CreateHandoffVisualCritiqueProviderOptions {
+    generate: ModelGenerateFn;
+    model: string;
+    /**
+     * Optional preview renderer. When supplied, the provider screenshots the
+     * rendered HTML and sends an image block to the VLM. When omitted, the VLM
+     * scores the intent + YAML text only.
+     */
+    renderHtml?: HandoffRenderHtmlFn;
+    screenshot?: RenderScreenshotOptions;
+    /** Shared deadline for HTML preparation, screenshot capture and model assessment. */
+    timeoutMs?: number;
+}
+export type HandoffVisualCritiqueProvider = (input: HandoffVisualCritiqueInput) => Promise<HandoffVisualCritiqueResult>;
+export declare function createHandoffVisualCritiqueProvider(options: CreateHandoffVisualCritiqueProviderOptions): HandoffVisualCritiqueProvider;
+/**
+ * Route a handoff to the screenshot provider when its source already carries
+ * HTML (html / deck), and to the text provider for prototypes, which have no
+ * HTML to render. Keeps prototype handoffs scored when screenshots are on.
+ */
+export declare function createRoutedHandoffCritiqueProvider(providers: {
+    text: HandoffVisualCritiqueProvider;
+    screenshot: HandoffVisualCritiqueProvider;
+}): HandoffVisualCritiqueProvider;

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { createVlmVisualScoreProvider, type CreateVlmVisualScoreProviderOptions, type ModelGenerateFn, type GenerateHtmlFn, } from "./vlm-score-provider.js";
+export { createModelHtmlGenerator } from "./model-html-generator.js";
+export { renderSpecToPng, type RenderScreenshotOptions } from "./render-screenshot.js";
+export { createHandoffVisualCritiqueProvider, createRoutedHandoffCritiqueProvider, type CreateHandoffVisualCritiqueProviderOptions, type HandoffVisualCritiqueInput, type HandoffVisualCritiqueProvider, type HandoffVisualCritiqueResult, type HandoffVisualCritiqueSource, type HandoffRenderHtmlFn, } from "./handoff-visual-critique.js";

package/dist/index.js

@@ -0,0 +1,20 @@
+// @bun
+import{createHash as C}from"crypto";function m(e,t){return C("sha256").update(JSON.stringify([e,t])).digest("hex")}function y(e){return typeof e==="number"&&Number.isFinite(e)&&e>=0&&e<=1}var b="title|text|label|content|description|heading|subtitle|placeholder|body|caption|name",ge=new RegExp(`["']?\\b(?:${b})["']?\\s*:\\s*["']([^"'\\n]*)["']`,"gi"),fe=new RegExp(`^\\s*-?\\s*(?:${b})\\s*:\\s*([^\\s"'{\\[][^\\n]*)$`,"gim");function H(e){let t=e.trim();if(t.startsWith("{")&&t.endsWith("}"))return t;let r=0,a=-1,n=!1,s=!1;for(let o=0;o<e.length;o+=1){let i=e[o];if(n){if(s)s=!1;else if(i==="\\")s=!0;else if(i==='"')n=!1;continue}if(i==='"')n=!0;else if(i==="{"){if(r===0)a=o;r+=1}else if(i==="}"){if(r-=1,r===0&&a!==-1)return e.slice(a,o+1)}}return null}function f(e){let t=H(e);if(t===null)return null;let r;try{r=JSON.parse(t)}catch{return null}if(typeof r.score!=="number"||!Number.isFinite(r.score))return null;if(r.score<0||r.score>1)return null;return typeof r.reasoning==="string"?{score:r.score,reasoning:r.reasoning}:{score:r.score}}function O(e,t){return`<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <script src="${t}"></script>
+  <style>body { margin: 0; background: white; }</style>
+</head>
+<body>${e}</body>
+</html>`}var M="https://cdn.tailwindcss.com";async function g(e,t={}){let r;try{r=await import("puppeteer-core")}catch{return console.warn("[vlm-scorer] puppeteer-core not installed; screenshots unavailable"),null}let a=t.width??1200,n=t.height??800,s=t.deviceScaleFactor??1,o=t.settleMs??500,i=/<html[\s>]/i.test(e)?e:O(e,t.tailwindPlayUrl??M),u=null;try{u=await r.launch({headless:!0,...t.chromePath?{executablePath:t.chromePath}:{},args:["--no-sandbox","--disable-setuid-sandbox"]});let c=await u.newPage();if(await c.setViewport({width:a,height:n,deviceScaleFactor:s}),await c.setContent(i,{waitUntil:"networkidle0",timeout:15000}),o>0)await new Promise((l)=>setTimeout(l,o));let d=await c.screenshot({type:"png",fullPage:!1});return Buffer.isBuffer(d)?d:Buffer.from(d)}catch(c){return console.warn(`[vlm-scorer] screenshot failed: ${c instanceof Error?c.message:String(c)}`),null}finally{if(u)await u.close().catch(()=>{return})}}var T=30000,F=["You are a UI quality evaluator.","Score how well the provided UI satisfies the given intent on a 0\u20131 scale.","Consider: layout clarity, visual hierarchy, contrast, component choice, content coherence.",'Reply with a single JSON object: {"score": 0.xx, "reasoning": "\u2026"}.',"Score must be a number in [0, 1]. Do not output anything outside the JSON."].join(" ");function A(e){let t=e.timeoutMs??T,r=async(n,s)=>{let o=m(n,s),i=e.generateHtml?"unassessed":"model-text",u=(p)=>({status:"unavailable",method:i,subjectHash:o,reason:p});if(!n.trim())return u("empty-input");let c=new AbortController,d,l=new Promise((p)=>{d=setTimeout(()=>{c.abort(),p(u("timeout"))},t)}),P=async()=>{try{let p=e.generateHtml?await _(e,n,s,c.signal):L(n,s);if(c.signal.aborted)return u("timeout");if(p===null)return u("screenshot-unavailable");if(e.generateHtml)i="model-screenshot";let v=await e.generate({model:e.model,system:F,messages:[{role:"user",content:p}],responseFormat:{type:"json_object"}},{signal:c.signal}),w=typeof v.message.content==="string"?v.message.content:v.message.content.filter((h)=>h.type==="text").map((h)=>h.text).join(""),x=f(w);return x?{status:"scored",method:i,subjectHash:o,score:x.score}:u("invalid-score")}catch{return u(c.signal.aborted?"timeout":"provider-failed")}};try{return await Promise.race([P(),l])}finally{clearTimeout(d)}},a=async(n,s)=>{let o=await r(n,s);if(o.status==="scored")return o.score;if(e.fallback){let i=await e.fallback(n,s);if(y(i))return i}throw Error(`visual-evaluation-unavailable: ${o.reason}`)};return a.evaluate=r,a}async function _(e,t,r,a){if(!e.generateHtml)throw Error("generateHtml required for image content path");let n=await e.generateHtml(t,r);if(a.aborted)return null;let s=await g(n,e.screenshot);if(s===null)return null;let o=s.toString("base64");return[{type:"text",text:`Intent: ${r}
+
+Evaluate the screenshot.`},{type:"image_base64",mediaType:"image/png",data:o}]}function L(e,t){return[`Intent: ${t}`,"","Skill definition (SKILL.md body) that produces the UI:","---",e,"---","","Rate this skill definition's UI quality 0\u20131 for the given intent."].join(`
+`)}var q=["You turn a UI skill description and a user intent into one self-contained HTML fragment.","Use Tailwind utility classes only, no external assets, no scripts.","Output only the HTML, with no explanation and no code fences."].join(" "),j=/^```(?:html)?\s*\n?([\s\S]*?)\n?```\s*$/i;function N(e){return async(t,r)=>{let n=(await e.generate({model:e.model,system:q,messages:[{role:"user",content:`Intent:
+${r}
+
+Skill:
+${t}`}]})).message.content,o=(typeof n==="string"?n:n.filter((u)=>u.type==="text").map((u)=>u.text).join("")).trim(),i=(j.exec(o)?.[1]??o).trim();if(!i)throw Error("model-html-generator: empty reply");return i}}var B=30000,G=["You are a UI visual critique evaluator for UIBot handoff drafts.","Score whether the generated UIBotSpec draft preserves the prototype/demo intent.","Consider visual hierarchy, density, contrast, layout clarity, component choice, and interaction affordance.",'Reply with one JSON object: {"score": 0.xx, "reasoning": "short reason"}.',"Score must be a number in [0, 1]. Do not output anything outside JSON."].join(" ");function U(e){return typeof e.message.content==="string"?e.message.content:e.message.content.filter((t)=>t.type==="text").map((t)=>t.text).join("")}function Q(e,t,r,a){return new Promise((n,s)=>{let o=setTimeout(()=>{a.abort(),s(Error(`${r} timeout after ${t}ms`))},t);e.then((i)=>{clearTimeout(o),n(i)},(i)=>{clearTimeout(o),s(i)})})}function R(e){if(e.kind==="prototype")return`prototype:${e.prototypeId} title:${e.prototype.title}`;return`${e.kind}:${e.title??"(untitled)"}${e.designMdId?` designMd:${e.designMdId}`:""}`}function W(e){return[`Source: ${R(e.source)}`,"","Prototype/demo intent:",e.intent,"","Generated UIBotSpec YAML draft:","```yaml",e.draftYaml,"```","","Return a JSON score for how well the draft preserves the intended visual result."].join(`
+`)}async function Y(e,t,r,a){let n=await t(e);r.throwIfAborted();let s=await g(n,a);if(r.throwIfAborted(),s===null)return null;return[{type:"text",text:[`Source: ${R(e.source)}`,"","Prototype/demo intent:",e.intent,"","Generated UIBotSpec YAML draft:",e.draftYaml,"","Evaluate the screenshot and YAML together."].join(`
+`)},{type:"image_base64",mediaType:"image/png",data:s.toString("base64")}]}function J(e){let t=e.timeoutMs??B;return async function(a){let n=new AbortController,s=m(a.draftYaml,a.intent),o=e.renderHtml?"unassessed":"model-text",i=(c)=>({status:"unavailable",reason:c,evidence:{status:"unavailable",method:o,subjectHash:s,reason:c}});if(!a.draftYaml.trim())return i("empty-input");let u=async()=>{let c=e.renderHtml?await Y(a,e.renderHtml,n.signal,e.screenshot):W(a);if(c===null)return i("screenshot-unavailable");if(e.renderHtml)o="model-screenshot";let d=await e.generate({model:e.model,system:G,messages:[{role:"user",content:c}],responseFormat:{type:"json_object"}},{signal:n.signal}),l=f(U(d));if(!l)return i("invalid-score");return{status:"scored",score:l.score,evidence:{status:"scored",method:o,subjectHash:s,score:l.score},...l.reasoning?{reasoning:l.reasoning}:{}}};try{return await Q(u(),t,"handoff visual critique",n)}catch{return i(n.signal.aborted?"timeout":"provider-failed")}}}function z(e){return(t)=>t.source.kind==="prototype"?e.text(t):e.screenshot(t)}export{J as createHandoffVisualCritiqueProvider,N as createModelHtmlGenerator,z as createRoutedHandoffCritiqueProvider,A as createVlmVisualScoreProvider,g as renderSpecToPng};

package/dist/model-html-generator.d.ts

@@ -0,0 +1,11 @@
+import type { GenerateHtmlFn, ModelGenerateFn } from "./vlm-score-provider.js";
+/**
+ * Build the `generateHtml` hook for screenshot scoring from a model call: the
+ * skill body and intent go in, an HTML fragment comes out for rendering.
+ * Throws on an empty reply so the scorer reports `provider-failed` rather than
+ * screenshotting a blank page.
+ */
+export declare function createModelHtmlGenerator(options: {
+    generate: ModelGenerateFn;
+    model: string;
+}): GenerateHtmlFn;

package/dist/parse-score.d.ts

@@ -0,0 +1,6 @@
+interface ParsedCritique {
+    score: number;
+    reasoning?: string;
+}
+export declare function parseScore(text: string): ParsedCritique | null;
+export {};

package/dist/render-screenshot.d.ts

@@ -0,0 +1,16 @@
+export interface RenderScreenshotOptions {
+    width?: number;
+    height?: number;
+    deviceScaleFactor?: number;
+    /** Optional Chrome executable path (e.g. "/usr/bin/google-chrome-stable"). */
+    chromePath?: string;
+    /** ms to wait after setContent before screenshotting. Default 500. */
+    settleMs?: number;
+    /**
+     * Tailwind Play script URL. Defaults to the public CDN. Override with a
+     * self-hosted copy (e.g. `http://127.0.0.1:3200/tailwind-play.js`) to avoid
+     * CDN dependency in production or offline environments.
+     */
+    tailwindPlayUrl?: string;
+}
+export declare function renderSpecToPng(input: string, options?: RenderScreenshotOptions): Promise<Buffer | null>;

package/dist/vlm-score-provider.d.ts

@@ -0,0 +1,28 @@
+import type { ModelGenerateRequest, ModelGenerateResult } from "@uibot/contracts";
+import { type VisualScoreProvider } from "@uibot/evaluation";
+import { type RenderScreenshotOptions } from "./render-screenshot.js";
+export type ModelGenerateFn = (req: ModelGenerateRequest, context?: {
+    signal?: AbortSignal;
+}) => Promise<ModelGenerateResult>;
+/** Produce HTML from a skill body + intent. Consumer-supplied because this
+ *  typically calls an LLM; the VLM scorer does not assume a generation flow. */
+export type GenerateHtmlFn = (skillBody: string, intent: string) => Promise<string>;
+export interface CreateVlmVisualScoreProviderOptions {
+    /** Runs a `ModelGenerateRequest` against a vision-capable model. */
+    generate: ModelGenerateFn;
+    /** Vision-capable model id, e.g. "qwen-vl-max-latest" or "claude-sonnet-4-6". */
+    model: string;
+    /**
+     * Optional: generate HTML from skill body + intent. When provided, the scorer
+     * renders the HTML to a PNG and sends an image block to the VLM. When omitted,
+     * the scorer sends the raw skill body text to the VLM for structural scoring.
+     */
+    generateHtml?: GenerateHtmlFn;
+    /** Screenshot options passed to `renderHtmlToPng`. */
+    screenshot?: RenderScreenshotOptions;
+    /** Explicit compatibility fallback for direct numeric calls only. Evidence remains unavailable. */
+    fallback?: VisualScoreProvider;
+    /** Deadline for rendering and model assessment (ms). Default 30_000. */
+    timeoutMs?: number;
+}
+export declare function createVlmVisualScoreProvider(options: CreateVlmVisualScoreProviderOptions): VisualScoreProvider;

package/package.json

@@ -1,6 +1,49 @@
 {
   "name": "@uibot/vlm-scorer",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.2.1",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Foundj/uibot.git",
+    "directory": "packages/vlm-scorer"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "!dist/**/*.map",
+    "!dist/**/*.test.d.ts",
+    "!dist/**/*.spec.d.ts"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@uibot/contracts": "^0.2.1",
+    "@uibot/evaluation": "^0.2.1",
+    "@uibot/compiler": "^0.2.1",
+    "@uibot/renderer-react": "^0.2.1",
+    "react-dom": "19.2.1"
+  },
+  "peerDependencies": {
+    "puppeteer-core": "^23.0.0"
+  },
+  "peerDependenciesMeta": {
+    "puppeteer-core": {
+      "optional": true
+    }
+  },
+  "scripts": {
+    "build": "rm -rf dist && bun build --minify ./src/index.ts --outdir ./dist --format esm --target bun && bun x tsc -p tsconfig.json",
+    "typecheck": "bun x tsc -p tsconfig.json --noEmit",
+    "test": "bun test"
+  }
+}

package/README.md

@@ -1,3 +0,0 @@
-# Temporary Holding Version
-
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.