@hmharness/domain-harmony 0.6.5 → 0.6.7

package/dist/uiregress.d.ts

@@ -13,15 +13,40 @@ export interface UiRegressionResult {
     saw: string | null;
     description: string;
     screenshot: string | null;
+    /** true when NO provider actually looked at the image (all blind/failed).
+     *  The verdict is then about the vision chain, NOT about the app: never
+     *  render it as a product FAIL - a tool that reports FAIL when its own
+     *  eyes are shut is worse than no tool. */
+    visionUnavailable?: boolean;
+    /** per-provider errors collected while walking the vision chain */
+    visionErrors?: string[];
 }
+/** Injectable vision call (tests substitute it; production uses chatVision). */
+export type VisionCall = (provider: ProviderConfig, prompt: string, imageDataUrl: string) => Promise<string>;
+/**
+ * Walk the vision chain until a provider truly LOOKS at the image. A reply
+ * that is a refusal ("I can't view the image" - what a text-only provider
+ * returns with HTTP 200) is treated as a provider failure and the next
+ * candidate is tried. Returns the first real description, or the collected
+ * errors when every provider was blind/dead.
+ */
+export declare function describeWithChain(chain: ProviderConfig[], prompt: string, imageDataUrl: string, call?: VisionCall): Promise<{
+    described: string | null;
+    errors: string[];
+}>;
+/** Keyword assertion on a genuine description (any hit wins). */
+export declare function assertKeywords(described: string, expect: string[]): string | null;
 /** Capture the device screen via hdc; returns the local file path. */
 export declare function captureDeviceScreen(hdc: string, target: string | undefined, outDir: string): Promise<string>;
 /** One regression case through the real device + vision chain. */
 export declare function runUiRegression(opts: {
     hdc: string;
     target?: string;
-    vision: ProviderConfig;
+    /** one provider, or the ordered chain (vision + visionFallbacks) */
+    vision: ProviderConfig | ProviderConfig[];
     cases: UiRegressionCase[];
     outDir: string;
+    /** test seam: substitute the multimodal call */
+    visionCall?: VisionCall;
 }): Promise<UiRegressionResult[]>;
 export declare const harmonyUiRegression: Tool;

package/dist/uiregress.js

@@ -19,8 +19,36 @@ import { execFile } from 'node:child_process';
 import { mkdir, readFile, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { promisify } from 'node:util';
-import { chatVision } from '@hmharness/kernel';
+import { chatVision, foundNothing, isVisionRefusal } from '@hmharness/kernel';
 const execCb = promisify(execFile);
+/**
+ * Walk the vision chain until a provider truly LOOKS at the image. A reply
+ * that is a refusal ("I can't view the image" - what a text-only provider
+ * returns with HTTP 200) is treated as a provider failure and the next
+ * candidate is tried. Returns the first real description, or the collected
+ * errors when every provider was blind/dead.
+ */
+export async function describeWithChain(chain, prompt, imageDataUrl, call = chatVision) {
+    const errors = [];
+    for (const provider of chain) {
+        try {
+            const text = await call(provider, prompt, imageDataUrl);
+            if (isVisionRefusal(text)) {
+                errors.push(`${provider.model}: answered without seeing the image ("${text.trim().slice(0, 80)}")`);
+                continue;
+            }
+            return { described: text, errors };
+        }
+        catch (err) {
+            errors.push(`${provider.model}: ${String(err).slice(0, 120)}`);
+        }
+    }
+    return { described: null, errors };
+}
+/** Keyword assertion on a genuine description (any hit wins). */
+export function assertKeywords(described, expect) {
+    return expect.find((k) => described.toLowerCase().includes(k.toLowerCase())) ?? null;
+}
 /** Capture the device screen via hdc; returns the local file path. */
 export async function captureDeviceScreen(hdc, target, outDir) {
     await mkdir(outDir, { recursive: true });
@@ -45,6 +73,7 @@ export async function captureDeviceScreen(hdc, target, outDir) {
 }
 /** One regression case through the real device + vision chain. */
 export async function runUiRegression(opts) {
+    const chain = Array.isArray(opts.vision) ? opts.vision : [opts.vision];
     const results = [];
     for (const c of opts.cases) {
         const pre = opts.target ? ['-t', opts.target] : [];
@@ -66,17 +95,38 @@ export async function runUiRegression(opts) {
             results.push({ case: c.name, pass: false, saw: null, description: 'screenshot failed: ' + String(err).slice(0, 120), screenshot: null });
             continue;
         }
-        // vision describe
-        try {
-            const b64 = (await readFile(shot)).toString('base64');
-            const text = await chatVision(opts.vision, 'Describe this device screen briefly. Then on the last line output exactly: FOUND: <the most prominent UI text you can read>.', `data:image/jpeg;base64,${b64}`);
-            const described = text.trim();
-            const saw = c.expect.find((k) => described.toLowerCase().includes(k.toLowerCase())) ?? null;
-            results.push({ case: c.name, pass: Boolean(saw), saw, description: described.slice(0, 400), screenshot: shot });
+        // vision describe (through the chain; refusals do not count as seeing)
+        const b64 = (await readFile(shot)).toString('base64');
+        const { described, errors } = await describeWithChain(chain, 'Describe this device screen briefly. Then on the last line output exactly: FOUND: <the most prominent UI text you can read>.', `data:image/jpeg;base64,${b64}`, opts.visionCall);
+        if (described === null) {
+            // EVERY provider failed to look: this says nothing about the app.
+            results.push({
+                case: c.name,
+                pass: false,
+                saw: null,
+                visionUnavailable: true,
+                visionErrors: errors,
+                description: `VISION PROVIDER FAILURE (no UI verdict): none of the ${chain.length} configured vision provider(s) could see the image - ${errors.join('; ')}`,
+                screenshot: shot,
+            });
+            continue;
         }
-        catch (err) {
-            results.push({ case: c.name, pass: false, saw: null, description: 'vision failed: ' + String(err).slice(0, 150), screenshot: shot });
+        const text = described.trim();
+        const saw = assertKeywords(text, c.expect);
+        if (!saw && foundNothing(text)) {
+            // the model itself says it read no text: no evidence either way
+            results.push({
+                case: c.name,
+                pass: false,
+                saw: null,
+                visionUnavailable: true,
+                visionErrors: [...errors, `${chain[0].model}: reported no readable text on screen ("${text.slice(0, 120)}")`],
+                description: `NO VERDICT - the vision model reported no readable text on screen (blank screen, or a provider that cannot see): ${text.slice(0, 240)}`,
+                screenshot: shot,
+            });
+            continue;
         }
+        results.push({ case: c.name, pass: Boolean(saw), saw, description: text.slice(0, 400), screenshot: shot, visionErrors: errors });
     }
     return results;
 }
@@ -99,18 +149,20 @@ export const harmonyUiRegression = {
         const expect = Array.isArray(args.expect) ? args.expect.map(String).filter(Boolean) : [];
         if (!bundle || expect.length === 0)
             return { output: 'bundle and non-empty expect[] required', isError: true };
-        // vision provider from config (kernel routing)
-        const { loadConfig, resolveProvider } = await import('@hmharness/kernel');
+        // vision provider from config (kernel routing): the full chain, so a
+        // blind or dead provider is retried rather than graded as a UI verdict
+        const { loadConfig, visionProviderChain } = await import('@hmharness/kernel');
         const cfg = await loadConfig();
         let vision;
         try {
-            vision = resolveProvider(cfg, 'vision');
-            if (!vision.apiKey)
+            vision = visionProviderChain(cfg);
+            if (vision.length === 0 || !vision[0].apiKey)
                 throw new Error('no key');
         }
         catch {
             return { output: 'No vision provider configured (vision block or providers+routing.vision) - harmony_ui_regression needs one.', isError: true };
         }
+        const chainNames = vision.map((p) => `${p.model}${p.supportsVision === false ? ' (marked text-only!)' : ''}`);
         // hdc
         const deveco = process.env.HM_DEVECO_HOME ?? 'C:\\DevEco-Studio';
         let hdc = 'hdc';
@@ -136,12 +188,31 @@ export const harmonyUiRegression = {
             outDir,
         });
         const r = results[0];
+        if (r.visionUnavailable) {
+            // The machine could not look at the screen. Report that loudly and do
+            // NOT dress it up as a product verdict.
+            return {
+                output: [
+                    `UI regression: ${r.case}`,
+                    `NO VERDICT - vision provider failure, not a UI result`,
+                    `vision chain tried (${chainNames.join(' -> ')}):`,
+                    ...(r.visionErrors ?? []).map((e) => `  - ${e}`),
+                    ...(r.screenshot ? [`screenshot: ${r.screenshot}`] : []),
+                    'Fix: point routing.vision at a model that accepts images, or mark the text-only one with "supportsVision": false so it is skipped.',
+                    'Ground truth without vision: hdc shell uitest dumpLayout -p /data/local/tmp/layout.json (the view tree carries every node text).',
+                ].join('\n'),
+                isError: true,
+            };
+        }
         const lines = [
             `UI regression: ${r.case}`,
             r.saw ? `PASS - saw "${r.saw}" on screen` : `FAIL - none of [${expect.join(', ')}] visible`,
             ...(r.screenshot ? [`screenshot: ${r.screenshot}`] : []),
             `vision said: ${r.description}`,
         ];
+        if (r.visionErrors?.length) {
+            lines.push(`degraded chain (earlier provider(s) did not see the image): ${r.visionErrors.join('; ')}`);
+        }
         return { output: lines.join('\n'), isError: !r.pass };
     },
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hmharness/domain-harmony",
-  "version": "0.6.5",
+  "version": "0.6.7",
   "description": "hmharness HarmonyOS domain core: device, toolchain, build, and project lifecycle capabilities. HarmonyOS is the framework's native domain, not an add-on.",
   "type": "module",
   "main": "dist/index.js",
@@ -15,7 +15,7 @@
     "build": "tsc -p tsconfig.build.json"
   },
   "dependencies": {
-    "@hmharness/kernel": "0.6.5"
+    "@hmharness/kernel": "0.6.7"
   },
   "files": [
     "dist"