onbuzz 4.7.2 → 4.8.1

package/src/services/grounding/__tests__/groundingMetrics.test.js

@@ -0,0 +1,156 @@
+/**
+ * Tests for groundingMetrics — pure functions, no I/O, fast.
+ *
+ * Why these matter: every accuracy claim in the bench (and in
+ * production audit) depends on parseCoords / pointInRect / summarise
+ * producing exactly the values everyone expects.
+ */
+
+import { describe, test, expect } from '@jest/globals';
+import {
+  parseCoords,
+  distance,
+  rectCenter,
+  pointInRect,
+  summarise,
+} from '../groundingMetrics.js';
+
+describe('parseCoords', () => {
+  test('canonical form "x=N, y=N"', () => {
+    expect(parseCoords('x=100, y=200')).toEqual({ x: 100, y: 200 });
+  });
+
+  test('forgiving of whitespace + punctuation', () => {
+    expect(parseCoords('x  =   100 ;  y =  200'))   .toEqual({ x: 100, y: 200 });
+    expect(parseCoords('   x=1, y=2.\n'))            .toEqual({ x: 1, y: 2 });
+    expect(parseCoords('the answer is: x=42, y=99')).toEqual({ x: 42, y: 99 });
+  });
+
+  test('case insensitive', () => {
+    expect(parseCoords('X=10, Y=20')).toEqual({ x: 10, y: 20 });
+  });
+
+  test('returns null on no match', () => {
+    expect(parseCoords('')).toBeNull();
+    expect(parseCoords('no coords here')).toBeNull();
+    expect(parseCoords(undefined)).toBeNull();
+    expect(parseCoords(null)).toBeNull();
+    expect(parseCoords(42)).toBeNull();
+  });
+
+  test('rejects malformed numbers', () => {
+    expect(parseCoords('x=abc, y=def')).toBeNull();
+  });
+
+  test('first match wins (model may emit examples)', () => {
+    expect(parseCoords('the format is x=N, y=N. Answer: x=300, y=400'))
+      .toEqual({ x: 300, y: 400 });
+  });
+
+  test('handles negative coords (edge case for cropped views)', () => {
+    expect(parseCoords('x=-5, y=-10')).toEqual({ x: -5, y: -10 });
+  });
+});
+
+describe('distance', () => {
+  test('Euclidean for typical points', () => {
+    expect(distance({ x: 0, y: 0 }, { x: 3, y: 4 })).toBe(5);
+  });
+
+  test('zero when points coincide', () => {
+    expect(distance({ x: 7, y: 7 }, { x: 7, y: 7 })).toBe(0);
+  });
+
+  test('NaN on invalid inputs (does not throw)', () => {
+    expect(Number.isNaN(distance(null, { x: 1, y: 1 }))).toBe(true);
+    expect(Number.isNaN(distance({ x: 1, y: 1 }, { x: 'a', y: 1 }))).toBe(true);
+  });
+});
+
+describe('rectCenter', () => {
+  test('returns geometric center', () => {
+    expect(rectCenter({ x: 100, y: 100, width: 200, height: 50 }))
+      .toEqual({ x: 200, y: 125 });
+  });
+
+  test('handles odd dimensions (returns fractional)', () => {
+    expect(rectCenter({ x: 0, y: 0, width: 1, height: 1 }))
+      .toEqual({ x: 0.5, y: 0.5 });
+  });
+});
+
+describe('pointInRect', () => {
+  const RECT = { x: 100, y: 100, width: 50, height: 30 };
+
+  test('inside', () => {
+    expect(pointInRect({ x: 120, y: 110 }, RECT)).toBe(true);
+  });
+
+  test('on boundary counts as inside', () => {
+    expect(pointInRect({ x: 100, y: 100 }, RECT)).toBe(true);   // top-left
+    expect(pointInRect({ x: 150, y: 130 }, RECT)).toBe(true);   // bot-right
+  });
+
+  test('outside', () => {
+    expect(pointInRect({ x: 99,  y: 110 }, RECT)).toBe(false);
+    expect(pointInRect({ x: 151, y: 110 }, RECT)).toBe(false);
+    expect(pointInRect({ x: 120, y: 99  }, RECT)).toBe(false);
+    expect(pointInRect({ x: 120, y: 131 }, RECT)).toBe(false);
+  });
+
+  test('false on invalid inputs', () => {
+    expect(pointInRect(null, RECT)).toBe(false);
+    expect(pointInRect({ x: 1, y: 1 }, null)).toBe(false);
+    expect(pointInRect({ x: 1, y: 1 }, { x: 0, y: 0 })).toBe(false); // missing w/h
+  });
+});
+
+describe('summarise', () => {
+  test('aggregates a typical trial set', () => {
+    const trials = [
+      { err: 10, hit: true,  elapsedMs:  100, tokens: 500 },
+      { err: 20, hit: true,  elapsedMs:  200, tokens: 800 },
+      { err: 50, hit: false, elapsedMs:  300, tokens: 600 },
+    ];
+    const s = summarise(trials);
+    expect(s.n).toBe(3);
+    expect(s.hitRate).toBeCloseTo(2 / 3, 3);
+    expect(s.meanErrPx).toBeCloseTo((10 + 20 + 50) / 3, 3);
+    expect(s.meanLatencyMs).toBe(200);
+    expect(s.meanTokens).toBeCloseTo((500 + 800 + 600) / 3, 3);
+    expect(s.stdevErrPx).toBeGreaterThan(0);
+  });
+
+  test('returns zero summary on empty / invalid input', () => {
+    const z = summarise([]);
+    expect(z.n).toBe(0);
+    expect(z.hitRate).toBe(0);
+    expect(summarise(null).n).toBe(0);
+  });
+
+  test('handles trials with no err field (skip-on-no-coord scenario)', () => {
+    const trials = [
+      { hit: false, elapsedMs: 50, tokens: 100 },  // no err: model didn't produce coords
+      { err: 5, hit: true, elapsedMs: 60, tokens: 200 },
+    ];
+    const s = summarise(trials);
+    expect(s.meanErrPx).toBe(5);    // only the valid trial counts
+    expect(s.hitRate).toBe(0.5);    // both trials counted for hit rate
+  });
+
+  test('stdev=0 with single trial (cannot compute without 2 samples)', () => {
+    const s = summarise([{ err: 10, hit: true, elapsedMs: 100, tokens: 500 }]);
+    expect(s.stdevErrPx).toBe(0);
+  });
+
+  test('p50 and p95 are sorted-percentile lookups', () => {
+    const trials = Array.from({ length: 20 }, (_, i) => ({
+      err: i, hit: true, elapsedMs: (i + 1) * 100, tokens: 100,
+    }));
+    const s = summarise(trials);
+    // sorted ascending: 100..2000; p50 → ~1000ms, p95 → ~1900ms
+    expect(s.p50LatencyMs).toBeGreaterThanOrEqual(900);
+    expect(s.p50LatencyMs).toBeLessThanOrEqual(1100);
+    expect(s.p95LatencyMs).toBeGreaterThanOrEqual(1800);
+  });
+});

package/src/services/grounding/__tests__/registry.test.js

@@ -0,0 +1,122 @@
+/**
+ * Registry tests — lock the contract every UI / config / agent path
+ * leans on:
+ *   1. listGroundingModels() returns objects with the shape the form
+ *      generator expects (id, displayName, knobs[], capabilities)
+ *   2. createGroundingModel() resolves the registered id and rejects
+ *      unknown ids with a typed error
+ *   3. The MOCK adapter is hidden by default (so it never accidentally
+ *      surfaces in the production UI dropdown)
+ */
+
+import { describe, test, expect } from '@jest/globals';
+import {
+  createGroundingModel,
+  listGroundingModels,
+  isRegistered,
+  listModelIds,
+} from '../registry.js';
+import { ModelId, ErrorCode } from '../constants.js';
+import KimiAdapter from '../KimiAdapter.js';
+import MockAdapter from '../MockAdapter.js';
+
+describe('listGroundingModels', () => {
+  test('returns Kimi by default; hides MOCK', () => {
+    const out = listGroundingModels();
+    const ids = out.map(m => m.id);
+    expect(ids).toContain(ModelId.KIMI_K2_6);
+    expect(ids).not.toContain(ModelId.MOCK);
+  });
+
+  test('includeInternal=true reveals MOCK (for tests / dev tools)', () => {
+    const out = listGroundingModels({ includeInternal: true });
+    expect(out.map(m => m.id)).toContain(ModelId.MOCK);
+  });
+
+  test('each entry has the shape the admin form-generator needs', () => {
+    for (const m of listGroundingModels({ includeInternal: true })) {
+      expect(typeof m.id).toBe('string');
+      expect(typeof m.displayName).toBe('string');
+      expect(m.displayName.length).toBeGreaterThan(0);
+      expect(Array.isArray(m.knobs)).toBe(true);
+      expect(typeof m.capabilities).toBe('object');
+    }
+  });
+
+  test('Kimi entry exposes the expected knob set (locks the UI contract)', () => {
+    const kimi = listGroundingModels().find(m => m.id === ModelId.KIMI_K2_6);
+    expect(kimi).toBeTruthy();
+    const knobIds = kimi.knobs.map(k => k.id).sort();
+    expect(knobIds).toContain('reasoning_effort');
+    expect(knobIds).toContain('temperature');
+    expect(knobIds).toContain('top_p');
+    expect(knobIds).toContain('image_detail');
+  });
+});
+
+describe('createGroundingModel', () => {
+  test('creates a KimiAdapter instance for the Kimi id', () => {
+    const m = createGroundingModel(ModelId.KIMI_K2_6, {
+      endpoint: 'https://e/', apiKey: 'k',
+    });
+    expect(m).toBeInstanceOf(KimiAdapter);
+  });
+
+  test('creates a MockAdapter for the mock id', () => {
+    const m = createGroundingModel(ModelId.MOCK, {});
+    expect(m).toBeInstanceOf(MockAdapter);
+  });
+
+  test('throws typed error for unknown id', () => {
+    expect.assertions(2);
+    try {
+      createGroundingModel('imaginary-model', {});
+    } catch (err) {
+      expect(err.code).toBe(ErrorCode.UNKNOWN_MODEL);
+      expect(err.message).toMatch(/imaginary-model/);
+    }
+  });
+});
+
+describe('isRegistered + listModelIds', () => {
+  test('isRegistered reflects the actual map', () => {
+    expect(isRegistered(ModelId.KIMI_K2_6)).toBe(true);
+    expect(isRegistered(ModelId.MOCK)).toBe(true);
+    expect(isRegistered('not-here')).toBe(false);
+  });
+
+  test('listModelIds includes internal entries (unfiltered)', () => {
+    const ids = listModelIds();
+    expect(ids).toContain(ModelId.KIMI_K2_6);
+    expect(ids).toContain(ModelId.MOCK);
+  });
+});
+
+// ─── REGRESSION: catalog-name contract ─────────────────────────────────
+// The backend's POST /llm/grounding does a CASE-SENSITIVE lookup of the
+// `model` field against the Loxia model catalog (autopilot-model-
+// catalog). If the CLI sends a name that doesn't match the catalog
+// exactly, the catalog lookup falls through to "no deployment found",
+// Foundry gets a bogus deployment name, and the response is:
+//   "Backend 404: API deployment for this resource does not exist"
+// (observed in the OS Agent log on 2026-05-13 — every intent-click
+// failed because the catalog has "Kimi-K2.6" while the CLI was sending
+// "kimi-k2.6").
+//
+// This test pins the constant to the catalog's canonical spelling. If
+// someone tries to "normalise" it to lowercase or rename the model
+// without a coordinated catalog change, this test fails BEFORE shipping
+// instead of in production.
+describe('REGRESSION: catalog-name contract for KIMI_K2_6', () => {
+  test('ModelId.KIMI_K2_6 matches the catalog spelling "Kimi-K2.6" exactly', () => {
+    expect(ModelId.KIMI_K2_6).toBe('Kimi-K2.6');
+  });
+
+  test('the value is a non-empty PascalCase string with the K2.6 version', () => {
+    expect(typeof ModelId.KIMI_K2_6).toBe('string');
+    expect(ModelId.KIMI_K2_6.length).toBeGreaterThan(0);
+    // Catalog uses PascalCase; if this ever changes the proxy lookup
+    // will return 404. Lock the shape explicitly.
+    expect(ModelId.KIMI_K2_6).toMatch(/^Kimi-K2\.6$/);
+  });
+});

package/src/services/grounding/constants.js

@@ -0,0 +1,121 @@
+/**
+ * Named constants for the visual-grounding subsystem.
+ *
+ * Single source of truth — every number, enum, default and timeout
+ * the grounding layer cares about lives here. New magic numbers in
+ * adapter / bench / DesktopTool code should add a constant here
+ * instead of inlining the value.
+ *
+ * Grouped by concern so consumers can `import { ReasoningEffort }` etc.
+ */
+
+// ── Reasoning-effort levels supported by Kimi K2.6 on Azure AI Foundry ────────
+// Empirically (see Phase-0 bench): `low` is the accuracy sweet spot
+// (~3px error / ~80s) while `none` is the fastest (~2s) but noisy.
+// `medium` and `high` cost more without measurable accuracy gain.
+export const ReasoningEffort = Object.freeze({
+  NONE:   'none',
+  LOW:    'low',
+  MEDIUM: 'medium',
+  HIGH:   'high',
+});
+
+export const REASONING_EFFORT_VALUES = Object.freeze([
+  ReasoningEffort.NONE,
+  ReasoningEffort.LOW,
+  ReasoningEffort.MEDIUM,
+  ReasoningEffort.HIGH,
+]);
+
+/** Default for click-decision calls. See bench results: 3-7px mean error. */
+export const DEFAULT_REASONING_EFFORT = ReasoningEffort.LOW;
+
+// ── Sampling parameters (per Moonshot's published recommendations) ────────────
+// https://huggingface.co/moonshotai/Kimi-K2.6 — model card "Recommended Parameters"
+export const SamplingDefaults = Object.freeze({
+  THINKING_ON:  Object.freeze({ temperature: 1.0,  top_p: 1.0  }),
+  THINKING_OFF: Object.freeze({ temperature: 0.6,  top_p: 0.95 }),
+});
+
+// ── Token / latency budgets ───────────────────────────────────────────────────
+// Kimi reasoning traces routinely consume 5-15k tokens before emitting an
+// answer. Below 4k it frequently runs out mid-trace (see Phase-0 bench).
+export const TokenBudget = Object.freeze({
+  /** Min ceiling to avoid truncation when reasoning is on. */
+  THINKING_MIN_MAX_TOKENS: 8_000,
+  /** Safe default for grounding requests. */
+  THINKING_DEFAULT_MAX_TOKENS: 16_000,
+  /** Short responses fit comfortably below this when reasoning is off. */
+  INSTANT_DEFAULT_MAX_TOKENS: 500,
+});
+
+export const Timeouts = Object.freeze({
+  /** Full reasoning round-trip can take 80-120s; allow generous headroom. */
+  GROUNDING_REQUEST_MS: 180_000,
+  /** No-reasoning calls return fast; this is the impatient ceiling. */
+  GROUNDING_REQUEST_FAST_MS: 30_000,
+});
+
+// ── Image-input parameters ────────────────────────────────────────────────────
+export const ImageDetail = Object.freeze({
+  LOW:  'low',
+  HIGH: 'high',
+  AUTO: 'auto',
+});
+
+export const DEFAULT_IMAGE_DETAIL = ImageDetail.HIGH;
+
+// ── Azure AI Foundry endpoints ────────────────────────────────────────────────
+export const Foundry = Object.freeze({
+  /** API version that supports Kimi K2.6 chat + vision on Foundry. */
+  API_VERSION: '2024-05-01-preview',
+  /** Foundry's Azure AI inference endpoint path (NOT the OpenAI /chat path). */
+  CHAT_COMPLETIONS_PATH: 'models/chat/completions',
+});
+
+// ── Model identifiers (registry keys) ────────────────────────────────────────
+// CRITICAL: in proxy mode (the default in production), this string is
+// sent to the backend's POST /llm/grounding which does a CASE-SENSITIVE
+// lookup against the Loxia model catalog before forwarding to Foundry.
+// The catalog is the source of truth; a mismatch returns
+//   "Backend 404: API deployment for this resource does not exist"
+// with the model name echoed back. The catalog currently registers the
+// entry as "Kimi-K2.6" (Pascal case), so we match it verbatim.
+// (Direct-mode bench runs accept the lowercase name because Foundry
+//  itself has both casings as aliases — but we standardise on the
+//  catalog spelling because proxy mode is the production path.)
+export const ModelId = Object.freeze({
+  KIMI_K2_6:           'Kimi-K2.6',
+  CLAUDE_SONNET_4_6:   'claude-sonnet-4-6',
+  CLAUDE_OPUS_4_6:     'claude-opus-4-6',
+  OPENAI_COMPUTER_USE: 'computer-use-preview',
+  MOCK:                'mock',
+});
+
+export const DEFAULT_GROUNDING_MODEL_ID = ModelId.KIMI_K2_6;
+
+// ── Knob types for the admin-UI form generator ────────────────────────────────
+export const KnobType = Object.freeze({
+  ENUM:    'enum',
+  NUMBER:  'number',
+  BOOLEAN: 'boolean',
+  STRING:  'string',
+});
+
+// ── Error codes (stable, machine-readable, used by callers + UI) ─────────────
+export const ErrorCode = Object.freeze({
+  /** Grounding model rejected the request shape. */
+  INVALID_REQUEST:     'INVALID_REQUEST',
+  /** Provider auth / API key missing or rejected. */
+  AUTH_FAILED:         'AUTH_FAILED',
+  /** Provider 5xx or network error after retries. */
+  PROVIDER_ERROR:      'PROVIDER_ERROR',
+  /** Model did not return a parseable coord or tool_call. */
+  UNPARSEABLE_RESPONSE:'UNPARSEABLE_RESPONSE',
+  /** Caller asked for a knob this adapter doesn't support. */
+  UNSUPPORTED_KNOB:    'UNSUPPORTED_KNOB',
+  /** Caller asked for a model not in the registry. */
+  UNKNOWN_MODEL:       'UNKNOWN_MODEL',
+  /** Adapter timed out before the provider responded. */
+  TIMEOUT:             'TIMEOUT',
+});

package/src/services/grounding/groundingMetrics.js

@@ -0,0 +1,121 @@
+/**
+ * Pure-function utilities shared by:
+ *   - the Phase-0 benchmark harness (computes aggregate accuracy)
+ *   - DesktopTool runtime (decides whether a coord landed in-target)
+ *   - audit-log readers (computes hit-rate over time windows)
+ *
+ * No I/O, no provider calls, no state. Easy to unit-test.
+ *
+ * @typedef {import('./types.js').Point} Point
+ * @typedef {import('./types.js').Rect}  Rect
+ */
+
+/** Parse the canonical "x=<int>, y=<int>" answer shape. */
+const COORD_PATTERN = /x\s*=\s*(-?\d+)\D+y\s*=\s*(-?\d+)/i;
+
+/**
+ * Extract pixel coords from a model's free-text answer. Returns null
+ * if no coord pair found — caller decides whether to fall back to
+ * tool_calls or re-prompt.
+ *
+ * @param {string} answer
+ * @returns {Point | null}
+ */
+export function parseCoords(answer) {
+  if (typeof answer !== 'string' || !answer) return null;
+  const m = COORD_PATTERN.exec(answer);
+  if (!m) return null;
+  const x = Number(m[1]);
+  const y = Number(m[2]);
+  if (!Number.isFinite(x) || !Number.isFinite(y)) return null;
+  return { x, y };
+}
+
+/** Euclidean distance between two points. */
+export function distance(a, b) {
+  if (!isPoint(a) || !isPoint(b)) return NaN;
+  const dx = a.x - b.x;
+  const dy = a.y - b.y;
+  return Math.sqrt(dx * dx + dy * dy);
+}
+
+/** Center of a rect, as a Point. */
+export function rectCenter(r) {
+  return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
+}
+
+/**
+ * Does the point fall inside (or on the boundary of) the rect?
+ * Used by the bench to decide HIT vs MISS.
+ */
+export function pointInRect(p, r) {
+  if (!isPoint(p) || !isRect(r)) return false;
+  return p.x >= r.x && p.x <= r.x + r.width
+      && p.y >= r.y && p.y <= r.y + r.height;
+}
+
+/**
+ * Aggregate stats over an array of per-trial results. Used by the
+ * bench to summarise a sweep (e.g. "config=low, n=12 trials, hit-rate=92%").
+ *
+ * @param {{ err?: number, hit?: boolean, elapsedMs?: number, tokens?: number }[]} trials
+ * @returns {{
+ *   n: number,
+ *   hitRate: number,
+ *   meanErrPx: number,
+ *   stdevErrPx: number,
+ *   meanLatencyMs: number,
+ *   p50LatencyMs: number,
+ *   p95LatencyMs: number,
+ *   meanTokens: number,
+ * }}
+ */
+export function summarise(trials) {
+  if (!Array.isArray(trials) || trials.length === 0) {
+    return zeroSummary();
+  }
+  const errs    = trials.map(t => t.err).filter(Number.isFinite);
+  const lats    = trials.map(t => t.elapsedMs).filter(Number.isFinite);
+  const toks    = trials.map(t => t.tokens).filter(Number.isFinite);
+  const hits    = trials.filter(t => t.hit === true).length;
+  return {
+    n: trials.length,
+    hitRate:        trials.length ? hits / trials.length : 0,
+    meanErrPx:      mean(errs),
+    stdevErrPx:     stdev(errs),
+    meanLatencyMs:  mean(lats),
+    p50LatencyMs:   percentile(lats, 0.5),
+    p95LatencyMs:   percentile(lats, 0.95),
+    meanTokens:     mean(toks),
+  };
+}
+
+// ── helpers (deliberately not exported — implementation detail) ───────────────
+
+function isPoint(p) {
+  return p && typeof p === 'object' && Number.isFinite(p.x) && Number.isFinite(p.y);
+}
+function isRect(r) {
+  return r && typeof r === 'object'
+    && Number.isFinite(r.x) && Number.isFinite(r.y)
+    && Number.isFinite(r.width) && Number.isFinite(r.height);
+}
+function mean(xs) {
+  return xs.length ? xs.reduce((s, v) => s + v, 0) / xs.length : 0;
+}
+function stdev(xs) {
+  if (xs.length < 2) return 0;
+  const m = mean(xs);
+  const v = xs.reduce((s, x) => s + (x - m) ** 2, 0) / (xs.length - 1);
+  return Math.sqrt(v);
+}
+function percentile(xs, p) {
+  if (!xs.length) return 0;
+  const sorted = [...xs].sort((a, b) => a - b);
+  const idx = Math.min(sorted.length - 1, Math.max(0, Math.round(p * (sorted.length - 1))));
+  return sorted[idx];
+}
+function zeroSummary() {
+  return { n: 0, hitRate: 0, meanErrPx: 0, stdevErrPx: 0,
+    meanLatencyMs: 0, p50LatencyMs: 0, p95LatencyMs: 0, meanTokens: 0 };
+}

package/src/services/grounding/index.js

@@ -0,0 +1,53 @@
+/**
+ * Public API for the grounding subsystem.
+ *
+ * Single import surface for the rest of the codebase. Internal
+ * implementation files (adapters, registry, metrics) should NOT be
+ * imported directly from anywhere outside `src/services/grounding/*`.
+ *
+ * Usage (Phase-0 bench + future DesktopTool):
+ *
+ *     import {
+ *       createGroundingModel,
+ *       listGroundingModels,
+ *       ModelId,
+ *       ReasoningEffort,
+ *       parseCoords,
+ *       summarise,
+ *     } from '../services/grounding/index.js';
+ */
+
+export { GroundingModel, makeError } from './GroundingModel.js';
+
+export {
+  createGroundingModel,
+  listGroundingModels,
+  isRegistered,
+  listModelIds,
+} from './registry.js';
+
+export {
+  parseCoords,
+  distance,
+  rectCenter,
+  pointInRect,
+  summarise,
+} from './groundingMetrics.js';
+
+export {
+  // Enums (operator-facing values)
+  ReasoningEffort,
+  REASONING_EFFORT_VALUES,
+  DEFAULT_REASONING_EFFORT,
+  ImageDetail,
+  DEFAULT_IMAGE_DETAIL,
+  ModelId,
+  DEFAULT_GROUNDING_MODEL_ID,
+  KnobType,
+  ErrorCode,
+  // Tuning surfaces (for code that constructs sensible defaults)
+  SamplingDefaults,
+  TokenBudget,
+  Timeouts,
+  Foundry,
+} from './constants.js';