@intentcomputing/koko-agent-sdk 1.0.1

package/src/assist/OnDeviceAssist.ts

@@ -0,0 +1,133 @@
+import type { ViewNode } from '../types';
+import { NativeBridge } from '../native/NativeBridge';
+import { sensitiveScreenKind } from '../privacy/sensitiveScreen';
+import { classifyGoalIntent, type AssistIntent, type IntentResult } from './intentClassify';
+import { verifiedGoalOcrLabels, verifiedWalletChoices } from './ocrTarget';
+import {
+  assistHintsModelText,
+  rankNodesForGoal,
+  type RankedNode,
+} from './nodeRank';
+
+export type OnDeviceAssistResult = {
+  intent: AssistIntent;
+  confidence: number;
+  tokens: string[];
+  ocrTokens: string[];
+  /** Goal-derived labels verified against local OCR, never raw page text. */
+  goalOcrLabels?: string[];
+  /** Only allowlisted public wallet product names, verified locally. */
+  walletChoices?: string[];
+  ranked: RankedNode[];
+  /** Compact planner block (empty when nothing useful). */
+  modelText: string;
+  backend: string;
+};
+
+export type BuildAssistOptions = {
+  /** Skip Vision OCR (tests / hosts without camera-style capture). Default false. */
+  skipOcr?: boolean;
+  /** Cap OCR tokens merged into ranking. */
+  maxOcrTokens?: number;
+};
+
+/**
+ * On-device assist: JS intent + node rank, optional native OCR.
+ * Enriches the cloud planner `ui_screen_model` — does not replace the LLM.
+ */
+export async function buildOnDeviceAssist(
+  goal: string,
+  tree: ViewNode | null | undefined,
+  opts: BuildAssistOptions = {},
+): Promise<OnDeviceAssistResult> {
+  const classified: IntentResult = classifyGoalIntent(goal);
+  if (sensitiveScreenKind(tree)) {
+    return { intent: classified.intent, confidence: classified.confidence,
+      tokens: [], ocrTokens: [], ranked: [], modelText: '', backend: 'local-private-screen' };
+  }
+  let ocrTokens: string[] = [];
+  let ocrTexts: Awaited<
+    ReturnType<typeof NativeBridge.recognizeScreenText>
+  >['texts'] = [];
+  let backend = 'js';
+
+  if (!opts.skipOcr && NativeBridge.isAvailable()) {
+    try {
+      const ocr = await NativeBridge.recognizeScreenText({
+        maxTokens: opts.maxOcrTokens ?? 24,
+        hideOverlays: true,
+      });
+      ocrTokens = (ocr.tokens || []).filter(Boolean).slice(0, opts.maxOcrTokens ?? 24);
+      ocrTexts = (ocr.texts || []).slice(0, 64);
+      if (ocr.backend) backend = `js+${ocr.backend}`;
+    } catch {
+      // Soft-fail: dump-only assist still helps the planner.
+    }
+  }
+
+  const ranked = tree
+    ? rankNodesForGoal(tree, {
+        intent: classified.intent,
+        tokens: classified.tokens,
+        ocrTokens,
+        limit: 8,
+      })
+    : [];
+
+  const modelText = assistHintsModelText(
+    classified.intent,
+    classified.confidence,
+    ranked,
+    ocrTokens,
+    backend,
+    ocrTexts,
+  );
+
+  return {
+    intent: classified.intent,
+    confidence: classified.confidence,
+    tokens: classified.tokens,
+    ocrTokens,
+    goalOcrLabels: verifiedGoalOcrLabels(goal, tree, ocrTexts),
+    walletChoices: verifiedWalletChoices(goal, tree, ocrTexts),
+    ranked,
+    modelText,
+    backend,
+  };
+}
+
+/** Sync path for unit tests / hosts that already have OCR tokens. */
+export function buildOnDeviceAssistSync(
+  goal: string,
+  tree: ViewNode | null | undefined,
+  ocrTokens: string[] = [],
+  backend = 'js',
+): OnDeviceAssistResult {
+  const classified = classifyGoalIntent(goal);
+  if (sensitiveScreenKind(tree)) return { intent: classified.intent,
+    confidence: classified.confidence, tokens: [], ocrTokens: [], ranked: [],
+    modelText: '', backend: 'local-private-screen' };
+  const ranked = tree
+    ? rankNodesForGoal(tree, {
+        intent: classified.intent,
+        tokens: classified.tokens,
+        ocrTokens,
+        limit: 8,
+      })
+    : [];
+  return {
+    intent: classified.intent,
+    confidence: classified.confidence,
+    tokens: classified.tokens,
+    ocrTokens,
+    ranked,
+    modelText: assistHintsModelText(
+      classified.intent,
+      classified.confidence,
+      ranked,
+      ocrTokens,
+      backend,
+    ),
+    backend,
+  };
+}

package/src/assist/intentClassify.ts

@@ -0,0 +1,122 @@
+/**
+ * On-device goal intent classification (structural, host-agnostic).
+ * Complements Core ML / Vision OCR — no package IDs or product copy.
+ */
+
+export type AssistIntent =
+  | 'tap'
+  | 'type'
+  | 'scroll'
+  | 'back'
+  | 'search'
+  | 'settings'
+  | 'unknown';
+
+export type IntentResult = {
+  intent: AssistIntent;
+  confidence: number;
+  /** Tokens useful for node ranking (from the goal itself). */
+  tokens: string[];
+};
+
+const STOP = new Set(
+  [
+    '打开',
+    '进入',
+    '点击',
+    '点选',
+    '按下',
+    '点',
+    '一下',
+    '这个',
+    '那个',
+    '按钮',
+    '去',
+    '到',
+    '把',
+    '将',
+    '请',
+    '帮我',
+    '我想',
+    'the',
+    'a',
+    'an',
+    'to',
+    'open',
+    'tap',
+    'click',
+    'press',
+    'go',
+  ].map((s) => s.toLowerCase()),
+);
+
+/** Extract short goal tokens for ranking (CJK chunks + latin words). */
+export function goalTokens(goal: string): string[] {
+  const raw = String(goal || '').trim();
+  if (!raw) return [];
+  const out: string[] = [];
+  const latin = raw.match(/[A-Za-z0-9_]{2,}/g) || [];
+  for (const w of latin) {
+    const n = w.toLowerCase();
+    if (!STOP.has(n)) out.push(n);
+  }
+  // CJK runs of 2–8 chars
+  const cjk = raw.match(/[\u4e00-\u9fff]{2,8}/g) || [];
+  for (const w of cjk) {
+    if (!STOP.has(w)) out.push(w);
+  }
+  return [...new Set(out)].slice(0, 12);
+}
+
+/**
+ * Cheap on-device intent — runs everywhere (JS). iOS Vision OCR enriches tokens separately.
+ */
+export function classifyGoalIntent(goal: string): IntentResult {
+  const g = String(goal || '').trim();
+  const tokens = goalTokens(g);
+  if (!g) return { intent: 'unknown', confidence: 0, tokens };
+
+  if (
+    /返回|后退|回去|关闭弹窗|关掉|dismiss|go\s*back|close\s*(this|dialog|sheet)?/i.test(
+      g,
+    )
+  ) {
+    return { intent: 'back', confidence: 0.85, tokens };
+  }
+  if (
+    /搜索|查找|搜一下|search|find\s+(for|the)|query/i.test(g) &&
+    !/打开搜索|进入搜索设置/i.test(g)
+  ) {
+    return { intent: 'search', confidence: 0.8, tokens };
+  }
+  if (
+    /输入|填写|键入|打字|type|enter\s+text|fill\s+in|写入/i.test(g) ||
+    /输入[「"'].+[」"']/.test(g)
+  ) {
+    return { intent: 'type', confidence: 0.82, tokens };
+  }
+  if (
+    /设置|設定|偏好|语言|語言|主题|外观|locale|language|theme|settings|preferences/i.test(
+      g,
+    )
+  ) {
+    return { intent: 'settings', confidence: 0.78, tokens };
+  }
+  if (
+    /滚动|下滑|上滑|滑到底|翻页|scroll|swipe\s*(up|down)|下一屏/i.test(g)
+  ) {
+    return { intent: 'scroll', confidence: 0.8, tokens };
+  }
+  if (
+    /点击|点选|按下|点一下|打开|进入|切换|选择|确认|提交|tap|click|press|open|select|confirm/i.test(
+      g,
+    )
+  ) {
+    return { intent: 'tap', confidence: 0.75, tokens };
+  }
+  // Bare label goals → tap
+  if (tokens.length >= 1 && g.length <= 24) {
+    return { intent: 'tap', confidence: 0.55, tokens };
+  }
+  return { intent: 'unknown', confidence: 0.3, tokens };
+}

package/src/assist/nodeRank.ts

@@ -0,0 +1,167 @@
+import type { ViewNode } from '../types';
+import { flattenTree } from '../selector/match';
+import type { AssistIntent } from './intentClassify';
+
+export type RankedNode = {
+  id: string;
+  label: string;
+  score: number;
+  clickable: boolean;
+  kindHint: 'button' | 'field' | 'scroll' | 'other';
+};
+
+export type OcrTextHit = {
+  text: string;
+  confidence: number;
+  frame?: { x: number; y: number; width: number; height: number };
+};
+
+function nodeLabel(n: ViewNode): string {
+  return (n.text || n.accessibilityLabel || n.testID || '').trim();
+}
+
+function kindHint(n: ViewNode, intent: AssistIntent): RankedNode['kindHint'] {
+  const t = `${n.type || ''} ${nodeLabel(n)}`.toLowerCase();
+  if (
+    /textinput|textfield|edittext|searchbar|uitextfield|rcttextfield/i.test(t) ||
+    intent === 'type' ||
+    intent === 'search'
+  ) {
+    if (/textinput|textfield|edittext|search/i.test(t)) return 'field';
+  }
+  if (/scroll|list|flatlist|collection|table/i.test(t)) return 'scroll';
+  if (n.clickable || /button|touchable|pressable/i.test(t)) return 'button';
+  return 'other';
+}
+
+function tokenHit(label: string, tokens: string[]): number {
+  if (!label || !tokens.length) return 0;
+  const soft = label.toLowerCase().replace(/\s+/g, '');
+  let best = 0;
+  for (const tok of tokens) {
+    const t = tok.toLowerCase().replace(/\s+/g, '');
+    if (!t) continue;
+    if (soft === t) best = Math.max(best, 1);
+    else if (soft.includes(t) && t.length >= 2) {
+      best = Math.max(best, 0.55 + 0.35 * (t.length / Math.max(soft.length, 1)));
+    } else if (t.includes(soft) && soft.length >= 2) {
+      best = Math.max(best, 0.4 + 0.3 * (soft.length / t.length));
+    }
+  }
+  return Math.min(1, best);
+}
+
+/**
+ * Rank dump nodes for an on-device assist pass (goal tokens + OCR tokens + intent).
+ */
+export function rankNodesForGoal(
+  tree: ViewNode,
+  opts: {
+    intent: AssistIntent;
+    tokens: string[];
+    /** Extra tokens from Vision OCR (iOS). */
+    ocrTokens?: string[];
+    limit?: number;
+  },
+): RankedNode[] {
+  const limit = opts.limit ?? 8;
+  const goalTok = [...new Set((opts.tokens || []).filter(Boolean))];
+  // OCR is supportive only — do not let whole-screen OCR drown the goal.
+  const ocrTok = [...new Set((opts.ocrTokens || []).filter(Boolean))].filter(
+    (t) => !goalTok.some((g) => g.toLowerCase() === t.toLowerCase()),
+  );
+  const flat = flattenTree(tree);
+  const scored: RankedNode[] = [];
+
+  for (const n of flat) {
+    if (!n.visible) continue;
+    const label = nodeLabel(n);
+    if (!label && !n.clickable) continue;
+    const goalHit = tokenHit(label || n.testID || '', goalTok);
+    const ocrHit = tokenHit(label || n.testID || '', ocrTok);
+    // Goal match dominates; OCR contributes at most ~0.35.
+    const hit = Math.max(goalHit, ocrHit * 0.35);
+    if (hit <= 0 && !(opts.intent === 'scroll' && /scroll|list/i.test(n.type))) {
+      continue;
+    }
+    let score = hit;
+    const kind = kindHint(n, opts.intent);
+    if (goalHit > 0) score += 0.08; // explicit goal evidence
+    if (n.clickable) score += 0.18;
+    else score -= 0.12; // non-clickable text nodes rarely the action target
+    if (n.enabled === false) score -= 0.25;
+    if (opts.intent === 'tap' || opts.intent === 'search' || opts.intent === 'settings') {
+      if (kind === 'button') score += 0.08;
+    }
+    if (opts.intent === 'type' || opts.intent === 'search') {
+      if (kind === 'field') score += 0.15;
+    }
+    if (opts.intent === 'scroll' && kind === 'scroll') score += 0.2;
+    // Prefer mid-screen affordances slightly (not status-bar chrome).
+    const cy = n.frame.y + n.frame.height / 2;
+    if (cy > 40 && cy < 900) score += 0.03;
+    scored.push({
+      id: n.id,
+      label: label || n.testID || n.type,
+      score: Math.min(1, score),
+      clickable: !!n.clickable,
+      kindHint: kind,
+    });
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  // Dedupe by label keeping best score
+  const seen = new Set<string>();
+  const out: RankedNode[] = [];
+  for (const r of scored) {
+    const key = r.label.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(r);
+    if (out.length >= limit) break;
+  }
+  return out;
+}
+
+/** Compact block for the cloud planner screen model. */
+export function assistHintsModelText(
+  intent: AssistIntent,
+  confidence: number,
+  ranked: RankedNode[],
+  ocrTokens: string[],
+  backend: string,
+  ocrTexts: OcrTextHit[] = [],
+): string {
+  if (!ranked.length && !ocrTokens.length && intent === 'unknown') return '';
+  const lines: string[] = [
+    `On-device assist (${backend}): intent=${intent} conf=${confidence.toFixed(2)}`,
+  ];
+  if (ranked.length) {
+    lines.push('top_candidates:');
+    for (const r of ranked.slice(0, 6)) {
+      lines.push(
+        `- score=${r.score.toFixed(2)} id=${r.id} label="${r.label.replace(/"/g, '')}" clickable=${r.clickable} kind=${r.kindHint}`,
+      );
+    }
+  }
+  if (ocrTokens.length) {
+    lines.push(`ocr_tokens: ${ocrTokens.slice(0, 16).join(', ')}`);
+  }
+  const positioned = ocrTexts.filter(
+    (hit) => hit.text.trim() && hit.frame && hit.frame.width > 0 && hit.frame.height > 0,
+  );
+  if (positioned.length) {
+    lines.push(
+      'ocr_targets (visible text; point is window-absolute evidence for tap(point)):',
+    );
+    for (const hit of positioned.slice(0, 16)) {
+      const f = hit.frame!;
+      const x = Math.round(f.x + f.width / 2);
+      const y = Math.round(f.y + f.height / 2);
+      lines.push(
+        `- text="${hit.text.replace(/"/g, '')}" point=(${x},${y}) confidence=${hit.confidence.toFixed(2)}`,
+      );
+    }
+  }
+  return lines.join('\n');
+}

package/src/assist/ocrTarget.ts

@@ -0,0 +1,146 @@
+import type { Point, Frame, ViewNode } from '../types';
+
+export type PositionedOcrText = {
+  text: string;
+  confidence: number;
+  frame?: Frame;
+};
+
+function normalized(value: string): string {
+  return value
+    .toLocaleLowerCase()
+    .replace(/[^\p{L}\p{N}]+/gu, '');
+}
+
+/** Visible web regions let a short translated action stay scoped to the page. */
+export function visibleWebViewFrames(root: ViewNode): Frame[] {
+  const frames: Frame[] = [];
+  const walk = (node: ViewNode) => {
+    if (node.visible === false) return;
+    if (/webview/i.test(node.type) && node.frame.width > 0 && node.frame.height > 0) {
+      const left = Math.max(root.frame.x, node.frame.x);
+      const top = Math.max(root.frame.y, node.frame.y);
+      const right = Math.min(root.frame.x + root.frame.width, node.frame.x + node.frame.width);
+      const bottom = Math.min(root.frame.y + root.frame.height, node.frame.y + node.frame.height);
+      if (right > left && bottom > top) frames.push({ x: left, y: top, width: right - left, height: bottom - top });
+    }
+    for (const child of node.children) walk(child);
+  };
+  walk(root);
+  return frames;
+}
+
+export function ocrHitInFrames(hit: PositionedOcrText, frames: Frame[]): boolean {
+  if (!hit.frame) return false;
+  const x = hit.frame.x + hit.frame.width / 2, y = hit.frame.y + hit.frame.height / 2;
+  return frames.some(f => x >= f.x && x <= f.x + f.width && y >= f.y && y <= f.y + f.height);
+}
+
+/** Candidate labels are derived only from the user's goal, never screen text. */
+export function goalOcrCandidates(goal: string): string[] {
+  if (/password|passcode|private\s*key|mnemonic|recovery\s*phrase|seed\s*phrase|密码|助记词|私钥|验证码/i.test(goal)) return [];
+  const stop = new Set(['please', 'for', 'me', 'my', 'the', 'a', 'an', 'to', 'and', 'then', 'of', 'in', 'on', 'from', 'with']);
+  const words = (goal.match(/[A-Za-z][A-Za-z0-9'-]*|[\u3400-\u9fff]{2,16}/g) || [])
+    .filter(word => !stop.has(word.toLowerCase())).slice(0, 20);
+  const out: string[] = [];
+  for (const size of [3, 2, 1]) {
+    for (let i = 0; i + size <= words.length; i++) {
+      const label = words.slice(i, i + size).join(' ');
+      if (label.length >= 3 && label.length <= 60 && !out.includes(label)) out.push(label);
+    }
+  }
+  // Chinese goals often have no spaces between verbs and objects.
+  for (const word of words.filter(w => /[\u3400-\u9fff]/.test(w))) {
+    for (let length = Math.min(6, word.length); length >= 2; length--) {
+      for (let i = 0; i + length <= word.length; i++) {
+        const label = word.slice(i, i + length);
+        if (!out.includes(label)) out.push(label);
+      }
+    }
+  }
+  return out.slice(0, 100);
+}
+
+export function verifiedGoalOcrLabels(goal: string, root: ViewNode | null | undefined, texts: PositionedOcrText[]): string[] {
+  if (!root) return [];
+  const regions = visibleWebViewFrames(root);
+  if (!regions.length) return [];
+  const onPage = texts.filter(hit => ocrHitInFrames(hit, regions));
+  return goalOcrCandidates(goal).filter(label =>
+    !!findOcrTextHit(onPage, label, { translatedWebRegions: regions }),
+  ).slice(0, 6);
+}
+
+// These are public product names, never account names, addresses or page text.
+export const PUBLIC_WALLET_CHOICES = [
+  'WalletConnect', 'MetaMask', 'Coinbase Wallet', 'Uniswap Mobile', 'Uniswap Wallet',
+  'Binance Wallet', 'Trust Wallet', 'Phantom', 'OKX Wallet', 'Rabby Wallet', 'OneKey', 'UKey', 'UKey Wallet', 'Solflare',
+] as const;
+
+export function walletConnectionAllowed(goal: string): boolean {
+  return /(connect|连接|連接)/i.test(goal) && /(wallet|钱包|錢包)/i.test(goal) &&
+    !/\b(?:do not|don't|never|without)\s+(?:connect|link|select|choose)\b|(?:不要|禁止|不|无需|無需)(?:连接|連接|选择|選擇)/i.test(goal) &&
+    !/password|passcode|private\s*key|mnemonic|recovery\s*phrase|seed\s*phrase|密码|助记词|私钥|验证码/i.test(goal);
+}
+
+export function verifiedWalletChoices(goal: string, root: ViewNode | null | undefined, texts: PositionedOcrText[]): string[] {
+  if (!root || !walletConnectionAllowed(goal)) return [];
+  const regions = visibleWebViewFrames(root);
+  if (!regions.length) return [];
+  const page = texts.filter(hit => ocrHitInFrames(hit, regions));
+  // Require a visible chooser heading as well as multiple known product names.
+  if (!['connect wallet', '连接钱包', '連接錢包'].some(label => !!findOcrTextHit(page, label))) return [];
+  const choices = PUBLIC_WALLET_CHOICES.filter(label => !!findOcrTextHit(page, label));
+  return choices.length >= 2 ? choices.slice(0, 8) : [];
+}
+
+/**
+ * Resolve text that is visible in a WebView/Canvas but absent from the native
+ * accessibility tree. The frame comes from native OCR and is already in the
+ * same window-absolute logical coordinate space used by dispatchTap.
+ */
+export function findOcrTextPoint(
+  texts: PositionedOcrText[],
+  wanted: string,
+): Point | null {
+  const hit = findOcrTextHit(texts, wanted);
+  const frame = hit?.frame;
+  return frame
+    ? { x: frame.x + frame.width / 2, y: frame.y + frame.height / 2 }
+    : null;
+}
+
+export function findOcrTextHit(
+  texts: PositionedOcrText[],
+  wanted: string,
+  options: { translatedWebRegions?: Frame[] } = {},
+): PositionedOcrText | null {
+  const target = normalized(wanted);
+  if (target.length < 2) return null;
+
+  const matches: PositionedOcrText[] = [];
+  for (const hit of texts) {
+    const frame = hit.frame;
+    if (!frame || ![frame.x, frame.y, frame.width, frame.height, hit.confidence].every(Number.isFinite) ||
+        frame.x < 0 || frame.y < 0 || frame.width <= 0 || frame.height <= 0 || hit.confidence < 0.6) continue;
+    const text = normalized(hit.text);
+    if (text === target) matches.push(hit);
+  }
+  // A fragment in prose or two equal labels gives no reliable target.
+  if (matches.length === 1) return matches[0]!;
+  if (matches.length > 1) return null;
+
+  // Common translated connect controls. A shortened action word is considered
+  // only inside a visible web view, with strong OCR and a single match.
+  const connectLabels = new Set(['connect', 'connectwallet', '连接', '连接钱包', '連接', '連接錢包']);
+  const regions = options.translatedWebRegions || [];
+  if (!connectLabels.has(target) || !regions.length) return null;
+  const translated = texts.filter(hit => {
+    const f = hit.frame;
+    return connectLabels.has(normalized(hit.text)) && f &&
+      [f.x, f.y, f.width, f.height, hit.confidence].every(Number.isFinite) &&
+      f.x >= 0 && f.y >= 0 && f.width > 0 && f.height > 0 && hit.confidence >= 0.8 &&
+      ocrHitInFrames(hit, regions);
+  });
+  return translated.length === 1 ? translated[0]! : null;
+}

package/src/assist/webVisualChange.ts

@@ -0,0 +1,24 @@
+import { ocrHitInFrames, visibleWebViewFrames, type PositionedOcrText } from './ocrTarget';
+import type { ViewNode } from '../types';
+
+/** Local-only text sets: callers may report only whether content changed. */
+export function webVisualLabels(root: ViewNode | null, hits: PositionedOcrText[]): Set<string> | null {
+  if (!root) return null;
+  const frames = visibleWebViewFrames(root);
+  if (!frames.length) return null;
+  const labels = new Set<string>();
+  for (const hit of hits) {
+    if (hit.confidence < 0.7 || !ocrHitInFrames(hit, frames)) continue;
+    const text = hit.text.trim().toLocaleLowerCase().replace(/\s+/g, ' ');
+    // Ignore prices, counts and other rapidly changing numeric-only strings.
+    if (text.length >= 2 && /[\p{L}]/u.test(text)) labels.add(text);
+  }
+  return labels.size ? labels : null;
+}
+
+export function webVisualChanged(before: Set<string> | null, after: Set<string> | null): boolean {
+  if (!before || !after) return false;
+  let additions = 0;
+  for (const text of after) if (!before.has(text)) additions++;
+  return additions >= 2;
+}

package/src/client/ActionOutcomeUnknownError.ts

@@ -0,0 +1,18 @@
+import { message, resolveText, type LocalizedText } from '../i18n';
+
+/** A dispatched action may have committed; replay would risk a duplicate. */
+export class ActionOutcomeUnknownError extends Error {
+  readonly localizedMessage = message('runtime.uncertainOutcome');
+  constructor(readonly detail: string) {
+    super(resolveText(message('runtime.uncertainOutcome')));
+    this.name = 'ActionOutcomeUnknownError';
+  }
+}
+
+/** Validation rejected a target before an action was dispatched. Replan safely. */
+export class ActionValidationError extends Error {
+  constructor(readonly localizedMessage: LocalizedText) {
+    super(resolveText(localizedMessage));
+    this.name = 'ActionValidationError';
+  }
+}