@intentcomputing/koko-agent-sdk 1.0.1

package/src/host/HostActions.ts

@@ -0,0 +1,349 @@
+/**
+ * Optional host-declared actions — the low-friction way for complex Apps
+ * (OneKey / Tamagui) when native dump cannot see icons without a11y/testID.
+ *
+ * Hosts register controls only while mounted and visible. Callbacks supplement
+ * native controls; registration does not establish visibility or identity.
+ *
+ * IMPORTANT: host ids are NOT dump nodeIds. Never emit `id=host:…` in the
+ * screen model — planners will tap them and fail with "nodeId not found".
+ *
+ * Optional `run`: after a unique current native control is bound, a callback
+ * may execute its behavior instead of dispatching a geometric touch.
+ */
+
+import { flattenTree, textMatches } from '../selector/match';
+import type { ViewNode } from '../types';
+
+export type HostActionKind =
+  | 'button'
+  | 'tab'
+  | 'nav'
+  | 'field'
+  | 'sheet'
+  | 'other';
+
+/**
+ * Host action matching: also consider optional aliases (e.g. prior-locale tab copy).
+ */
+export type HostAction = {
+  /** Stable id, e.g. "tab.home" / "header.settings". */
+  id: string;
+  /** Human label shown to the planner (prefer host locale). */
+  label: string;
+  kind?: HostActionKind;
+  /** Prefer testID when the host already has one. */
+  testID?: string;
+  /** Free-text / a11y match fallback. */
+  text?: string;
+  /** Extra NL match strings (other locales / synonyms). */
+  aliases?: string[];
+  /** Optional deep-link or route hint for narration only. */
+  route?: string;
+  /**
+   * Host-provided invoke. Used only after unique native target binding.
+   * Register only while the CTA is mounted and visible; unregister on blur.
+   */
+  run?: () => void | Promise<void>;
+  /**
+   * When true (default for one-shot CTAs), unregister after a successful host run.
+   * Tabs / persistent chrome should set `oneshot: false`.
+   */
+  oneshot?: boolean;
+};
+
+type Listener = (actions: HostAction[]) => void;
+
+class HostActionsStoreImpl {
+  private actions: HostAction[] = [];
+  private listeners = new Set<Listener>();
+
+  /** Replace the whole catalog (typical: once at app start). */
+  setActions(actions: HostAction[]) {
+    this.actions = sanitize(actions);
+    this.emit();
+  }
+
+  /** Append / upsert by id. */
+  registerActions(actions: HostAction[]) {
+    const map = new Map(this.actions.map((a) => [a.id, a]));
+    for (const a of sanitize(actions)) map.set(a.id, a);
+    this.actions = [...map.values()];
+    this.emit();
+  }
+
+  /** Drop a previously registered action (e.g. EmptyWallet unmount). */
+  unregisterAction(id: string) {
+    const key = String(id ?? '').replace(/^host:/, '').trim();
+    if (!key) return;
+    const next = this.actions.filter((a) => a.id !== key);
+    if (next.length === this.actions.length) return;
+    this.actions = next;
+    this.emit();
+  }
+
+  clear() {
+    this.actions = [];
+    this.emit();
+  }
+
+  getActions(): HostAction[] {
+    return this.actions.slice();
+  }
+
+  getById(id: string): HostAction | undefined {
+    const key = id.replace(/^host:/, '').trim();
+    return this.actions.find((a) => a.id === key);
+  }
+
+  /** Match by label / text / testID (planner taps by copy, not host id). */
+  findForTap(needle: {
+    label?: string;
+    text?: string;
+    testID?: string;
+  }): HostAction | undefined {
+    const testID = (needle.testID || '').trim();
+    if (testID) {
+      const matches = this.actions.filter((a) => a.testID === testID);
+      // An explicit native identity must not fall back to a different callback.
+      return matches.length === 1 ? matches[0] : undefined;
+    }
+    const raw = (needle.text || needle.label || '').trim();
+    if (!raw) return undefined;
+    // Also match 「发现」 inside narration like「打开「发现」标签页」.
+    const quoted = raw.match(/「([^」]+)」/)?.[1]?.trim();
+    const needles = quoted && quoted !== raw ? [raw, quoted] : [raw];
+
+    const hits = this.actions.filter(a => {
+      const candidates = [a.label, a.text, ...(a.aliases ?? [])].filter(
+        Boolean,
+      ) as string[];
+      return needles.some(label => candidates.some(c => textMatches(c, label, true)));
+    });
+    // Repeated row labels need the native selector/geometry or a unique testID.
+    // Picking the first callback silently targets a different item.
+    return hits.length === 1 ? hits[0] : undefined;
+  }
+
+  subscribe(listener: Listener): () => void {
+    this.listeners.add(listener);
+    listener(this.getActions());
+    return () => this.listeners.delete(listener);
+  }
+
+  private emit() {
+    const snap = this.getActions();
+    for (const l of this.listeners) l(snap);
+  }
+}
+
+function sanitize(actions: HostAction[]): HostAction[] {
+  const out: HostAction[] = [];
+  for (const a of actions) {
+    const id = String(a?.id ?? '').trim();
+    const label = String(a?.label ?? '').trim();
+    if (!id || !label) continue;
+    const kind = a.kind ?? 'button';
+    const hasRun = typeof a.run === 'function';
+    // Tabs stay registered; button CTAs with run are one-shot by default.
+    const oneshot =
+      typeof a.oneshot === 'boolean'
+        ? a.oneshot
+        : hasRun && kind !== 'tab';
+    out.push({
+      id: id.slice(0, 80),
+      label: label.slice(0, 80),
+      kind,
+      testID: a.testID?.trim().slice(0, 120) || undefined,
+      text: a.text?.trim().slice(0, 80) || undefined,
+      aliases: Array.isArray(a.aliases)
+        ? a.aliases
+            .map((s) => String(s ?? '').trim().slice(0, 80))
+            .filter(Boolean)
+            .slice(0, 8)
+        : undefined,
+      route: a.route?.trim().slice(0, 120) || undefined,
+      // Keep host callback by reference — not serializable, intentional.
+      run: hasRun ? a.run : undefined,
+      oneshot,
+    });
+  }
+  return out.slice(0, 80);
+}
+
+function hostNeedle(a: HostAction): string {
+  return (a.text || a.label || '').trim();
+}
+
+/**
+ * A unique callback catalog entry is not a unique native control. Bind it to
+ * the node already chosen by the full selector (including index/nearText),
+ * otherwise the caller must preserve that selector and use a native touch.
+ */
+export function canRunHostActionForNode(
+  action: HostAction,
+  selected: ViewNode | null | undefined,
+  root: ViewNode | null | undefined,
+): boolean {
+  if (!selected || !root || typeof action.run !== 'function') return false;
+  const nodes = new Map<string, ViewNode>();
+  const parents = new Map<string, ViewNode>();
+  const collect = (node: ViewNode, parent?: ViewNode) => {
+    if (node.visible === false || node.enabled === false) return;
+    nodes.set(node.id, node);
+    if (parent) parents.set(node.id, parent);
+    for (const child of node.children) collect(child, node);
+  };
+  collect(root);
+  const current = nodes.get(selected.id);
+  if (!current || current.type !== selected.type || current.text !== selected.text ||
+      current.accessibilityLabel !== selected.accessibilityLabel || current.testID !== selected.testID) return false;
+  const usable = (node: ViewNode) => {
+    const frame = node.frame;
+    const cx = frame.x + frame.width / 2, cy = frame.y + frame.height / 2;
+    return node.id !== root.id && frame.width > 0 && frame.height > 0 &&
+      frame.width * frame.height < root.frame.width * root.frame.height * 0.55 &&
+      cx >= root.frame.x && cx <= root.frame.x + root.frame.width &&
+      cy >= root.frame.y && cy <= root.frame.y + root.frame.height;
+  };
+  const owner = (node: ViewNode): ViewNode | undefined => {
+    for (let current: ViewNode | undefined = node; current; current = parents.get(current.id)) {
+      if (usable(current) && (current.clickable || current.toggle || current.adjustable ||
+          /button|tabbaritem|bottomnavigationitem/i.test(current.type))) return current;
+    }
+    return undefined;
+  };
+  const targetOwner = owner(current);
+  if (action.testID) {
+    const matches = [...nodes.values()].filter(node => node.testID === action.testID && usable(node));
+    if (matches.length !== 1) return false;
+    const anchor = matches[0];
+    // A unique explicit testID may identify a non-clickable text leaf, which
+    // Fabric commonly nests inside the actual Pressable selected by geometry.
+    return anchor.id === current.id || (!!targetOwner && owner(anchor)?.id === targetOwner.id);
+  }
+  if (!targetOwner) return false;
+  const needles = [action.text || action.label, ...(action.aliases || [])].filter(Boolean);
+  const owners = new Set<string>();
+  for (const node of nodes.values()) {
+    if (!needles.some(label => textMatches(node.text, label, true) || textMatches(node.accessibilityLabel, label, true))) continue;
+    const control = owner(node);
+    if (control) owners.add(control.id);
+  }
+  return owners.size === 1 && owners.has(targetOwner.id);
+}
+
+/**
+ * Callbacks declare behavior, not visibility. Only current visible native
+ * evidence enters the model; missing native content can use fresh local OCR.
+ */
+export function filterVisibleHostActions(
+  actions: HostAction[],
+  root: ViewNode | null | undefined,
+): HostAction[] {
+  if (!actions.length) return [];
+  if (!root) return [];
+  const nodes = flattenTree(root).filter(n => n.visible !== false && n.enabled !== false);
+  const inDump = (a: HostAction, exact = false): boolean => {
+    if (a.testID && nodes.some((n) => n.testID === a.testID)) return true;
+    const needles = [hostNeedle(a), ...(a.aliases ?? [])]
+      .map((s) => s.trim())
+      .filter(Boolean);
+    if (!needles.length) return false;
+    for (const needle of needles) {
+      const hit = nodes.some((n) => {
+        if (exact) {
+          return (
+            textMatches(n.text, needle, true) ||
+            textMatches(n.accessibilityLabel, needle, true)
+          );
+        }
+        return (
+          textMatches(n.text, needle) ||
+          textMatches(n.accessibilityLabel, needle)
+        );
+      });
+      if (hit) return true;
+    }
+    return false;
+  };
+  return actions.filter((a) => {
+    // Exact: avoid「钱包」tab matching「创建助记词钱包」onboarding copy.
+    if (a.kind === 'tab') return inDump(a, true);
+    return inDump(a, true);
+  });
+}
+
+/**
+ * Compact lines merged into ui_screen_model.
+ * No fake node ids — tap must use text「…」 / testID, never id=host:….
+ */
+export function hostActionsModelText(actions: HostAction[]): string {
+  if (!actions.length) return '';
+  return actions
+    .map((a) => {
+      const kind = a.kind ?? 'button';
+      const tip = a.testID
+        ? ` testID=${a.testID}`
+        : a.text
+          ? ` text「${a.text}」`
+          : ` text「${a.label}」`;
+      const via = typeof a.run === 'function' ? '（host run）' : '（按文案 tap，禁止用 host id）';
+      return `- [host:${kind}] 「${a.label}」${tip}${via}`;
+    })
+    .join('\n');
+}
+
+export const HostActionsStore = new HostActionsStoreImpl();
+
+export function setHostActions(actions: HostAction[]) {
+  HostActionsStore.setActions(actions);
+}
+
+export function registerHostActions(actions: HostAction[]) {
+  HostActionsStore.registerActions(actions);
+}
+
+export function unregisterHostAction(id: string) {
+  HostActionsStore.unregisterAction(id);
+}
+
+export function clearHostActions() {
+  HostActionsStore.clear();
+}
+
+/**
+ * Upper bound on a single host action. These callbacks are app code we do not
+ * control: one that never settles wedges the whole goal with no step, no log
+ * line and no timeout above it. Observed as a goal sitting for minutes without
+ * ever reaching the planner.
+ */
+export const HOST_ACTION_TIMEOUT_MS = 8000;
+
+/**
+ * Await a host callback without letting it hang the agent. Resolves false when
+ * it throws or outruns its budget. False means the outcome is unknown: the
+ * callback may already have committed work, so callers must not replay a tap.
+ */
+export async function runHostAction(
+  run: () => void | Promise<void>,
+  timeoutMs: number = HOST_ACTION_TIMEOUT_MS,
+): Promise<boolean> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    await Promise.race([
+      Promise.resolve(run()),
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error(`host action exceeded ${timeoutMs}ms`)),
+          timeoutMs,
+        );
+      }),
+    ]);
+    return true;
+  } catch {
+    return false;
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}

package/src/i18n/index.ts

@@ -0,0 +1,229 @@
+import { runtimeEn, runtimeZh, type RuntimeMessageKey } from './runtimeMessages';
+import { HostAppearanceStore } from '../provider/HostAppearanceStore';
+import type { KokoLocale } from '../types';
+
+export type MessageKey = RuntimeMessageKey
+  | 'brand.defaultName'
+  | 'brand.defaultTagline'
+  | 'brand.defaultPlaceholder'
+  | 'brand.defaultMark'
+  | 'sheet.recentTasks'
+  | 'sheet.timeline'
+  | 'sheet.emptyTasks'
+  | 'sheet.emptyNodesTitle'
+  | 'sheet.emptyNodesHint'
+  | 'sheet.send'
+  | 'sheet.closeA11y'
+  | 'sheet.stopA11y'
+  | 'sheet.inputA11y'
+  | 'sheet.deleteTaskTitle'
+  | 'sheet.deleteTaskBody'
+  | 'sheet.deleteTask'
+  | 'status.running'
+  | 'status.success'
+  | 'status.failed'
+  | 'status.cancelled'
+  | 'kind.user'
+  | 'kind.plan'
+  | 'kind.dump'
+  | 'kind.find'
+  | 'kind.tap'
+  | 'kind.swipe'
+  | 'kind.type'
+  | 'kind.confirm'
+  | 'kind.info'
+  | 'kind.error'
+  | 'resume.note'
+  | 'resume.verifyFirst'
+  | 'progress.running'
+  | 'progress.success'
+  | 'progress.failed'
+  | 'progress.cancelled'
+  | 'progress.openA11y'
+  | 'progress.stopA11y'
+  | 'gate.check'
+  | 'gate.checking'
+  | 'gate.stillPrivate'
+  | 'gate.unreadable'
+  | 'gate.verifyResultFirst'
+  | 'gate.replayStopped'
+  | 'dialog.cancel'
+  | 'dialog.confirm'
+  | 'dialog.continue'
+  | 'dialog.askTitle'
+  | 'dialog.askPlaceholder'
+  | 'confirm.preflightTitle'
+  | 'confirm.preflightBody'
+  | 'confirm.inlineTitle'
+  | 'confirm.inlineBody';
+
+const zh: Record<MessageKey, string> = {
+  ...runtimeZh,
+  'brand.defaultName': 'Koko',
+  'brand.defaultTagline': '用自然语言操作当前 App',
+  'brand.defaultPlaceholder': '跟 Koko 说你想做什么…',
+  'brand.defaultMark': 'K',
+  'sheet.recentTasks': '最近任务',
+  'sheet.timeline': '执行记录',
+  'sheet.emptyTasks': '还没有任务，在下方说一句开始',
+  'sheet.emptyNodesTitle': '暂无步骤',
+  'sheet.emptyNodesHint': '发送指令后，这里会显示每一步操作',
+  'sheet.send': '发送',
+  'sheet.closeA11y': '关闭会话',
+  'sheet.stopA11y': '停止当前任务',
+  'sheet.inputA11y': '自然语言指令',
+  'sheet.deleteTaskTitle': '删除任务',
+  'sheet.deleteTaskBody': '删除「{title}」？此操作不可撤销。',
+  'sheet.deleteTask': '删除',
+  'status.running': '进行中',
+  'status.success': '完成',
+  'status.failed': '失败',
+  'status.cancelled': '已取消',
+  'kind.user': '用户',
+  'kind.plan': '规划',
+  'kind.dump': '扫描',
+  'kind.find': '定位',
+  'kind.tap': '点击',
+  'kind.swipe': '滑动',
+  'kind.type': '输入',
+  'kind.confirm': '确认',
+  'kind.info': '信息',
+  'kind.error': '错误',
+  'resume.note': 'App 重载中断了上一次执行，已自动继续',
+  'resume.verifyFirst':
+    '注意：App 在上一次执行中途重载了，这个目标可能已经完成。请先观察当前界面判断目标是否已达成；如已达成，直接返回 done(success=true) 并说明依据，不要重复操作。另外，重载前学到的界面文案可能已失效，请以当前界面为准。',
+  'progress.running': '执行中…',
+  'progress.success': '成功',
+  'progress.failed': '失败',
+  'progress.cancelled': '已取消',
+  'progress.openA11y': '打开会话',
+  'progress.stopA11y': '停止当前任务',
+  'gate.check': '检查状态',
+  'gate.checking': '正在本机检查是否已离开隐私页面…',
+  'gate.stillPrivate': '仍在隐私页面，请在宿主中点击下一步并完成操作；离开此页后自动继续',
+  'gate.unreadable': '暂时无法确认页面状态，请在宿主中完成操作后再检查',
+  'gate.verifyResultFirst': '已将进行中的操作交给用户处理。先核验这次操作的结果，继续剩余步骤；若失败或退回之前的入口/说明页，报告问题并停止，禁止从头重新创建、导入或提交。不能把返回入口当作尚未开始。',
+  'gate.replayStopped': '检测到人工接管后再次进入已发起过的操作，已停止，避免重复创建或提交。请先确认上一次操作的结果。',
+  'dialog.cancel': '取消',
+  'dialog.confirm': '确认',
+  'dialog.continue': '继续',
+  'dialog.askTitle': '需要你的确认',
+  'dialog.askPlaceholder': '输入补充信息',
+  'confirm.preflightTitle': '确认执行此任务？',
+  'confirm.preflightBody':
+    '该操作可能涉及资金或敏感数据：\n「{label}」\n\n确认后将自动执行，过程中可点停止取消。',
+  'confirm.inlineTitle': '需要你的确认',
+  'confirm.inlineBody': '即将执行敏感操作：{label}',
+};
+
+const en: Record<MessageKey, string> = {
+  ...runtimeEn,
+  'brand.defaultName': 'Koko',
+  'brand.defaultTagline': 'Control this app with natural language',
+  'brand.defaultPlaceholder': 'Tell Koko what you want to do…',
+  'brand.defaultMark': 'K',
+  'sheet.recentTasks': 'Recent tasks',
+  'sheet.timeline': 'Activity',
+  'sheet.emptyTasks': 'No tasks yet — type a goal below',
+  'sheet.emptyNodesTitle': 'No steps yet',
+  'sheet.emptyNodesHint': 'After you send a goal, each step shows up here',
+  'sheet.send': 'Send',
+  'sheet.closeA11y': 'Close chat',
+  'sheet.stopA11y': 'Stop current task',
+  'sheet.inputA11y': 'Natural language command',
+  'sheet.deleteTaskTitle': 'Delete task',
+  'sheet.deleteTaskBody': 'Delete “{title}”? This cannot be undone.',
+  'sheet.deleteTask': 'Delete',
+  'status.running': 'Running',
+  'status.success': 'Done',
+  'status.failed': 'Failed',
+  'status.cancelled': 'Cancelled',
+  'kind.user': 'User',
+  'kind.plan': 'Plan',
+  'kind.dump': 'Scan',
+  'kind.find': 'Find',
+  'kind.tap': 'Tap',
+  'kind.swipe': 'Swipe',
+  'kind.type': 'Type',
+  'kind.confirm': 'Confirm',
+  'kind.info': 'Info',
+  'kind.error': 'Error',
+  'resume.note': 'A reload interrupted the previous run; continuing it',
+  'resume.verifyFirst':
+    'Note: the App reloaded partway through the previous run, so this goal may already be complete. Inspect the current screen first and decide whether it is already satisfied; if it is, return done(success=true) with your evidence and change nothing. Labels learned before the reload may no longer exist — trust the current screen.',
+  'progress.running': 'Working…',
+  'progress.success': 'Done',
+  'progress.failed': 'Failed',
+  'progress.cancelled': 'Cancelled',
+  'progress.openA11y': 'Open chat',
+  'progress.stopA11y': 'Stop current task',
+  'gate.check': 'Check status',
+  'gate.checking': 'Checking locally whether you have left the private screen…',
+  'gate.stillPrivate': 'Still on a private screen. Complete the next steps in the app; the assistant resumes after you leave it.',
+  'gate.unreadable': 'Unable to confirm the screen. Complete the step in the app, then check again.',
+  'gate.verifyResultFirst': 'An in-flight operation was handed to the user. Verify its result and continue remaining steps. If it failed or returned to a previous entry or instruction screen, report the issue and stop. Do not restart creation, import, or submission from the beginning.',
+  'gate.replayStopped': 'Stopped a repeated operation after user handoff to avoid duplicate creation or submission. Check the previous operation result first.',
+  'dialog.cancel': 'Cancel',
+  'dialog.confirm': 'Confirm',
+  'dialog.continue': 'Continue',
+  'dialog.askTitle': 'We need your input',
+  'dialog.askPlaceholder': 'Type a reply',
+  'confirm.preflightTitle': 'Run this task?',
+  'confirm.preflightBody':
+    'This may involve money or sensitive data:\n「{label}」\n\nAfter you confirm, it runs unattended — tap stop to cancel.',
+  'confirm.inlineTitle': 'Confirmation needed',
+  'confirm.inlineBody': 'About to run a sensitive action: {label}',
+};
+
+const catalogs: Record<KokoLocale, Record<MessageKey, string>> = { zh, en };
+
+export function normalizeLocale(locale?: string | null): KokoLocale {
+  if (!locale) return 'en';
+  const lower = locale.toLowerCase();
+  if (lower.startsWith('zh')) return 'zh';
+  if (lower.startsWith('en')) return 'en';
+  return 'en';
+}
+
+export function gatewayLocale(locale: KokoLocale): string {
+  return locale === 'zh' ? 'zh-CN' : 'en-US';
+}
+
+export function t(
+  locale: KokoLocale,
+  key: MessageKey,
+  vars?: Record<string, string | number>,
+): string {
+  const table = catalogs[locale] ?? catalogs.en;
+  let out = table[key] ?? catalogs.en[key] ?? key;
+  if (vars) {
+    for (const [k, v] of Object.entries(vars)) {
+      out = out.replace(new RegExp(`\\{${k}\\}`, 'g'), () => String(v));
+    }
+  }
+  return out;
+}
+
+
+export type MessageVars = Record<string, string | number>;
+export type LocalizedMessage = {
+  readonly kokoMessage: true;
+  readonly key: MessageKey;
+  readonly vars?: MessageVars;
+  readonly fallbackLocale?: string;
+};
+export type LocalizedText = string | LocalizedMessage;
+
+/** Resolve at use time so an in-flight task follows host language changes. */
+export function currentLocale(fallbackLocale?: string | null): KokoLocale {
+  return normalizeLocale(HostAppearanceStore.get().locale ?? fallbackLocale);
+}
+export function message(key: MessageKey, vars?: MessageVars, fallbackLocale?: string): LocalizedMessage {
+  return { kokoMessage: true, key, vars, fallbackLocale };
+}
+export function resolveText(value: LocalizedText): string {
+  return typeof value === 'string' ? value : t(currentLocale(value.fallbackLocale), value.key, value.vars);
+}
+export function runtimeText(key: MessageKey, vars?: MessageVars, fallbackLocale?: string | null): string {
+  return t(currentLocale(fallbackLocale), key, vars);
+}