@intentcomputing/koko-agent-sdk 1.0.1

package/src/native/pruneOffscreen.ts

@@ -0,0 +1,72 @@
+import type { Frame, ViewNode } from '../types';
+
+/**
+ * Drop nodes that are mounted but nowhere on screen.
+ *
+ * Both platforms report a node as visible from its own flags alone — Android
+ * `VISIBLE && isShown()`, iOS `!hidden && alpha > 0`. Neither asks whether the
+ * node is inside the window, so a navigator screen that is still mounted but
+ * pushed aside keeps showing up in every dump. The agent then plans against a
+ * screen the user cannot see, and the user gate can stall a run on a prompt
+ * that is not there.
+ *
+ * Intersect against the window and every clipping/scroll ancestor. A mounted
+ * list row outside its scroll viewport must not become a clickable target.
+ */
+
+/** Rounding slack, in points: a node flush with the edge must survive. */
+const EPSILON = 0.5;
+
+function intersects(a: Frame, b: Frame): boolean {
+  return (
+    a.x < b.x + b.width - EPSILON &&
+    a.x + a.width > b.x + EPSILON &&
+    a.y < b.y + b.height - EPSILON &&
+    a.y + a.height > b.y + EPSILON
+  );
+}
+
+function clippedFrame(a: Frame, b: Frame): Frame | null {
+  if (!intersects(a, b)) return null;
+  const x = Math.max(a.x, b.x);
+  const y = Math.max(a.y, b.y);
+  return { x, y, width: Math.min(a.x + a.width, b.x + b.width) - x,
+    height: Math.min(a.y + a.height, b.y + b.height) - y };
+}
+
+function prune(node: ViewNode, window: Frame): ViewNode | null {
+  if (node.visible === false) return null;
+  const clip = node.clipsChildren === true || /scrollview|recyclerview|listview/i.test(node.type);
+  const childWindow = clip ? clippedFrame(node.frame, window) : window;
+  const children: ViewNode[] = [];
+  for (const child of childWindow ? node.children : []) {
+    const kept = prune(child, childWindow!);
+    if (kept) children.push(kept);
+  }
+  // A container can sit outside the window while laying children back inside
+  // it (carousels, drawers), so keep any ancestor that still leads somewhere
+  // on screen.
+  if (children.length === 0 && !intersects(node.frame, window)) return null;
+  const onScreen = intersects(node.frame, window);
+  const centerVisible = onScreen && node.frame.x + node.frame.width / 2 >= window.x &&
+    node.frame.x + node.frame.width / 2 <= window.x + window.width &&
+    node.frame.y + node.frame.height / 2 >= window.y &&
+    node.frame.y + node.frame.height / 2 <= window.y + window.height;
+  if (!children.length && !centerVisible) return null;
+  const changed = children.length !== node.children.length || children.some((c, i) => c !== node.children[i]);
+  if (!changed && centerVisible) return node;
+  // Native containers aggregate all descendant labels before clipping. Never
+  // retain a hidden row's text on an otherwise visible ancestor.
+  // A structural ancestor can be larger than the viewport while containing
+  // visible children. Keep it traversable but not actionable or labelled.
+  return { ...node, children, visible: centerVisible || children.length > 0,
+    clickable: node.clickable && centerVisible,
+    ...((!centerVisible || (changed && node.children.length)) ? {
+      text: centerVisible && node.text ? children.map((c) => c.text || c.accessibilityLabel || '').filter(Boolean).join(' ').slice(0, 240) || undefined : undefined,
+      accessibilityLabel: undefined,
+    } : {}) };
+}
+
+export function pruneOffscreen(root: ViewNode): ViewNode {
+  return prune(root, root.frame) ?? { ...root, children: [] };
+}

package/src/native/scrollViewport.ts

@@ -0,0 +1,51 @@
+import type { Frame, Point, ViewNode } from '../types';
+
+function intersect(a: Frame, b: Frame): Frame | null {
+  const x = Math.max(a.x, b.x), y = Math.max(a.y, b.y);
+  const width = Math.min(a.x + a.width, b.x + b.width) - x;
+  const height = Math.min(a.y + a.height, b.y + b.height) - y;
+  return width > 0 && height > 0 ? { x, y, width, height } : null;
+}
+
+/** Visible scroll regions, clipped by the window and every clipping ancestor. */
+export function scrollViewports(root: ViewNode): Array<{ node: ViewNode; frame: Frame }> {
+  const out: Array<{ node: ViewNode; frame: Frame }> = [];
+  const walk = (node: ViewNode, clip: Frame) => {
+    const frame = intersect(node.frame, clip);
+    if (node.scrollable === true && node.visible !== false && frame) out.push({ node, frame });
+    const clips = node.clipsChildren || /scrollview|recyclerview|listview/i.test(node.type);
+    const childClip = clips ? frame : clip;
+    if (childClip) for (const child of node.children) walk(child, childClip);
+  };
+  walk(root, root.frame);
+  return out;
+}
+
+export function boundedScrollGesture(frame: Frame, from: Point, to: Point): { from: Point; to: Point } {
+  const insetX = Math.min(12, frame.width / 4), insetY = Math.min(12, frame.height / 4);
+  const clamp = (p: Point): Point => ({
+    x: Math.max(frame.x + insetX, Math.min(frame.x + frame.width - insetX, p.x)),
+    y: Math.max(frame.y + insetY, Math.min(frame.y + frame.height - insetY, p.y)),
+  });
+  return { from: clamp(from), to: clamp(to) };
+}
+
+/** Position-only evidence: changing prices must not masquerade as scrolling. */
+export function scrollPosition(root: ViewNode, from: Point): string | null {
+  const viewport = scrollViewports(root)
+    .filter(({ frame: f }) => from.x >= f.x && from.x <= f.x + f.width &&
+      from.y >= f.y && from.y <= f.y + f.height)
+    .sort((a, b) => a.frame.width * a.frame.height - b.frame.width * b.frame.height)[0];
+  if (!viewport) return null;
+  const positions: string[] = [];
+  const walk = (n: ViewNode) => {
+    if (n.visible === false) return;
+    if (!n.children.length || n.clickable) positions.push(`${n.id}:${Math.round(n.frame.x)},${Math.round(n.frame.y)}`);
+    for (const child of n.children) walk(child);
+  };
+  walk(viewport.node);
+  // Only the hash leaves this local comparison; labels/values are never used.
+  let hash = 2166136261;
+  for (const c of positions.join('|')) hash = Math.imul(hash ^ c.charCodeAt(0), 16777619);
+  return (hash >>> 0).toString(16);
+}

package/src/privacy/redact.ts

@@ -0,0 +1,115 @@
+import type { ViewNode } from '../types';
+import { sensitiveScreenKind } from './sensitiveScreen';
+
+/**
+ * Client-side redaction before upload (defense in depth; server also redacts).
+ */
+
+export function redactViewTreeClient(node: ViewNode, maxDepth = 80): ViewNode {
+  if (sensitiveScreenKind(node)) {
+    return { ...nodeFields(node), text: '[private screen]', children: [] };
+  }
+  return walk(node, 0, maxDepth);
+}
+
+/** Applies to every outbound field, including OCR/model/history and test IDs. */
+export function redactOutbound<T>(value: T, tree?: ViewNode | null | Array<ViewNode | null | undefined>): T {
+  const secrets = new Set<string>();
+  const collect = (n: ViewNode) => {
+    if (n.secure) {
+      for (const v of [n.text, n.accessibilityLabel]) {
+        if (v && !/^[•●*\s]+$/.test(v)) secrets.add(v);
+      }
+    }
+    for (const child of n.children || []) collect(child);
+  };
+  for (const source of Array.isArray(tree) ? tree : [tree]) {
+    if (source) collect(source);
+  }
+  const visit = (v: unknown): unknown => {
+    if (typeof v === 'string') {
+      let out = v;
+      for (const secret of secrets) out = out.split(secret).join('[redacted]');
+      return out
+        .replace(/\b(?:0x)?[a-fA-F0-9]{64}\b/g, '[redacted]')
+        .replace(/\b[5KL][1-9A-HJ-NP-Za-km-z]{50,51}\b/g, '[redacted]')
+        .replace(/((?:password|passcode|密码|私钥|private\s*key)\s*[:=：]\s*)[^\s,;"}]+/gi, '$1[redacted]');
+    }
+    if (Array.isArray(v)) return v.map(visit);
+    if (v && typeof v === 'object') return Object.fromEntries(Object.entries(v).map(([k, x]) => [k, visit(x)]));
+    return v;
+  };
+  return visit(value) as T;
+}
+
+function scrub(text?: string): string | undefined {
+  if (!text) return undefined;
+  const t = text.replace(/\s+/g, ' ').trim();
+  if (!t) return undefined;
+  // A feature label is not its private value. Keep public navigation copy and
+  // addresses usable; value formats and assignments are scrubbed separately.
+  const safe = redactOutbound(t);
+  return safe.length > 80 ? `${safe.slice(0, 80)}…` : safe;
+}
+
+function walk(node: ViewNode, depth: number, maxDepth: number): ViewNode {
+  return {
+    ...nodeFields(node),
+    text: node.secure ? '[redacted]' : scrub(node.text),
+    accessibilityLabel: node.secure ? '[redacted]' : scrub(node.accessibilityLabel),
+    testID: node.secure ? undefined : scrub(node.testID),
+    children:
+      depth >= maxDepth
+        ? []
+        : node.children.slice(0, 40).map((c) => walk(c, depth + 1, maxDepth)),
+  };
+}
+
+/** Runtime trees cross a JS boundary: TypeScript types do not filter fields. */
+function nodeFields(node: ViewNode): Omit<ViewNode, 'children'> {
+  const finite = (value: unknown) => typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+  const out: Omit<ViewNode, 'children'> = {
+    id: typeof node.id === 'string' ? node.id : '',
+    type: typeof node.type === 'string' ? node.type : 'View',
+    frame: {
+      x: finite(node.frame?.x) ?? 0,
+      y: finite(node.frame?.y) ?? 0,
+      width: finite(node.frame?.width) ?? 0,
+      height: finite(node.frame?.height) ?? 0,
+    },
+    clickable: node.clickable === true,
+    enabled: node.enabled !== false,
+    visible: node.visible !== false,
+  };
+  for (const key of ['secure', 'selected', 'sheet', 'toggle', 'on', 'adjustable', 'scrollable', 'clipsChildren'] as const) {
+    if (typeof node[key] === 'boolean') out[key] = node[key];
+  }
+  for (const key of ['value', 'min', 'max'] as const) {
+    const value = finite(node[key]);
+    if (value !== undefined) out[key] = value;
+  }
+  return out;
+}
+
+export function viewTreeOutline(node: ViewNode, maxLines = 120): string {
+  if (sensitiveScreenKind(node)) return '[private screen — user completes locally]';
+  const lines: string[] = [];
+  const go = (n: ViewNode, indent: number) => {
+    if (lines.length >= maxLines) return;
+    const bits = [
+      n.clickable ? 'tap' : null,
+      n.text ? `"${n.text}"` : null,
+      n.accessibilityLabel && n.accessibilityLabel !== n.text
+        ? `a11y="${n.accessibilityLabel}"`
+        : null,
+      n.testID ? `id=${n.testID}` : null,
+    ]
+      .filter(Boolean)
+      .join(' ');
+    lines.push(`${'  '.repeat(indent)}${n.id} ${n.type} ${bits}`.trimEnd());
+    for (const c of n.children) go(c, indent + 1);
+  };
+  go(node, 0);
+  if (lines.length >= maxLines) lines.push('…(truncated)');
+  return lines.join('\n');
+}

package/src/privacy/screenEvidence.ts

@@ -0,0 +1,55 @@
+import type { ViewNode } from '../types';
+
+/** Shared by local gates and the final upload boundary; never inspect values. */
+export const PRIVATE_INPUT_PROMPT_RE =
+  /^(?:(?:请)?(?:输入|设置|确认|创建|验证|解锁|重置|更改|修改)(?:您?的|新|旧|登录|支付|交易|钱包|安全|数字|\d+位|\s)*(?:密码|安全码|验证码)|(?:登录|支付|交易|钱包)?密码|安全码|验证码|短信验证|(?:please\s+)?(?:enter|set|confirm|create|verify|unlock|reset|change)\s+(?:(?:your|new|old|wallet|payment|login|\d+[- ]digit)\s+)*(?:password|passcode|pin|otp|security\s+code|verification\s+code)|password|passcode|pin|otp|cvv)\s*[:：?？!！]?$/i;
+
+export function isPrivateInput(node: ViewNode): boolean {
+  if (node.secure) return true;
+  if (!/securetext|password|textfield|textinput|edittext|textview|textarea/i.test(node.type)) return false;
+  const metadata = `${node.testID || ''} ${node.accessibilityLabel || ''} ${node.type}`;
+  return /password|passwd|pass[-_\s]?code|\bpin\b|\botp\b|\bcvv\b|verification[-_\s]*code|security[-_\s]*code|密码|验证码|安全码/i.test(metadata) ||
+    (typeof node.text === 'string' && PRIVATE_INPUT_PROMPT_RE.test(node.text.trim()));
+}
+
+/** Text spoken by a node; test IDs are metadata, never user-facing copy. */
+export function nodeLabels(node: ViewNode): string[] {
+  return [...new Set([node.text, node.accessibilityLabel]
+    .filter((text): text is string => typeof text === 'string' && !!text.trim())
+    .map(text => text.replace(/\s+/g, ' ').trim()))];
+}
+
+export function isSdkChrome(node: ViewNode): boolean {
+  return /^koko-agent-(?:overlay|cursor|entry)(?:$|[/:-])/i.test(node.testID || '');
+}
+
+export function isSystemSheet(node: ViewNode): boolean {
+  return node.sheet === true || /uialertcontroller|alertdialog|biometricprompt/i.test(node.type);
+}
+
+/** A full-screen Pressable used to dismiss the keyboard is not an action row. */
+function isActionRow(node: ViewNode, root: ViewNode): boolean {
+  if (!node.clickable && !node.toggle && !node.adjustable) return false;
+  if (/button|menuitem|tabbaritem|navigationitem/i.test(node.type)) return true;
+  if (node === root || node.frame.height >= root.frame.height / 2) return false;
+  const hasIndependentAction = (n: ViewNode): boolean => n.children.some(child =>
+    child.clickable || child.toggle || child.adjustable || hasIndependentAction(child));
+  return !hasIndependentAction(node);
+}
+
+/** Static own copy, excluding aggregated ancestors and offered menu actions. */
+export function staticScreenLabels(root: ViewNode): Array<{ node: ViewNode; label: string }> {
+  const out: Array<{ node: ViewNode; label: string }> = [];
+  const hasCopy = (n: ViewNode): boolean => n.visible !== false && !isSdkChrome(n) &&
+    (!!nodeLabels(n).length || n.children.some(hasCopy));
+  const visit = (node: ViewNode, inAction: boolean) => {
+    if (isSdkChrome(node)) return;
+    const action = inAction || isActionRow(node, root);
+    if (node.visible !== false && !action && !node.children.some(hasCopy)) {
+      for (const label of nodeLabels(node)) out.push({ node, label });
+    }
+    for (const child of node.children) visit(child, action);
+  };
+  visit(root, false);
+  return out;
+}