@blario/react-native 0.0.0 → 0.1.0

package/src/agent/frames.ts

@@ -0,0 +1,193 @@
+import { parsePageContentRequest } from '../actions/parse.ts';
+import type { ActionErrorCode, ActionOutcome, PageContentRequest } from '../actions/types.ts';
+import { isRecord, nonEmptyString } from '../fiber/narrow.ts';
+
+export interface ScreenshotOptions {
+  quality: number;
+  maxWidth: number;
+}
+
+export type InboundFrame =
+  | { kind: 'ping' }
+  | { kind: 'agent_command'; commandId: string; data: unknown }
+  | { kind: 'agent_command_batch'; commandId: string; data: unknown }
+  | { kind: 'request_dom_state'; commandId: string }
+  | { kind: 'request_text_dom'; commandId: string; options: PageContentRequest }
+  | { kind: 'request_screenshot'; commandId: string; options: ScreenshotOptions }
+  | { kind: 'request_element_context'; commandId: string; agentIds: string[] }
+  | { kind: 'unknown'; name: string };
+
+export type RoutedFrame = Exclude<InboundFrame, { kind: 'ping' | 'unknown' }>;
+
+export interface DomStateResponseData {
+  schema_version: 3;
+  platform: 'react-native';
+  url: string;
+  title: string;
+  screen: string;
+  routes: string[];
+  html_snapshot: string;
+}
+
+export interface TextDomResponseData {
+  url: string;
+  title: string;
+  screen: string;
+  text_tree: string;
+  refs: never[];
+}
+
+export interface BoundingBox {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface ElementContextItem {
+  agent_id: string;
+  found: boolean;
+  tag_name?: string;
+  visible?: boolean;
+  line?: string;
+  bounding_box?: BoundingBox;
+  text_content?: string;
+  placeholder?: string;
+  nearest_label?: string;
+  sibling_text?: string[];
+}
+
+export interface ElementContextResponseData {
+  elements: ElementContextItem[];
+}
+
+export type PageContentSkipReason = 'no_op';
+
+export interface AgentCommandResponseData extends ActionOutcome {
+  page_content?: TextDomResponseData;
+  page_content_skipped?: PageContentSkipReason;
+}
+
+export interface BatchActionResult {
+  command_id: string;
+  success: boolean;
+  message: string;
+  error?: ActionErrorCode;
+  no_op?: boolean;
+  total_duration_ms: number;
+}
+
+export interface BatchDomStateResult {
+  command_id: string;
+  type: 'dom_state';
+  dom_state: DomStateResponseData;
+  total_duration_ms: number;
+}
+
+export interface BatchTextDomResult {
+  command_id: string;
+  type: 'text_dom';
+  text_dom: TextDomResponseData;
+  total_duration_ms: number;
+}
+
+export type BatchCommandResult = BatchActionResult | BatchDomStateResult | BatchTextDomResult;
+
+export interface BatchResponseData {
+  results: BatchCommandResult[];
+  stopped_at: number | null;
+  stopped_reason?: 'cancelled' | 'check_unreached';
+}
+
+export interface ScreenshotResponseData {
+  success: true;
+  image: string;
+  format: 'jpeg';
+  screen: string;
+  timestamp: number;
+  partial?: true;
+  omitted?: Array<'modal'>;
+}
+
+export interface ConsentScreenshotMetadata {
+  action_type: string;
+  element_description: string;
+  url: string;
+}
+
+export type ConsentScreenshotData =
+  | { success: true; image: string; screenshot_metadata: ConsentScreenshotMetadata; capture_duration_ms: number; sent_at: number }
+  | { success: false; screenshot_error: 'CAPTURE_FAILED'; sent_at: number };
+
+export type OutboundFrame =
+  | { type: 'command_ack'; command_id: string }
+  | { type: 'pong' }
+  | { type: 'sdk_response'; command_id: string; response_type: 'dom_state_response'; data: DomStateResponseData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'text_dom_response'; data: TextDomResponseData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'element_context_response'; data: ElementContextResponseData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'agent_command_response'; data: AgentCommandResponseData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'agent_command_response'; data: ConsentScreenshotData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'agent_command_batch_response'; data: BatchResponseData }
+  | { type: 'sdk_response'; command_id: string; response_type: 'screenshot_response'; data: ScreenshotResponseData }
+  | { type: 'sdk_error'; command_id: string; error: string; code?: string }
+  | { type: 'agent_cancel'; reason: string };
+
+const DEFAULT_SCREENSHOT: ScreenshotOptions = { quality: 0.7, maxWidth: 720 };
+const MAX_SCREENSHOT_WIDTH = 1280;
+
+function parseScreenshotOptions(data: unknown): ScreenshotOptions {
+  const options = isRecord(data) && isRecord(data.options) ? data.options : null;
+  if (options === null) {
+    return DEFAULT_SCREENSHOT;
+  }
+  return {
+    quality: typeof options.quality === 'number' ? options.quality : DEFAULT_SCREENSHOT.quality,
+    maxWidth: typeof options.maxWidth === 'number' ? Math.min(options.maxWidth, MAX_SCREENSHOT_WIDTH) : DEFAULT_SCREENSHOT.maxWidth,
+  };
+}
+
+function parseAgentIds(data: unknown): string[] {
+  const ids = isRecord(data) && Array.isArray(data.agent_ids) ? data.agent_ids : [];
+  return ids.filter((id): id is string => typeof id === 'string');
+}
+
+function parseJson(raw: string): unknown {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+export function parseInboundFrame(raw: string): InboundFrame | null {
+  const parsed = parseJson(raw);
+  if (!isRecord(parsed)) {
+    return null;
+  }
+  if (parsed.type === 'ping') {
+    return { kind: 'ping' };
+  }
+  const name = nonEmptyString(parsed.event);
+  if (name === null) {
+    return null;
+  }
+  const data = isRecord(parsed.data) ? parsed.data : {};
+  const commandId = nonEmptyString(data.batch_id) ?? nonEmptyString(parsed.command_id) ?? nonEmptyString(data.command_id) ?? '';
+  switch (name) {
+    case 'agent_command':
+      return { kind: 'agent_command', commandId, data };
+    case 'agent_command_batch':
+      return { kind: 'agent_command_batch', commandId, data };
+    case 'request_dom_state':
+    case 'dom_state_request':
+      return { kind: 'request_dom_state', commandId };
+    case 'request_text_dom':
+      return { kind: 'request_text_dom', commandId, options: parsePageContentRequest(data.options) };
+    case 'request_screenshot':
+      return { kind: 'request_screenshot', commandId, options: parseScreenshotOptions(data) };
+    case 'request_element_context':
+      return { kind: 'request_element_context', commandId, agentIds: parseAgentIds(data) };
+    default:
+      return { kind: 'unknown', name };
+  }
+}

package/src/agent/handlers/agent-command.ts

@@ -0,0 +1,74 @@
+import { describeUnparsedAction, parseAgentCommand } from '../../actions/parse.ts';
+import { viewOf } from '../../actions/target.ts';
+import type { ActionOutcome, AgentCommandData } from '../../actions/types.ts';
+import type { ConfirmationOutcome } from '../../confirmation/port.ts';
+import type { AgentCommandResponseData } from '../frames.ts';
+import { sendConsentScreenshot } from './consent-screenshot.ts';
+import type { HandlerContext } from './context.ts';
+import { readTextDom } from './text-dom.ts';
+
+const CONFIRMATION_RISKS: readonly string[] = ['irreversible', 'destructive'];
+const UNCONFIRMED_ACTIONS: readonly string[] = ['highlight', 'scroll_to'];
+
+function needsConfirmation(command: AgentCommandData): boolean {
+  return !UNCONFIRMED_ACTIONS.includes(command.action.type) && (command.confirmation_required || CONFIRMATION_RISKS.includes(command.risk_level));
+}
+
+function declined(outcome: Exclude<ConfirmationOutcome, 'confirmed'>): ActionOutcome {
+  return {
+    success: false,
+    no_op: false,
+    error: 'user_cancelled',
+    message: outcome === 'timed_out' ? 'User did not respond to the confirmation prompt' : 'User declined the action',
+  };
+}
+
+async function attachPageContent(
+  context: HandlerContext,
+  command: AgentCommandData,
+  response: AgentCommandResponseData,
+): Promise<AgentCommandResponseData> {
+  if (!command.include_page_content || !response.success) {
+    return response;
+  }
+  if (response.no_op) {
+    return { ...response, page_content_skipped: 'no_op' };
+  }
+  return { ...response, page_content: await readTextDom(context, command.page_content_options) };
+}
+
+export async function runAgentCommand(context: HandlerContext, command: AgentCommandData): Promise<AgentCommandResponseData> {
+  const prepared = await context.executor.prepare(command);
+  if (!prepared.ok) {
+    return prepared.outcome;
+  }
+  if (needsConfirmation(command)) {
+    const outcome = await context.confirmation.confirm({
+      riskLevel: command.risk_level,
+      action: command.action,
+      explanation: command.teaching_explanation,
+      target: prepared.target === null ? null : viewOf(prepared.target),
+    });
+    if (outcome !== 'confirmed') {
+      return declined(outcome);
+    }
+    await sendConsentScreenshot(context, command, prepared.target);
+  }
+  context.stage.beforeAction(command);
+  const outcome = await context.executor.execute(command, prepared.target, { isCancelled: () => context.stage.isCancelled() });
+  return attachPageContent(context, command, outcome);
+}
+
+export async function handleAgentCommand(context: HandlerContext, commandId: string, data: unknown): Promise<void> {
+  const command = parseAgentCommand(data);
+  if (command === null) {
+    context.send({ type: 'sdk_error', command_id: commandId, error: describeUnparsedAction(data), code: 'unsupported_action' });
+    return;
+  }
+  context.send({
+    type: 'sdk_response',
+    command_id: commandId,
+    response_type: 'agent_command_response',
+    data: await runAgentCommand(context, command),
+  });
+}

package/src/agent/handlers/batch.ts

@@ -0,0 +1,72 @@
+import type { AgentCommandData } from '../../actions/types.ts';
+import { SYSTEM_CLOCK } from '../../actions/settle.ts';
+import { parseBatchEntries, type BatchEntry } from '../batch-commands.ts';
+import type { BatchActionResult, BatchCommandResult, BatchResponseData } from '../frames.ts';
+import { runAgentCommand } from './agent-command.ts';
+import type { HandlerContext } from './context.ts';
+import { readDomState } from './dom-state.ts';
+import { readTextDom } from './text-dom.ts';
+
+async function runAction(context: HandlerContext, command: AgentCommandData): Promise<BatchActionResult> {
+  const startedAt = Date.now();
+  const response = await runAgentCommand(context, command);
+  return {
+    command_id: command.command_id,
+    success: response.success,
+    message: response.message ?? (response.success ? `${command.action.type} completed` : 'Action failed'),
+    ...(response.error === undefined ? {} : { error: response.error }),
+    no_op: response.no_op,
+    total_duration_ms: Date.now() - startedAt,
+  };
+}
+
+async function runEntry(context: HandlerContext, entry: BatchEntry): Promise<BatchCommandResult> {
+  const startedAt = Date.now();
+  switch (entry.kind) {
+    case 'action':
+      return runAction(context, entry.command);
+    case 'dom_state':
+      return { command_id: entry.commandId, type: 'dom_state', dom_state: await readDomState(context), total_duration_ms: Date.now() - startedAt };
+    case 'text_dom':
+      return {
+        command_id: entry.commandId,
+        type: 'text_dom',
+        text_dom: await readTextDom(context, entry.options),
+        total_duration_ms: Date.now() - startedAt,
+      };
+    case 'invalid':
+      return { command_id: entry.commandId, success: false, message: entry.reason, error: 'unsupported_action', total_duration_ms: 0 };
+  }
+}
+
+function isFailure(result: BatchCommandResult): result is BatchActionResult {
+  return 'success' in result && !result.success;
+}
+
+export async function runBatch(context: HandlerContext, entries: BatchEntry[]): Promise<BatchResponseData> {
+  const results: BatchCommandResult[] = [];
+  for (const [index, entry] of entries.entries()) {
+    if (context.stage.isCancelled()) {
+      return { results, stopped_at: index, stopped_reason: 'cancelled' };
+    }
+    const result = await runEntry(context, entry);
+    results.push(result);
+    if (isFailure(result)) {
+      const cancelled = result.error === 'user_cancelled';
+      return { results, stopped_at: cancelled ? index + 1 : index, ...(cancelled ? { stopped_reason: 'cancelled' as const } : {}) };
+    }
+    if (entry.kind === 'action' && entry.command.dom_settle_ms > 0) {
+      await SYSTEM_CLOCK.sleep(entry.command.dom_settle_ms);
+    }
+  }
+  return { results, stopped_at: null };
+}
+
+export async function handleBatch(context: HandlerContext, batchId: string, data: unknown): Promise<void> {
+  context.send({
+    type: 'sdk_response',
+    command_id: batchId,
+    response_type: 'agent_command_batch_response',
+    data: await runBatch(context, parseBatchEntries(data)),
+  });
+}

package/src/agent/handlers/consent-screenshot.ts

@@ -0,0 +1,44 @@
+import { nameOf, type ResolvedTarget } from '../../actions/target.ts';
+import type { AgentCommandData } from '../../actions/types.ts';
+import type { ScreenshotOptions } from '../frames.ts';
+import type { HandlerContext } from './context.ts';
+
+const CONSENT_CAPTURE: ScreenshotOptions = { quality: 0.6, maxWidth: 1280 };
+
+export async function sendConsentScreenshot(context: HandlerContext, command: AgentCommandData, target: ResolvedTarget | null): Promise<void> {
+  if (!context.screenshotter.isAvailable() || context.stage.isCancelled()) {
+    return;
+  }
+  const commandId = `${command.command_id}:screenshot`;
+  const startedAt = Date.now();
+  const result = await context.screenshotter.capture(CONSENT_CAPTURE);
+  if (context.stage.isCancelled()) {
+    return;
+  }
+  if (!result.ok) {
+    context.send({
+      type: 'sdk_response',
+      command_id: commandId,
+      response_type: 'agent_command_response',
+      data: { success: false, screenshot_error: 'CAPTURE_FAILED', sent_at: Date.now() },
+    });
+    return;
+  }
+  const { screen } = await context.capturer.capture('peek');
+  context.send({
+    type: 'sdk_response',
+    command_id: commandId,
+    response_type: 'agent_command_response',
+    data: {
+      success: true,
+      image: result.dataUrl,
+      screenshot_metadata: {
+        action_type: command.action.type,
+        element_description: (target === null ? null : nameOf(target)) ?? command.action.type,
+        url: screen,
+      },
+      capture_duration_ms: Date.now() - startedAt,
+      sent_at: Date.now(),
+    },
+  });
+}

package/src/agent/handlers/context.ts

@@ -0,0 +1,17 @@
+import type { ActionExecutor } from '../../actions/executor.ts';
+import type { ConfirmationPort } from '../../confirmation/port.ts';
+import type { Screenshotter } from '../../screenshot/screenshotter.ts';
+import type { SnapshotCapturer } from '../../snapshot/capture.ts';
+import type { ElementContextReader } from '../../snapshot/element-context.ts';
+import type { OutboundFrame } from '../frames.ts';
+import type { AgentStage } from '../stage.ts';
+
+export interface HandlerContext {
+  capturer: SnapshotCapturer;
+  executor: ActionExecutor;
+  screenshotter: Screenshotter;
+  elementContext: ElementContextReader;
+  confirmation: ConfirmationPort;
+  stage: AgentStage;
+  send: (frame: OutboundFrame) => void;
+}

package/src/agent/handlers/dom-state.ts

@@ -0,0 +1,19 @@
+import type { DomStateResponseData } from '../frames.ts';
+import type { HandlerContext } from './context.ts';
+
+export async function readDomState(context: HandlerContext): Promise<DomStateResponseData> {
+  const snapshot = await context.capturer.capture('register');
+  return {
+    schema_version: 3,
+    platform: 'react-native',
+    url: snapshot.screen,
+    title: '',
+    screen: snapshot.screen,
+    routes: snapshot.routes,
+    html_snapshot: snapshot.text,
+  };
+}
+
+export async function handleDomState(context: HandlerContext, commandId: string): Promise<void> {
+  context.send({ type: 'sdk_response', command_id: commandId, response_type: 'dom_state_response', data: await readDomState(context) });
+}

package/src/agent/handlers/element-context.ts

@@ -0,0 +1,9 @@
+import type { HandlerContext } from './context.ts';
+
+export async function handleElementContext(context: HandlerContext, commandId: string, agentIds: string[]): Promise<void> {
+  const elements = [];
+  for (const agentId of agentIds) {
+    elements.push(await context.elementContext.read(agentId));
+  }
+  context.send({ type: 'sdk_response', command_id: commandId, response_type: 'element_context_response', data: { elements } });
+}

package/src/agent/handlers/screenshot.ts

@@ -0,0 +1,24 @@
+import type { ScreenshotOptions } from '../frames.ts';
+import type { HandlerContext } from './context.ts';
+
+export async function handleScreenshot(context: HandlerContext, commandId: string, options: ScreenshotOptions): Promise<void> {
+  const result = await context.screenshotter.capture(options);
+  if (!result.ok) {
+    context.send({ type: 'sdk_error', command_id: commandId, error: result.reason, code: 'CAPTURE_FAILED' });
+    return;
+  }
+  const snapshot = await context.capturer.capture('peek');
+  context.send({
+    type: 'sdk_response',
+    command_id: commandId,
+    response_type: 'screenshot_response',
+    data: {
+      success: true,
+      image: result.dataUrl,
+      format: 'jpeg',
+      screen: snapshot.screen,
+      timestamp: Date.now(),
+      ...(snapshot.modalOpen && !result.capturesModals ? { partial: true as const, omitted: ['modal' as const] } : {}),
+    },
+  });
+}

package/src/agent/handlers/text-dom.ts

@@ -0,0 +1,17 @@
+import type { PageContentRequest } from '../../actions/types.ts';
+import type { TextDomResponseData } from '../frames.ts';
+import type { HandlerContext } from './context.ts';
+
+export async function readTextDom(context: HandlerContext, options: PageContentRequest): Promise<TextDomResponseData> {
+  const tree = await context.capturer.captureText(options);
+  return { url: tree.screen, title: '', screen: tree.screen, text_tree: tree.text, refs: [] };
+}
+
+export async function handleTextDom(context: HandlerContext, commandId: string, options: PageContentRequest): Promise<void> {
+  context.send({
+    type: 'sdk_response',
+    command_id: commandId,
+    response_type: 'text_dom_response',
+    data: await readTextDom(context, options),
+  });
+}

package/src/agent/stage.ts

@@ -0,0 +1,22 @@
+import type { AgentCommandData } from '../actions/types.ts';
+
+export interface Admission {
+  kind: 'command' | 'batch';
+  keepText: string | null;
+}
+
+export interface AgentStage {
+  received: () => void;
+  admit: (admission: Admission) => Promise<boolean>;
+  beforeAction: (command: AgentCommandData) => void;
+  settled: () => void;
+  isCancelled: () => boolean;
+}
+
+export const OPEN_STAGE: AgentStage = {
+  received: () => undefined,
+  admit: () => Promise.resolve(true),
+  beforeAction: () => undefined,
+  settled: () => undefined,
+  isCancelled: () => false,
+};

package/src/agent-mode/action-description.ts

@@ -0,0 +1,25 @@
+import type { AgentAction } from '../actions/types.ts';
+import type { UiStrings } from '../ui/strings.ts';
+
+const TYPED_VALUE_LIMIT = 40;
+
+function fill(template: string, value: string): string {
+  return template.replace('{value}', value);
+}
+
+export function describeAction(action: AgentAction, strings: UiStrings): string {
+  switch (action.type) {
+    case 'click':
+      return strings.actionClick;
+    case 'type':
+      return fill(strings.actionType, action.value.length > TYPED_VALUE_LIMIT ? `${action.value.slice(0, TYPED_VALUE_LIMIT)}...` : action.value);
+    case 'check':
+      return action.checked === true ? strings.actionCheck : strings.actionUncheck;
+    case 'scroll_to':
+      return strings.actionScrollTo;
+    case 'press_key':
+      return fill(strings.actionPressKey, action.key);
+    default:
+      return strings.actionFallback;
+  }
+}

package/src/agent-mode/bubbles.ts

@@ -0,0 +1,31 @@
+export interface AgentBubble {
+  id: string;
+  text: string;
+  leaving: boolean;
+}
+
+const READING_WPM = 120;
+const MIN_READING_MS = 6_000;
+const MAX_READING_MS = 18_000;
+
+export function readingTimeMs(text: string): number {
+  const words = text.trim().split(/\s+/).length;
+  const milliseconds = (words / READING_WPM) * 60 * 1000;
+  return Math.max(MIN_READING_MS, Math.min(MAX_READING_MS, milliseconds));
+}
+
+export function isShowing(bubbles: readonly AgentBubble[], text: string): boolean {
+  return bubbles.some((bubble) => bubble.text === text);
+}
+
+export function keepOnly(bubbles: readonly AgentBubble[], keepText: string | null): AgentBubble[] {
+  return keepText === null ? [] : bubbles.filter((bubble) => bubble.text === keepText);
+}
+
+export function markLeaving(bubbles: readonly AgentBubble[], id: string): AgentBubble[] {
+  return bubbles.map((bubble) => (bubble.id === id ? { ...bubble, leaving: true } : bubble));
+}
+
+export function without(bubbles: readonly AgentBubble[], id: string): AgentBubble[] {
+  return bubbles.filter((bubble) => bubble.id !== id);
+}

package/src/agent-mode/colors.ts

@@ -0,0 +1,40 @@
+const HEX = /^#?([0-9a-f]{3}|[0-9a-f]{6})$/i;
+
+export interface Rgb {
+  r: number;
+  g: number;
+  b: number;
+}
+
+export function parseHex(color: string): Rgb | null {
+  const match = HEX.exec(color.trim());
+  const digits = match?.[1];
+  if (digits === undefined) {
+    return null;
+  }
+  const full = digits.length === 3 ? digits.replace(/./g, (digit) => digit + digit) : digits;
+  return { r: parseInt(full.slice(0, 2), 16), g: parseInt(full.slice(2, 4), 16), b: parseInt(full.slice(4, 6), 16) };
+}
+
+function channel(value: number): number {
+  const linear = value / 255;
+  return linear <= 0.03928 ? linear / 12.92 : ((linear + 0.055) / 1.055) ** 2.4;
+}
+
+export function luminance(color: string): number | null {
+  const rgb = parseHex(color);
+  return rgb === null ? null : 0.2126 * channel(rgb.r) + 0.7152 * channel(rgb.g) + 0.0722 * channel(rgb.b);
+}
+
+export const ON_LIGHT = '#14161a';
+export const ON_DARK = '#ffffff';
+
+/** Text that reads on a brand-coloured surface: near-black on a light brand, white otherwise. */
+export function onColor(color: string): string {
+  return (luminance(color) ?? 0) > 0.45 ? ON_LIGHT : ON_DARK;
+}
+
+export function withAlpha(color: string, alpha: number): string {
+  const rgb = parseHex(color);
+  return rgb === null ? color : `rgba(${rgb.r}, ${rgb.g}, ${rgb.b}, ${alpha})`;
+}