@kernel.chat/kbot 4.3.0 → 4.4.0

package/dist/adapters/peekaboo/commands.d.ts

@@ -0,0 +1,42 @@
+import type { PeekabooAgentResult, PeekabooClickResult, PeekabooOutcome, PeekabooPerformActionResult, PeekabooSeeResult, PeekabooSetValueResult, PeekabooTypeResult } from './types.js';
+export interface SeeOptions {
+    app?: string;
+    mode?: 'screen' | 'window';
+    retina?: boolean;
+}
+export declare function see(opts?: SeeOptions): Promise<PeekabooOutcome<PeekabooSeeResult>>;
+export interface ClickOptions {
+    snapshot: string;
+    on?: string;
+    coords?: [number, number];
+    wait?: number;
+}
+export declare function click(opts: ClickOptions): Promise<PeekabooOutcome<PeekabooClickResult>>;
+export interface TypeOptions {
+    text: string;
+    clear?: boolean;
+    delayMs?: number;
+}
+export declare function type_(opts: TypeOptions): Promise<PeekabooOutcome<PeekabooTypeResult>>;
+export interface SetValueOptions {
+    snapshot: string;
+    on: string;
+    value: string;
+}
+export declare function setValue(opts: SetValueOptions): Promise<PeekabooOutcome<PeekabooSetValueResult>>;
+export interface PerformActionOptions {
+    snapshot: string;
+    on: string;
+    action: string;
+}
+export declare function performAction(opts: PerformActionOptions): Promise<PeekabooOutcome<PeekabooPerformActionResult>>;
+export interface AgentOptions {
+    prompt: string;
+}
+/**
+ * Runs `peekaboo agent "$prompt"` and returns the final stdout. Unlike the
+ * structured commands the agent subcommand may emit free-form text, so we
+ * surface stdout verbatim under `output`.
+ */
+export declare function agent(opts: AgentOptions): Promise<PeekabooOutcome<PeekabooAgentResult>>;
+//# sourceMappingURL=commands.d.ts.map

package/dist/adapters/peekaboo/commands.js

@@ -0,0 +1,209 @@
+// Peekaboo high-level helpers — typed wrappers around the JSON CLI.
+//
+// Each helper assembles argv for `peekaboo <subcommand> --json`, runs it
+// through `runPeekaboo`, and parses stdout into the appropriate result
+// type. Non-zero exits and malformed JSON are returned as `PeekabooError`
+// rather than thrown — callers fan out via discriminated unions.
+import { runPeekaboo } from './runner.js';
+function failNonZero(code, stdout, stderr) {
+    return {
+        ok: false,
+        error: {
+            code: 'non-zero-exit',
+            message: `peekaboo exited ${code}`,
+            stdout,
+            stderr,
+            exitCode: code,
+        },
+    };
+}
+function failParse(message, stdout) {
+    return {
+        ok: false,
+        error: {
+            code: 'malformed-json',
+            message,
+            stdout,
+        },
+    };
+}
+function parseJson(stdout) {
+    try {
+        return { ok: true, value: JSON.parse(stdout) };
+    }
+    catch (e) {
+        return { ok: false, err: failParse(e.message, stdout) };
+    }
+}
+function isRecord(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+function asString(v, fallback = '') {
+    return typeof v === 'string' ? v : fallback;
+}
+function asBool(v, fallback = false) {
+    return typeof v === 'boolean' ? v : fallback;
+}
+function asNumber(v, fallback = 0) {
+    return typeof v === 'number' && Number.isFinite(v) ? v : fallback;
+}
+export async function see(opts = {}) {
+    const args = ['see', '--json'];
+    if (opts.app)
+        args.push('--app', opts.app);
+    if (opts.mode)
+        args.push('--mode', opts.mode);
+    if (opts.retina)
+        args.push('--retina');
+    const { stdout, stderr, code } = await runPeekaboo(args);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    const parsed = parseJson(stdout);
+    if (!parsed.ok)
+        return parsed.err;
+    const v = parsed.value;
+    if (!isRecord(v))
+        return failParse('see: expected object at root', stdout);
+    const rawElements = Array.isArray(v.elements) ? v.elements : [];
+    const elements = rawElements.flatMap((el) => {
+        if (!isRecord(el))
+            return [];
+        const frameRaw = isRecord(el.frame) ? el.frame : {};
+        return [
+            {
+                id: asString(el.id),
+                role: asString(el.role),
+                label: typeof el.label === 'string' ? el.label : undefined,
+                frame: {
+                    x: asNumber(frameRaw.x),
+                    y: asNumber(frameRaw.y),
+                    width: asNumber(frameRaw.width),
+                    height: asNumber(frameRaw.height),
+                },
+                settable: typeof el.settable === 'boolean' ? el.settable : undefined,
+                named_actions: Array.isArray(el.named_actions)
+                    ? el.named_actions.filter((a) => typeof a === 'string')
+                    : undefined,
+            },
+        ];
+    });
+    return {
+        ok: true,
+        snapshot: asString(v.snapshot),
+        app: typeof v.app === 'string' ? v.app : undefined,
+        window: typeof v.window === 'string' ? v.window : undefined,
+        elements,
+        screenshot_path: typeof v.screenshot_path === 'string' ? v.screenshot_path : undefined,
+    };
+}
+export async function click(opts) {
+    const args = ['click', '--json', '--snapshot', opts.snapshot];
+    if (opts.on)
+        args.push('--on', opts.on);
+    if (opts.coords)
+        args.push('--coords', `${opts.coords[0]},${opts.coords[1]}`);
+    if (typeof opts.wait === 'number')
+        args.push('--wait', String(opts.wait));
+    const { stdout, stderr, code } = await runPeekaboo(args);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    const parsed = parseJson(stdout);
+    if (!parsed.ok)
+        return parsed.err;
+    if (!isRecord(parsed.value))
+        return failParse('click: expected object at root', stdout);
+    const v = parsed.value;
+    return {
+        ok: true,
+        target: typeof v.target === 'string' ? v.target : undefined,
+        coords: Array.isArray(v.coords) && v.coords.length === 2
+            ? [asNumber(v.coords[0]), asNumber(v.coords[1])]
+            : undefined,
+    };
+}
+// `type` is reserved in TS; export the helper as `type_`.
+export async function type_(opts) {
+    const args = ['type', '--json', '--text', opts.text];
+    if (opts.clear)
+        args.push('--clear');
+    if (typeof opts.delayMs === 'number')
+        args.push('--delay', String(opts.delayMs));
+    const { stdout, stderr, code } = await runPeekaboo(args);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    const parsed = parseJson(stdout);
+    if (!parsed.ok)
+        return parsed.err;
+    if (!isRecord(parsed.value))
+        return failParse('type: expected object at root', stdout);
+    const v = parsed.value;
+    return {
+        ok: true,
+        typed: asString(v.typed, opts.text),
+        cleared: typeof v.cleared === 'boolean' ? v.cleared : undefined,
+    };
+}
+export async function setValue(opts) {
+    const args = [
+        'set-value',
+        '--json',
+        '--snapshot',
+        opts.snapshot,
+        '--on',
+        opts.on,
+        '--value',
+        opts.value,
+    ];
+    const { stdout, stderr, code } = await runPeekaboo(args);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    const parsed = parseJson(stdout);
+    if (!parsed.ok)
+        return parsed.err;
+    if (!isRecord(parsed.value))
+        return failParse('set-value: expected object at root', stdout);
+    const v = parsed.value;
+    return {
+        ok: true,
+        target: asString(v.target, opts.on),
+        value: asString(v.value, opts.value),
+    };
+}
+export async function performAction(opts) {
+    const args = [
+        'perform-action',
+        '--json',
+        '--snapshot',
+        opts.snapshot,
+        '--on',
+        opts.on,
+        '--action',
+        opts.action,
+    ];
+    const { stdout, stderr, code } = await runPeekaboo(args);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    const parsed = parseJson(stdout);
+    if (!parsed.ok)
+        return parsed.err;
+    if (!isRecord(parsed.value))
+        return failParse('perform-action: expected object at root', stdout);
+    const v = parsed.value;
+    return {
+        ok: true,
+        target: asString(v.target, opts.on),
+        action: asString(v.action, opts.action),
+    };
+}
+/**
+ * Runs `peekaboo agent "$prompt"` and returns the final stdout. Unlike the
+ * structured commands the agent subcommand may emit free-form text, so we
+ * surface stdout verbatim under `output`.
+ */
+export async function agent(opts) {
+    const { stdout, stderr, code } = await runPeekaboo(['agent', opts.prompt]);
+    if (code !== 0)
+        return failNonZero(code, stdout, stderr);
+    return { ok: true, output: stdout };
+}
+//# sourceMappingURL=commands.js.map

package/dist/adapters/peekaboo/index.d.ts

@@ -0,0 +1,4 @@
+export type { PeekabooFrame, PeekabooElement, PeekabooSeeResult, PeekabooClickResult, PeekabooTypeResult, PeekabooSetValueResult, PeekabooPerformActionResult, PeekabooAgentResult, PeekabooError, PeekabooOutcome, } from './types.js';
+export { runPeekaboo, peekabooAvailable, type RunOptions, type RunResult } from './runner.js';
+export { see, click, type_, setValue, performAction, agent, type SeeOptions, type ClickOptions, type TypeOptions, type SetValueOptions, type PerformActionOptions, type AgentOptions, } from './commands.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/adapters/peekaboo/index.js

@@ -0,0 +1,9 @@
+// Peekaboo adapter — public surface.
+//
+// Wraps the `peekaboo` macOS CLI behind a typed interface so kbot tools and
+// agents can drive the screen-capture + GUI-automation features without
+// taking a runtime dependency on the binary or the @steipete/peekaboo
+// distribution. All execution flows through `runPeekaboo`.
+export { runPeekaboo, peekabooAvailable } from './runner.js';
+export { see, click, type_, setValue, performAction, agent, } from './commands.js';
+//# sourceMappingURL=index.js.map

package/dist/adapters/peekaboo/runner.d.ts

@@ -0,0 +1,25 @@
+export interface RunOptions {
+    /** Optional stdin payload. */
+    input?: string;
+    /** Working directory for the child. */
+    cwd?: string;
+    /** Hard timeout in milliseconds. */
+    timeoutMs?: number;
+}
+export interface RunResult {
+    stdout: string;
+    stderr: string;
+    code: number;
+}
+/**
+ * Run the peekaboo CLI with the given args. Resolves with stdout/stderr/code
+ * regardless of exit status; never rejects on non-zero exit. Rejects only
+ * when the binary cannot be spawned at all (ENOENT, permissions, etc.).
+ */
+export declare function runPeekaboo(args: string[], opts?: RunOptions): Promise<RunResult>;
+/**
+ * Returns true when the peekaboo binary responds to `--version`. Used by
+ * higher-level tools to gate Peekaboo features without crashing the host.
+ */
+export declare function peekabooAvailable(): Promise<boolean>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/adapters/peekaboo/runner.js

@@ -0,0 +1,75 @@
+// Peekaboo binary runner — thin child_process wrapper.
+//
+// All execution of the `peekaboo` CLI flows through `runPeekaboo`. The
+// binary path is resolved from the PEEKABOO_BIN env var (default: 'peekaboo')
+// so tests can stub it and hosts can pin a specific install. No shell is
+// involved — args go through execFile to keep arg quoting predictable.
+import { execFile } from 'node:child_process';
+function resolveBinary() {
+    return process.env.PEEKABOO_BIN && process.env.PEEKABOO_BIN.trim().length > 0
+        ? process.env.PEEKABOO_BIN
+        : 'peekaboo';
+}
+/**
+ * Run the peekaboo CLI with the given args. Resolves with stdout/stderr/code
+ * regardless of exit status; never rejects on non-zero exit. Rejects only
+ * when the binary cannot be spawned at all (ENOENT, permissions, etc.).
+ */
+export function runPeekaboo(args, opts = {}) {
+    const bin = resolveBinary();
+    return new Promise((resolve, reject) => {
+        const child = execFile(bin, args, {
+            cwd: opts.cwd,
+            timeout: opts.timeoutMs,
+            maxBuffer: 64 * 1024 * 1024,
+        }, (err, stdout, stderr) => {
+            // execFile populates err for non-zero exit too. We surface the result
+            // either way and only reject when the spawn itself failed.
+            const out = stdout;
+            const errOut = stderr;
+            const stdoutStr = typeof out === 'string'
+                ? out
+                : out instanceof Buffer
+                    ? out.toString('utf8')
+                    : '';
+            const stderrStr = typeof errOut === 'string'
+                ? errOut
+                : errOut instanceof Buffer
+                    ? errOut.toString('utf8')
+                    : '';
+            if (err) {
+                const errno = err.code;
+                // Spawn-level failures (binary missing, EACCES, etc.) reject.
+                if (errno === 'ENOENT' || errno === 'EACCES' || errno === 'EPERM') {
+                    reject(err);
+                    return;
+                }
+                // Otherwise treat as "process ran, exited non-zero".
+                const code = typeof err.code === 'number'
+                    ? (err.code)
+                    : 1;
+                resolve({ stdout: stdoutStr, stderr: stderrStr, code });
+                return;
+            }
+            resolve({ stdout: stdoutStr, stderr: stderrStr, code: 0 });
+        });
+        if (opts.input !== undefined && child.stdin) {
+            child.stdin.write(opts.input);
+            child.stdin.end();
+        }
+    });
+}
+/**
+ * Returns true when the peekaboo binary responds to `--version`. Used by
+ * higher-level tools to gate Peekaboo features without crashing the host.
+ */
+export async function peekabooAvailable() {
+    try {
+        const r = await runPeekaboo(['--version'], { timeoutMs: 5000 });
+        return r.code === 0;
+    }
+    catch {
+        return false;
+    }
+}
+//# sourceMappingURL=runner.js.map

package/dist/adapters/peekaboo/types.d.ts

@@ -0,0 +1,69 @@
+export interface PeekabooFrame {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+}
+export interface PeekabooElement {
+    /** Element handle, e.g. "B1" (button), "T1" (text field). */
+    id: string;
+    role: string;
+    label?: string;
+    frame: PeekabooFrame;
+    /** Whether the element accepts a value (text fields, sliders, etc.). */
+    settable?: boolean;
+    /** Action names the element advertises via the AX API. */
+    named_actions?: string[];
+}
+export interface PeekabooSeeResult {
+    /** Snapshot id used by subsequent `--snapshot $id` arguments. */
+    snapshot: string;
+    app?: string;
+    window?: string;
+    elements: PeekabooElement[];
+    /** Optional path on disk where the screenshot was written. */
+    screenshot_path?: string;
+}
+export interface PeekabooClickResult {
+    ok: boolean;
+    target?: string;
+    coords?: [number, number];
+}
+export interface PeekabooTypeResult {
+    ok: boolean;
+    typed: string;
+    cleared?: boolean;
+}
+export interface PeekabooSetValueResult {
+    ok: boolean;
+    target: string;
+    value: string;
+}
+export interface PeekabooPerformActionResult {
+    ok: boolean;
+    target: string;
+    action: string;
+}
+export interface PeekabooAgentResult {
+    ok: boolean;
+    output: string;
+}
+/**
+ * Structured error returned by all command helpers when the binary exits
+ * non-zero or emits malformed JSON. Helpers return `{ ok: false, error }`
+ * rather than throw so callers can route via discriminated unions.
+ */
+export interface PeekabooError {
+    ok: false;
+    error: {
+        code: 'non-zero-exit' | 'malformed-json' | 'binary-missing' | 'unknown';
+        message: string;
+        stderr?: string;
+        stdout?: string;
+        exitCode?: number;
+    };
+}
+export type PeekabooOutcome<T> = ({
+    ok: true;
+} & T) | PeekabooError;
+//# sourceMappingURL=types.d.ts.map

package/dist/adapters/peekaboo/types.js

@@ -0,0 +1,8 @@
+// Peekaboo CLI surface — minimal type model.
+//
+// Mirrors the JSON shape emitted by the `peekaboo` macOS CLI
+// (https://github.com/openclaw/Peekaboo) without taking a runtime
+// dependency. kbot stays binary-agnostic; this adapter only ever
+// speaks JSON across the process boundary.
+export {};
+//# sourceMappingURL=types.js.map

package/dist/tools/computer.js

@@ -17,7 +17,19 @@ import { join } from 'node:path';
 import { readFileSync, unlinkSync, existsSync, mkdirSync, rmSync } from 'node:fs';
 import { registerTool } from './index.js';
 import { Coordinator } from '../computer-use-coordinator.js';
+import { peekabooAvailable, see as peekabooSee, click as peekabooClick, type_ as peekabooType, } from '../adapters/peekaboo/index.js';
 const platform = process.platform;
+// ── Peekaboo AX-first fallback ───────────────────────────────────
+// When the peekaboo CLI is installed, try AX-aware automation before
+// synthetic input. Cached at module-load, refreshed if env hint changes.
+let _peekabooReady = null;
+function peekabooReady() {
+    if (process.env.KBOT_DISABLE_PEEKABOO === '1')
+        return Promise.resolve(false);
+    if (!_peekabooReady)
+        _peekabooReady = peekabooAvailable().catch(() => false);
+    return _peekabooReady;
+}
 const LOCK_DIR = join(homedir(), '.kbot');
 // Legacy single-session lock path. Retained as a constant for back-compat
 // with any callers that referenced it; the actual locking is now performed
@@ -485,6 +497,27 @@ export function registerComputerTools() {
                 return 'Error: x and y must be numbers';
             }
             try {
+                // AX-first fallback: when peekaboo is on PATH, try AX-aware click
+                // before synthetic input. Falls through to AppleScript+cliclick on
+                // any failure so existing behavior is preserved.
+                if (platform === 'darwin' && app && await peekabooReady()) {
+                    try {
+                        const snap = await peekabooSee({ app });
+                        if (snap.ok) {
+                            const target = (args.element_id ?? args.label ?? null);
+                            if (target) {
+                                const result = await peekabooClick({
+                                    snapshot: snap.snapshot,
+                                    on: String(target),
+                                });
+                                if (result.ok)
+                                    return `clicked via AX: ${target}`;
+                                // fall through to synthetic on AX failure
+                            }
+                        }
+                    }
+                    catch { /* fall through */ }
+                }
                 if (platform === 'darwin') {
                     try {
                         if (button === 'double') {
@@ -712,6 +745,21 @@ export function registerComputerTools() {
                 const text = String(args.text);
                 if (!text)
                     return 'Error: text is required';
+                // AX-first fallback: when peekaboo is on PATH, try AX-aware typing
+                // before synthetic keystrokes. Falls through to AppleScript on any
+                // failure so existing behavior is preserved.
+                if (platform === 'darwin' && await peekabooReady()) {
+                    try {
+                        const clear = args.clear === true;
+                        const delayMs = typeof args.delayMs === 'number' ? args.delayMs : undefined;
+                        const result = await peekabooType({ text, clear, delayMs });
+                        if (result.ok) {
+                            return `Typed via AX: ${text.slice(0, 80)}${text.length > 80 ? '...' : ''}`;
+                        }
+                        // fall through to synthetic on AX failure
+                    }
+                    catch { /* fall through */ }
+                }
                 if (platform === 'darwin') {
                     const escaped = escapeAppleScript(text);
                     try {
@@ -1104,5 +1152,21 @@ export function registerComputerTools() {
             return `Computer use session ended.${releasedNote}`;
         },
     });
+    // ── Peekaboo status (AX-first diagnostics) ──
+    registerTool({
+        name: 'peekaboo_status',
+        description: 'Report whether the AX-first peekaboo CLI is available on PATH for accessibility-aware automation.',
+        parameters: {},
+        tier: 'free',
+        async execute() {
+            if (process.env.KBOT_DISABLE_PEEKABOO === '1') {
+                return 'AX-first unavailable (disabled via KBOT_DISABLE_PEEKABOO=1)';
+            }
+            const ready = await peekabooReady();
+            return ready
+                ? 'AX-first available via peekaboo'
+                : 'AX-first unavailable (peekaboo CLI not on PATH)';
+        },
+    });
 }
 //# sourceMappingURL=computer.js.map

package/dist/tools/peekaboo.d.ts

@@ -0,0 +1,4 @@
+import { type ToolDefinition } from './index.js';
+export declare const peekabooTools: readonly ToolDefinition[];
+export declare function registerPeekabooTools(): void;
+//# sourceMappingURL=peekaboo.d.ts.map

package/dist/tools/peekaboo.js

@@ -0,0 +1,371 @@
+// Peekaboo tools — kbot tool-registry surface for the macOS Peekaboo CLI.
+//
+// Wraps src/adapters/peekaboo so the LLM tool layer can drive AX-aware
+// snapshots, clicks, typing, value-set, named-action invocation, and the
+// peekaboo agent subcommand. App-bound calls are gated by the same per-app
+// approval contract computer.ts uses; because computer.ts does not export
+// its in-process `approvedApps` set, this module falls back to checking
+// the on-disk Coordinator lock file at ~/.kbot/computer-use/<app>.lock as
+// a best-effort cross-process signal — see requireApproval() below.
+import { existsSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { registerTool } from './index.js';
+import { see, click, type_, setValue, performAction, agent, peekabooAvailable, } from '../adapters/peekaboo/index.js';
+// ── Approval gate ──────────────────────────────────────────────────────
+//
+// computer.ts holds an in-process `approvedApps` Set keyed by lowercase app
+// name, plus a Coordinator that writes a per-app lock file at
+// ~/.kbot/computer-use/<app>.lock when an *active* claim is held.
+//
+// Limitation: the `approvedApps` set is not exported and the Coordinator
+// lock file only exists during an active claim, not after a bare
+// app_approve. So from a sibling module like this one we cannot directly
+// observe approval state. We treat the lock-file presence as the strongest
+// available cross-process approval signal — if the user has driven the app
+// through computer.ts at all this session, the lock file will exist
+// (computer.ts re-acquires + retains during the call). When we can't see
+// it, we fail closed with a clear pointer to `app_approve`.
+const LOCK_ROOT = process.env.KBOT_COMPUTER_USE_ROOT || join(homedir(), '.kbot', 'computer-use');
+/** Mirror of computer-use-coordinator.ts#sanitizeApp — keep in sync. */
+function sanitizeApp(app) {
+    return app.replace(/[/\\ -]/g, '_');
+}
+function lockPath(app) {
+    return join(LOCK_ROOT, `${sanitizeApp(app.toLowerCase())}.lock`);
+}
+/** Returns null when the gate passes, or an `Error: ...` string on denial. */
+function requireApproval(app) {
+    if (!app)
+        return null;
+    if (existsSync(lockPath(app)))
+        return null;
+    return `Error: ${app} is not approved for computer use. Run app_approve first (or drive a computer.ts tool against ${app} so the Coordinator lock is created).`;
+}
+// ── Outcome → string helpers ───────────────────────────────────────────
+function outcomeToString(out) {
+    if (!out.ok) {
+        const err = out.error;
+        const detail = err.stderr?.trim() || err.message;
+        return `Error: peekaboo ${err.code}: ${detail}`;
+    }
+    // Strip the discriminant before pretty-printing the success payload.
+    const { ok: _ok, ...payload } = out;
+    void _ok;
+    return JSON.stringify(payload, null, 2);
+}
+async function ensureBinary() {
+    const ok = await peekabooAvailable();
+    if (ok)
+        return null;
+    return "Error: peekaboo CLI not found on PATH. Install via 'brew install steipete/tap/peekaboo' or 'npm i -g @steipete/peekaboo'.";
+}
+// ── Tool definitions ───────────────────────────────────────────────────
+const peekabooSee = {
+    name: 'peekaboo_see',
+    description: 'Capture an AX snapshot of an app or the screen via the Peekaboo CLI. Returns a snapshot id plus a list of labeled element ids (B1, T1, …) usable by peekaboo_click / peekaboo_set_value / peekaboo_perform_action.',
+    parameters: {
+        app: {
+            type: 'string',
+            description: 'Target app name (optional). When supplied, the app must be approved via app_approve.',
+        },
+        mode: {
+            type: 'string',
+            description: 'Capture mode: "screen" (default) or "window".',
+        },
+        retina: {
+            type: 'boolean',
+            description: 'Capture at retina resolution (optional).',
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const app = typeof args.app === 'string' && args.app.length > 0 ? args.app : undefined;
+        if (app) {
+            const gate = requireApproval(app);
+            if (gate)
+                return gate;
+        }
+        const mode = args.mode === 'window' || args.mode === 'screen' ? args.mode : undefined;
+        const retina = args.retina === true;
+        try {
+            const out = await see({
+                ...(app ? { app } : {}),
+                ...(mode ? { mode } : {}),
+                ...(retina ? { retina: true } : {}),
+            });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+const peekabooClick = {
+    name: 'peekaboo_click',
+    description: 'Click against a Peekaboo snapshot. Provide either `on` (element id or query) or `coords` ("x,y"). Requires a prior peekaboo_see.',
+    parameters: {
+        app: {
+            type: 'string',
+            description: 'App being targeted. Required for the approval gate.',
+            required: true,
+        },
+        snapshot: {
+            type: 'string',
+            description: 'Snapshot id from peekaboo_see.',
+            required: true,
+        },
+        on: {
+            type: 'string',
+            description: 'Element id (e.g. "B1") or query string. Mutually exclusive with coords.',
+        },
+        coords: {
+            type: 'string',
+            description: 'Click coordinates as "x,y" (numbers). Mutually exclusive with on.',
+        },
+        wait: {
+            type: 'number',
+            description: 'Optional pre-click wait in milliseconds.',
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const app = String(args.app ?? '');
+        if (!app)
+            return 'Error: app is required.';
+        const gate = requireApproval(app);
+        if (gate)
+            return gate;
+        const snapshot = String(args.snapshot ?? '');
+        if (!snapshot)
+            return 'Error: snapshot is required.';
+        const on = typeof args.on === 'string' && args.on.length > 0 ? args.on : undefined;
+        let coords;
+        if (typeof args.coords === 'string' && args.coords.length > 0) {
+            const parts = args.coords.split(',').map((p) => Number(p.trim()));
+            if (parts.length !== 2 || parts.some((n) => !Number.isFinite(n))) {
+                return 'Error: coords must be "x,y" (numbers).';
+            }
+            coords = [parts[0], parts[1]];
+        }
+        if (!on && !coords)
+            return 'Error: provide either `on` or `coords`.';
+        const wait = typeof args.wait === 'number' && Number.isFinite(args.wait) ? args.wait : undefined;
+        try {
+            const out = await click({
+                snapshot,
+                ...(on ? { on } : {}),
+                ...(coords ? { coords } : {}),
+                ...(wait !== undefined ? { wait } : {}),
+            });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+const peekabooType = {
+    name: 'peekaboo_type',
+    description: 'Type text into the focused field via the Peekaboo CLI. Use peekaboo_click to focus first.',
+    parameters: {
+        app: {
+            type: 'string',
+            description: 'App being targeted. Required for the approval gate.',
+            required: true,
+        },
+        text: {
+            type: 'string',
+            description: 'Text to type.',
+            required: true,
+        },
+        clear: {
+            type: 'boolean',
+            description: 'Clear the field before typing (optional).',
+        },
+        delay_ms: {
+            type: 'number',
+            description: 'Per-character delay in milliseconds (optional).',
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const app = String(args.app ?? '');
+        if (!app)
+            return 'Error: app is required.';
+        const gate = requireApproval(app);
+        if (gate)
+            return gate;
+        const text = typeof args.text === 'string' ? args.text : '';
+        if (!text)
+            return 'Error: text is required.';
+        const clear = args.clear === true;
+        const delayMs = typeof args.delay_ms === 'number' && Number.isFinite(args.delay_ms) ? args.delay_ms : undefined;
+        try {
+            const out = await type_({
+                text,
+                ...(clear ? { clear: true } : {}),
+                ...(delayMs !== undefined ? { delayMs } : {}),
+            });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+const peekabooSetValue = {
+    name: 'peekaboo_set_value',
+    description: 'Set a settable AX value directly on an element (skips clicking). Faster than click+type for text fields, sliders, etc.',
+    parameters: {
+        app: {
+            type: 'string',
+            description: 'App being targeted. Required for the approval gate.',
+            required: true,
+        },
+        snapshot: {
+            type: 'string',
+            description: 'Snapshot id from peekaboo_see.',
+            required: true,
+        },
+        on: {
+            type: 'string',
+            description: 'Target element id or query (must be settable).',
+            required: true,
+        },
+        value: {
+            type: 'string',
+            description: 'Value to assign.',
+            required: true,
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const app = String(args.app ?? '');
+        if (!app)
+            return 'Error: app is required.';
+        const gate = requireApproval(app);
+        if (gate)
+            return gate;
+        const snapshot = String(args.snapshot ?? '');
+        const on = String(args.on ?? '');
+        const value = typeof args.value === 'string' ? args.value : '';
+        if (!snapshot)
+            return 'Error: snapshot is required.';
+        if (!on)
+            return 'Error: on is required.';
+        try {
+            const out = await setValue({ snapshot, on, value });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+const peekabooPerformAction = {
+    name: 'peekaboo_perform_action',
+    description: 'Invoke a named AX action (e.g. AXPress, AXShowMenu) on an element from a Peekaboo snapshot.',
+    parameters: {
+        app: {
+            type: 'string',
+            description: 'App being targeted. Required for the approval gate.',
+            required: true,
+        },
+        snapshot: {
+            type: 'string',
+            description: 'Snapshot id from peekaboo_see.',
+            required: true,
+        },
+        on: {
+            type: 'string',
+            description: 'Target element id or query.',
+            required: true,
+        },
+        action: {
+            type: 'string',
+            description: 'Named AX action (e.g. "AXPress", "AXShowMenu").',
+            required: true,
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const app = String(args.app ?? '');
+        if (!app)
+            return 'Error: app is required.';
+        const gate = requireApproval(app);
+        if (gate)
+            return gate;
+        const snapshot = String(args.snapshot ?? '');
+        const on = String(args.on ?? '');
+        const action = String(args.action ?? '');
+        if (!snapshot)
+            return 'Error: snapshot is required.';
+        if (!on)
+            return 'Error: on is required.';
+        if (!action)
+            return 'Error: action is required.';
+        try {
+            const out = await performAction({ snapshot, on, action });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+const peekabooAgent = {
+    name: 'peekaboo_agent',
+    description: "Hand a natural-language automation prompt to peekaboo's own agent subcommand. Returns the agent's stdout verbatim. Not gated by app approval — peekaboo's agent decides which apps to drive.",
+    parameters: {
+        prompt: {
+            type: 'string',
+            description: 'Free-form instruction for the peekaboo agent.',
+            required: true,
+        },
+    },
+    tier: 'free',
+    async execute(args) {
+        const binErr = await ensureBinary();
+        if (binErr)
+            return binErr;
+        const prompt = typeof args.prompt === 'string' ? args.prompt : '';
+        if (!prompt)
+            return 'Error: prompt is required.';
+        try {
+            const out = await agent({ prompt });
+            return outcomeToString(out);
+        }
+        catch (e) {
+            return `Error: ${e.message}`;
+        }
+    },
+};
+export const peekabooTools = [
+    peekabooSee,
+    peekabooClick,
+    peekabooType,
+    peekabooSetValue,
+    peekabooPerformAction,
+    peekabooAgent,
+];
+export function registerPeekabooTools() {
+    for (const t of peekabooTools)
+        registerTool(t);
+}
+//# sourceMappingURL=peekaboo.js.map

package/dist/tools/swarm-2026-04.js

@@ -12,6 +12,7 @@ import { SECURITY_AGENT_TOOLS } from './security-agent-tools.js';
 import { anthropicManagedAgentTools } from './anthropic-managed-agents-tools.js';
 import { forecastSummaryTool } from './forecast-summary.js';
 import { securityAuditLocalTool } from './security-audit-local.js';
+import { registerPeekabooTools } from './peekaboo.js';
 function adaptCoordinatorTool(t) {
     const props = t.inputSchema.properties ?? {};
     const required = new Set(t.inputSchema.required ?? []);
@@ -84,6 +85,7 @@ export function registerSwarm2026Tools() {
     registerTool(fileLibraryGetTool);
     registerTool(forecastSummaryTool);
     registerTool(securityAuditLocalTool);
+    registerPeekabooTools();
     for (const t of workspaceAgentTools)
         registerTool(t);
     for (const t of anthropicManagedAgentTools)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kernel.chat/kbot",
-  "version": "4.3.0",
+  "version": "4.4.0",
   "description": "Open-source terminal AI agent. 100+ specialist skills, 35 specialist agents, 20 providers. Dreams, learns, watches your system. Controls your phone. Fully local, fully sovereign. MIT. v4.0 — evidence-based curation.",
   "type": "module",
   "repository": {

package/skills/native-automation/peekaboo-snapshot-act/SKILL.md

@@ -0,0 +1,91 @@
+---
+name: peekaboo-snapshot-act
+description: Use when an agent needs to drive a macOS native app reliably — snapshot-then-act via Peekaboo's accessibility-aware element IDs beats screenshot-per-click for any app with proper AX support.
+version: 1.0.0
+author: kbot
+license: MIT
+metadata:
+  kbot:
+    tags: [computer-use, macos, accessibility, native-apps, automation]
+    related_skills: [computer-use-coordination, ableton-osc-control]
+---
+
+# Peekaboo Snapshot-Act
+
+Every input surface has a right shape. Web is DOM — address elements by ref through Chrome MCP or Playwright. Audio is OSC — address tracks and clips through AbletonOSC's Live Object Model. Native macOS is AX — address controls through the Accessibility tree. Snapshot-and-act replaces screenshot-and-guess: capture the AX tree once, reference labeled element IDs many times, and let the OS resolve coordinates.
+
+## When to Use
+
+- Native macOS app automation where the target exposes Accessibility (Notes, Mail, System Settings, Music, Finder, Xcode, Numbers, Keynote, Pages).
+- Forms with multiple fields where one snapshot fills them all.
+- Repeated interactions inside the same app state where stable element IDs survive across calls.
+- Any task where pixel coordinates would drift between window resizes, theme changes, or display scales.
+
+Do **not** use this skill for:
+
+- Browser tabs — use Chrome MCP (`mcp__claude-in-chrome__*`); the DOM is the right surface.
+- Ableton Live or other audio software with OSC bridges — use `ableton_*` tools; OSC is the right surface.
+- Apps with no Accessibility support — fall through to synthetic input via `mcp__computer-use__*` and accept the cost.
+
+## Iron Laws
+
+```
+ONE SNAPSHOT, MANY ACTIONS.
+ELEMENT ID OVER COORDINATES.
+PERFORM-ACTION OVER CLICK.
+```
+
+A snapshot is a contract: while the UI does not change, the IDs are stable. Capture once, act many times, re-snapshot only on visible state change. An element ID survives where coordinates do not — themes shift, windows resize, scroll positions move; `B7` does not. And when an AX action is named (`AXPress`, `AXShowMenu`, `AXIncrement`), perform it directly; clicking the rendered pixel is a worse approximation of the user's intent.
+
+## Five Phases
+
+### Phase 1 — Approve & focus
+
+Bring the target app forward and clear the per-app session lock before any Peekaboo call.
+
+- `app_approve` — gate the sensitive-app warning; respect the user's per-app trust state.
+- `app_launch` — bring the app to the front so AX queries hit the right process.
+- One-time at the OS level: grant Peekaboo **Screen Recording** and **Accessibility** in System Settings → Privacy & Security. Without both, `see` returns an empty tree.
+
+### Phase 2 — Capture surface
+
+Pull the AX snapshot once.
+
+```
+peekaboo see --app <Name> --json
+```
+
+The response contains a snapshot ID and labeled element IDs by role: `B1` for the first button, `T1` for the first text field, `L1` for a link, `M1` for a menu. Read the labels, not the pixels. The snapshot ID is the handle every subsequent call references.
+
+### Phase 3 — Choose the right verb
+
+Three verbs cover almost every native interaction. Pick the most specific one that fits.
+
+- `set-value` — for any settable field (text inputs, sliders, steppers). Faster and more reliable than `click + type`. Sets the AX value directly.
+- `perform-action` — for any named AX action (`AXPress`, `AXShowMenu`, `AXConfirm`, `AXIncrement`, `AXDecrement`). Names the intent the OS already understands.
+- `click` — only when neither of the above applies (custom non-AX views, web content embedded in a native shell).
+
+### Phase 4 — Reuse the snapshot
+
+Successive actions reference the same `--snapshot $ID` until the UI changes. Filling a five-field form is one snapshot and five `set-value` calls, not five snapshots and five clicks. Re-snapshot only when the visible state actually changes — a panel opens, a sheet appears, a navigation transitions. Re-snapshotting before every action defeats the entire pattern and is slower than synthetic input.
+
+### Phase 5 — Fall back gracefully
+
+If the AX path fails — element ID stale, app exposes no AX tree, action returns an error — fall through to synthetic input via `kbot_click` or `mcp__computer-use__*` and log the fallback. Record which app and which action degraded so the next session knows. Graceful degradation beats a hard failure; opaque retry loops do not.
+
+## Anti-Patterns
+
+- Re-snapshotting before every click. The whole point is reuse — one snapshot, many actions.
+- Using coordinates when an element ID exists. IDs survive resize, theme, and scale changes; coordinates do not.
+- Ignoring `set-value` and falling through to `click + type` for text fields. Slower, less reliable, and breaks on focus drift.
+- Driving Chrome with Peekaboo. Chrome MCP exists for a reason; the DOM is the right surface for the web.
+- Skipping `app_approve`. The per-app session lock and sensitive-app warnings still apply — Peekaboo does not bypass kbot's trust model.
+
+## How kbot Helps
+
+- `peekaboo_see` — wraps `peekaboo see --json` with kbot's lock + approval flow; returns snapshot ID and element IDs.
+- `peekaboo_click` / `peekaboo_type` / `peekaboo_set_value` / `peekaboo_perform_action` — the four verbs, each lock-aware and approval-gated.
+- `peekaboo_agent` — composite tool for multi-step flows that snapshot once and act many times under one approval.
+- `kbot_click` — falls through to the AX-first path automatically when the `peekaboo` binary is on `PATH`; no caller change required.
+- `app_approve` — gates per-app sensitive-app warnings before any Peekaboo call lands.
+- Coordinator — the same per-app sub-locks apply across Peekaboo and the existing `computer.ts` synthetic-input path; both share one lock file, so AX and pixel routes never race against each other.