@kernel.chat/kbot 4.4.0 → 4.4.1

package/dist/adapters/peekaboo/commands.d.ts

@@ -23,13 +23,13 @@ export interface SetValueOptions {
     on: string;
     value: string;
 }
-export declare function setValue(opts: SetValueOptions): Promise<PeekabooOutcome<PeekabooSetValueResult>>;
+export declare function setValue(_opts: SetValueOptions): Promise<PeekabooOutcome<PeekabooSetValueResult>>;
 export interface PerformActionOptions {
     snapshot: string;
     on: string;
     action: string;
 }
-export declare function performAction(opts: PerformActionOptions): Promise<PeekabooOutcome<PeekabooPerformActionResult>>;
+export declare function performAction(_opts: PerformActionOptions): Promise<PeekabooOutcome<PeekabooPerformActionResult>>;
 export interface AgentOptions {
     prompt: string;
 }

package/dist/adapters/peekaboo/commands.js

@@ -4,6 +4,10 @@
 // through `runPeekaboo`, and parses stdout into the appropriate result
 // type. Non-zero exits and malformed JSON are returned as `PeekabooError`
 // rather than thrown — callers fan out via discriminated unions.
+//
+// Every peekaboo 3.0.0-beta4 JSON command wraps its payload in
+// `{ success, data, error? }`. Helpers unwrap that envelope and treat
+// `success: false` as a structured failure rather than a parse error.
 import { runPeekaboo } from './runner.js';
 function failNonZero(code, stdout, stderr) {
     return {
@@ -41,12 +45,42 @@ function isRecord(v) {
 function asString(v, fallback = '') {
     return typeof v === 'string' ? v : fallback;
 }
-function asBool(v, fallback = false) {
-    return typeof v === 'boolean' ? v : fallback;
+function asOptString(v) {
+    return typeof v === 'string' ? v : undefined;
+}
+function asOptBool(v) {
+    return typeof v === 'boolean' ? v : undefined;
+}
+function asOptNumber(v) {
+    return typeof v === 'number' && Number.isFinite(v) ? v : undefined;
 }
 function asNumber(v, fallback = 0) {
     return typeof v === 'number' && Number.isFinite(v) ? v : fallback;
 }
+/**
+ * Unwrap the `{ success, data, error? }` envelope every peekaboo 3.0.0-beta4
+ * JSON command emits. On `success: true` returns the inner `data` record; on
+ * `success: false` (or missing fields) returns a structured PeekabooError so
+ * callers can propagate it through their discriminated union.
+ */
+function unwrapEnvelope(command, parsed, stdout) {
+    if (!isRecord(parsed)) {
+        return { ok: false, err: failParse(`${command}: expected object at root`, stdout) };
+    }
+    if (parsed.success === false) {
+        const errMsg = isRecord(parsed.error)
+            ? asString(parsed.error.message, asString(parsed.error.code, 'unknown error'))
+            : asString(parsed.error, 'unknown error');
+        return {
+            ok: false,
+            err: failParse(`${command}: success=false: ${errMsg}`, stdout),
+        };
+    }
+    if (!isRecord(parsed.data)) {
+        return { ok: false, err: failParse(`${command}: missing data field`, stdout) };
+    }
+    return { ok: true, data: parsed.data };
+}
 export async function see(opts = {}) {
     const args = ['see', '--json'];
     if (opts.app)
@@ -61,39 +95,39 @@ export async function see(opts = {}) {
     const parsed = parseJson(stdout);
     if (!parsed.ok)
         return parsed.err;
-    const v = parsed.value;
-    if (!isRecord(v))
-        return failParse('see: expected object at root', stdout);
-    const rawElements = Array.isArray(v.elements) ? v.elements : [];
+    const env = unwrapEnvelope('see', parsed.value, stdout);
+    if (!env.ok)
+        return env.err;
+    const data = env.data;
+    const rawElements = Array.isArray(data.ui_elements) ? data.ui_elements : [];
     const elements = rawElements.flatMap((el) => {
         if (!isRecord(el))
             return [];
-        const frameRaw = isRecord(el.frame) ? el.frame : {};
         return [
             {
                 id: asString(el.id),
                 role: asString(el.role),
-                label: typeof el.label === 'string' ? el.label : undefined,
-                frame: {
-                    x: asNumber(frameRaw.x),
-                    y: asNumber(frameRaw.y),
-                    width: asNumber(frameRaw.width),
-                    height: asNumber(frameRaw.height),
-                },
-                settable: typeof el.settable === 'boolean' ? el.settable : undefined,
-                named_actions: Array.isArray(el.named_actions)
-                    ? el.named_actions.filter((a) => typeof a === 'string')
-                    : undefined,
+                roleDescription: asOptString(el.role_description),
+                label: asOptString(el.label),
+                description: asOptString(el.description),
+                help: asOptString(el.help),
+                identifier: asOptString(el.identifier),
+                title: asOptString(el.title),
+                isActionable: asOptBool(el.is_actionable),
             },
         ];
     });
     return {
         ok: true,
-        snapshot: asString(v.snapshot),
-        app: typeof v.app === 'string' ? v.app : undefined,
-        window: typeof v.window === 'string' ? v.window : undefined,
+        snapshot: asString(data.snapshot_id),
         elements,
-        screenshot_path: typeof v.screenshot_path === 'string' ? v.screenshot_path : undefined,
+        applicationName: asOptString(data.application_name),
+        windowTitle: asOptString(data.window_title),
+        elementCount: asOptNumber(data.element_count),
+        interactableCount: asOptNumber(data.interactable_count),
+        captureMode: asOptString(data.capture_mode),
+        uiMap: asOptString(data.ui_map),
+        screenshotPath: asOptString(data.screenshot_path),
     };
 }
 export async function click(opts) {
@@ -103,27 +137,29 @@ export async function click(opts) {
     if (opts.coords)
         args.push('--coords', `${opts.coords[0]},${opts.coords[1]}`);
     if (typeof opts.wait === 'number')
-        args.push('--wait', String(opts.wait));
+        args.push('--wait-for', String(opts.wait));
     const { stdout, stderr, code } = await runPeekaboo(args);
     if (code !== 0)
         return failNonZero(code, stdout, stderr);
     const parsed = parseJson(stdout);
     if (!parsed.ok)
         return parsed.err;
-    if (!isRecord(parsed.value))
-        return failParse('click: expected object at root', stdout);
-    const v = parsed.value;
+    const env = unwrapEnvelope('click', parsed.value, stdout);
+    if (!env.ok)
+        return env.err;
+    const v = env.data;
     return {
         ok: true,
-        target: typeof v.target === 'string' ? v.target : undefined,
+        target: asOptString(v.target),
         coords: Array.isArray(v.coords) && v.coords.length === 2
             ? [asNumber(v.coords[0]), asNumber(v.coords[1])]
             : undefined,
     };
 }
 // `type` is reserved in TS; export the helper as `type_`.
+// peekaboo 3.0.0-beta4 takes the text as a positional argument, not a --text flag.
 export async function type_(opts) {
-    const args = ['type', '--json', '--text', opts.text];
+    const args = ['type', '--json', opts.text];
     if (opts.clear)
         args.push('--clear');
     if (typeof opts.delayMs === 'number')
@@ -134,65 +170,34 @@ export async function type_(opts) {
     const parsed = parseJson(stdout);
     if (!parsed.ok)
         return parsed.err;
-    if (!isRecord(parsed.value))
-        return failParse('type: expected object at root', stdout);
-    const v = parsed.value;
+    const env = unwrapEnvelope('type', parsed.value, stdout);
+    if (!env.ok)
+        return env.err;
+    const v = env.data;
     return {
         ok: true,
         typed: asString(v.typed, opts.text),
-        cleared: typeof v.cleared === 'boolean' ? v.cleared : undefined,
+        cleared: asOptBool(v.cleared),
     };
 }
-export async function setValue(opts) {
-    const args = [
-        'set-value',
-        '--json',
-        '--snapshot',
-        opts.snapshot,
-        '--on',
-        opts.on,
-        '--value',
-        opts.value,
-    ];
-    const { stdout, stderr, code } = await runPeekaboo(args);
-    if (code !== 0)
-        return failNonZero(code, stdout, stderr);
-    const parsed = parseJson(stdout);
-    if (!parsed.ok)
-        return parsed.err;
-    if (!isRecord(parsed.value))
-        return failParse('set-value: expected object at root', stdout);
-    const v = parsed.value;
+export async function setValue(_opts) {
     return {
-        ok: true,
-        target: asString(v.target, opts.on),
-        value: asString(v.value, opts.value),
+        ok: false,
+        error: {
+            code: 'unknown',
+            message: "peekaboo 3.0.0-beta4 does not expose a 'set-value' top-level command. " +
+                'Track upstream: https://github.com/openclaw/Peekaboo',
+        },
     };
 }
-export async function performAction(opts) {
-    const args = [
-        'perform-action',
-        '--json',
-        '--snapshot',
-        opts.snapshot,
-        '--on',
-        opts.on,
-        '--action',
-        opts.action,
-    ];
-    const { stdout, stderr, code } = await runPeekaboo(args);
-    if (code !== 0)
-        return failNonZero(code, stdout, stderr);
-    const parsed = parseJson(stdout);
-    if (!parsed.ok)
-        return parsed.err;
-    if (!isRecord(parsed.value))
-        return failParse('perform-action: expected object at root', stdout);
-    const v = parsed.value;
+export async function performAction(_opts) {
     return {
-        ok: true,
-        target: asString(v.target, opts.on),
-        action: asString(v.action, opts.action),
+        ok: false,
+        error: {
+            code: 'unknown',
+            message: "peekaboo 3.0.0-beta4 does not expose a 'perform-action' top-level command. " +
+                'Track upstream: https://github.com/openclaw/Peekaboo',
+        },
     };
 }
 /**

package/dist/adapters/peekaboo/index.d.ts

@@ -1,4 +1,4 @@
-export type { PeekabooFrame, PeekabooElement, PeekabooSeeResult, PeekabooClickResult, PeekabooTypeResult, PeekabooSetValueResult, PeekabooPerformActionResult, PeekabooAgentResult, PeekabooError, PeekabooOutcome, } from './types.js';
+export type { PeekabooElement, PeekabooSeeResult, PeekabooClickResult, PeekabooTypeResult, PeekabooSetValueResult, PeekabooPerformActionResult, PeekabooAgentResult, PeekabooError, PeekabooOutcome, } from './types.js';
 export { runPeekaboo, peekabooAvailable, type RunOptions, type RunResult } from './runner.js';
 export { see, click, type_, setValue, performAction, agent, type SeeOptions, type ClickOptions, type TypeOptions, type SetValueOptions, type PerformActionOptions, type AgentOptions, } from './commands.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/adapters/peekaboo/types.d.ts

@@ -1,28 +1,34 @@
-export interface PeekabooFrame {
-    x: number;
-    y: number;
-    width: number;
-    height: number;
-}
 export interface PeekabooElement {
-    /** Element handle, e.g. "B1" (button), "T1" (text field). */
+    /** Element handle, e.g. "elem_19", "elem_169" — integer-suffixed in 3.0.0-beta4. */
     id: string;
     role: string;
+    /** Human-readable role description, e.g. "increment page button". */
+    roleDescription?: string;
     label?: string;
-    frame: PeekabooFrame;
-    /** Whether the element accepts a value (text fields, sliders, etc.). */
-    settable?: boolean;
-    /** Action names the element advertises via the AX API. */
-    named_actions?: string[];
+    description?: string;
+    /** AX help text, e.g. "Share the selected items". */
+    help?: string;
+    /** Stable AX identifier when the app sets one (e.g. "QuickActionMoreButton"). */
+    identifier?: string;
+    /** Window/element title; often empty for buttons. */
+    title?: string;
+    /** Whether the element is interactable (clickable / focusable). */
+    isActionable?: boolean;
 }
 export interface PeekabooSeeResult {
     /** Snapshot id used by subsequent `--snapshot $id` arguments. */
     snapshot: string;
-    app?: string;
-    window?: string;
     elements: PeekabooElement[];
+    /** Application name as reported by Peekaboo. */
+    applicationName?: string;
+    windowTitle?: string;
+    elementCount?: number;
+    interactableCount?: number;
+    captureMode?: string;
+    /** Path to the JSON UI map written by Peekaboo. */
+    uiMap?: string;
     /** Optional path on disk where the screenshot was written. */
-    screenshot_path?: string;
+    screenshotPath?: string;
 }
 export interface PeekabooClickResult {
     ok: boolean;

package/dist/adapters/peekaboo/types.js

@@ -4,5 +4,10 @@
 // (https://github.com/openclaw/Peekaboo) without taking a runtime
 // dependency. kbot stays binary-agnostic; this adapter only ever
 // speaks JSON across the process boundary.
+//
+// Schema is calibrated to peekaboo 3.0.0-beta4: every command wraps
+// its payload in `{ success, data, error? }`, element ids look like
+// `elem_19` / `elem_169`, and elements expose AX role/label/help
+// fields rather than frame rectangles or named-actions arrays.
 export {};
 //# sourceMappingURL=types.js.map

package/dist/tools/peekaboo.js

@@ -63,7 +63,7 @@ async function ensureBinary() {
 // ── Tool definitions ───────────────────────────────────────────────────
 const peekabooSee = {
     name: 'peekaboo_see',
-    description: 'Capture an AX snapshot of an app or the screen via the Peekaboo CLI. Returns a snapshot id plus a list of labeled element ids (B1, T1, …) usable by peekaboo_click / peekaboo_set_value / peekaboo_perform_action.',
+    description: 'Capture an AX snapshot of an app or the screen via the Peekaboo CLI. Returns a snapshot id plus a list of element ids (e.g. elem_19, elem_169) usable by peekaboo_click / peekaboo_type.',
     parameters: {
         app: {
             type: 'string',
@@ -120,7 +120,7 @@ const peekabooClick = {
         },
         on: {
             type: 'string',
-            description: 'Element id (e.g. "B1") or query string. Mutually exclusive with coords.',
+            description: 'Element id (e.g. "elem_169") or query string. Mutually exclusive with coords.',
         },
         coords: {
             type: 'string',
@@ -225,7 +225,7 @@ const peekabooType = {
 };
 const peekabooSetValue = {
     name: 'peekaboo_set_value',
-    description: 'Set a settable AX value directly on an element (skips clicking). Faster than click+type for text fields, sliders, etc.',
+    description: "Set a settable AX value directly on an element (skips clicking). NOTE: requires Peekaboo CLI with the 'set-value' top-level command, which is absent in 3.0.0-beta4. Use peekaboo_click + peekaboo_type as a workaround.",
     parameters: {
         app: {
             type: 'string',
@@ -249,35 +249,16 @@ const peekabooSetValue = {
         },
     },
     tier: 'free',
-    async execute(args) {
-        const binErr = await ensureBinary();
-        if (binErr)
-            return binErr;
-        const app = String(args.app ?? '');
-        if (!app)
-            return 'Error: app is required.';
-        const gate = requireApproval(app);
-        if (gate)
-            return gate;
-        const snapshot = String(args.snapshot ?? '');
-        const on = String(args.on ?? '');
-        const value = typeof args.value === 'string' ? args.value : '';
-        if (!snapshot)
-            return 'Error: snapshot is required.';
-        if (!on)
-            return 'Error: on is required.';
-        try {
-            const out = await setValue({ snapshot, on, value });
-            return outcomeToString(out);
-        }
-        catch (e) {
-            return `Error: ${e.message}`;
-        }
+    async execute(_args) {
+        void setValue;
+        return ("Error: peekaboo_set_value requires Peekaboo CLI with the 'set-value' top-level command, " +
+            'which is not present in 3.0.0-beta4. Workaround: peekaboo_click then peekaboo_type. ' +
+            'Track upstream: https://github.com/openclaw/Peekaboo');
     },
 };
 const peekabooPerformAction = {
     name: 'peekaboo_perform_action',
-    description: 'Invoke a named AX action (e.g. AXPress, AXShowMenu) on an element from a Peekaboo snapshot.',
+    description: "Invoke a named AX action (e.g. AXPress, AXShowMenu) on an element from a Peekaboo snapshot. NOTE: requires Peekaboo CLI with the 'perform-action' top-level command, which is absent in 3.0.0-beta4. Use peekaboo_click as a workaround.",
     parameters: {
         app: {
             type: 'string',
@@ -301,32 +282,11 @@ const peekabooPerformAction = {
         },
     },
     tier: 'free',
-    async execute(args) {
-        const binErr = await ensureBinary();
-        if (binErr)
-            return binErr;
-        const app = String(args.app ?? '');
-        if (!app)
-            return 'Error: app is required.';
-        const gate = requireApproval(app);
-        if (gate)
-            return gate;
-        const snapshot = String(args.snapshot ?? '');
-        const on = String(args.on ?? '');
-        const action = String(args.action ?? '');
-        if (!snapshot)
-            return 'Error: snapshot is required.';
-        if (!on)
-            return 'Error: on is required.';
-        if (!action)
-            return 'Error: action is required.';
-        try {
-            const out = await performAction({ snapshot, on, action });
-            return outcomeToString(out);
-        }
-        catch (e) {
-            return `Error: ${e.message}`;
-        }
+    async execute(_args) {
+        void performAction;
+        return ("Error: peekaboo_perform_action requires Peekaboo CLI with the 'perform-action' top-level command, " +
+            'which is not present in 3.0.0-beta4. Workaround: peekaboo_click on the element. ' +
+            'Track upstream: https://github.com/openclaw/Peekaboo');
     },
 };
 const peekabooAgent = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kernel.chat/kbot",
-  "version": "4.4.0",
+  "version": "4.4.1",
   "description": "Open-source terminal AI agent. 100+ specialist skills, 35 specialist agents, 20 providers. Dreams, learns, watches your system. Controls your phone. Fully local, fully sovereign. MIT. v4.0 — evidence-based curation.",
   "type": "module",
   "repository": {

package/skills/native-automation/peekaboo-snapshot-act/SKILL.md

@@ -35,7 +35,7 @@ ELEMENT ID OVER COORDINATES.
 PERFORM-ACTION OVER CLICK.
 ```
 
-A snapshot is a contract: while the UI does not change, the IDs are stable. Capture once, act many times, re-snapshot only on visible state change. An element ID survives where coordinates do not — themes shift, windows resize, scroll positions move; `B7` does not. And when an AX action is named (`AXPress`, `AXShowMenu`, `AXIncrement`), perform it directly; clicking the rendered pixel is a worse approximation of the user's intent.
+A snapshot is a contract: while the UI does not change, the IDs are stable. Capture once, act many times, re-snapshot only on visible state change. An element ID survives where coordinates do not — themes shift, windows resize, scroll positions move; `elem_169` does not. And when an AX action is named (`AXPress`, `AXShowMenu`, `AXIncrement`), perform it directly; clicking the rendered pixel is a worse approximation of the user's intent.
 
 ## Five Phases
 
@@ -55,19 +55,21 @@ Pull the AX snapshot once.
 peekaboo see --app <Name> --json
 ```
 
-The response contains a snapshot ID and labeled element IDs by role: `B1` for the first button, `T1` for the first text field, `L1` for a link, `M1` for a menu. Read the labels, not the pixels. The snapshot ID is the handle every subsequent call references.
+The response contains a snapshot ID (`data.snapshot_id`) and a list of UI elements (`data.ui_elements`), each with an integer-suffixed id like `elem_19`, `elem_85`, `elem_169`. Read the labels and roles, not the pixels. The snapshot ID is the handle every subsequent call references.
+
+> Element IDs in 3.0.0-beta4 are `elem_NN` integers; the README's `B1`/`T2` examples target a future schema.
 
 ### Phase 3 — Choose the right verb
 
 Three verbs cover almost every native interaction. Pick the most specific one that fits.
 
-- `set-value` — for any settable field (text inputs, sliders, steppers). Faster and more reliable than `click + type`. Sets the AX value directly.
-- `perform-action` — for any named AX action (`AXPress`, `AXShowMenu`, `AXConfirm`, `AXIncrement`, `AXDecrement`). Names the intent the OS already understands.
-- `click` — only when neither of the above applies (custom non-AX views, web content embedded in a native shell).
+- `click` — the universal verb in 3.0.0-beta4. Targets an element by id (`--on elem_169`), by query string, or by raw `--coords x,y`.
+- `type` — text input. Assumes a focused field; pair with a prior `click` to focus.
+- `set-value` / `perform-action` — reserved for future Peekaboo releases. The 3.0.0-beta4 binary does not expose them as top-level commands; the kbot tools surface a clear error pointing at the workaround (`click` + `type`). Track upstream at https://github.com/openclaw/Peekaboo.
 
 ### Phase 4 — Reuse the snapshot
 
-Successive actions reference the same `--snapshot $ID` until the UI changes. Filling a five-field form is one snapshot and five `set-value` calls, not five snapshots and five clicks. Re-snapshot only when the visible state actually changes — a panel opens, a sheet appears, a navigation transitions. Re-snapshotting before every action defeats the entire pattern and is slower than synthetic input.
+Successive actions reference the same `--snapshot $ID` until the UI changes. Filling a five-field form is one snapshot and five `click`/`type` pairs, not five snapshots and five blind clicks. Re-snapshot only when the visible state actually changes — a panel opens, a sheet appears, a navigation transitions. Re-snapshotting before every action defeats the entire pattern and is slower than synthetic input.
 
 ### Phase 5 — Fall back gracefully
 
@@ -77,7 +79,7 @@ If the AX path fails — element ID stale, app exposes no AX tree, action return
 
 - Re-snapshotting before every click. The whole point is reuse — one snapshot, many actions.
 - Using coordinates when an element ID exists. IDs survive resize, theme, and scale changes; coordinates do not.
-- Ignoring `set-value` and falling through to `click + type` for text fields. Slower, less reliable, and breaks on focus drift.
+- Assuming the README's `set-value` / `perform-action` commands exist in the installed binary. Until 3.x ships them, `click + type` is the path.
 - Driving Chrome with Peekaboo. Chrome MCP exists for a reason; the DOM is the right surface for the web.
 - Skipping `app_approve`. The per-app session lock and sensitive-app warnings still apply — Peekaboo does not bypass kbot's trust model.