@theaiinc/yggdrasil-ratatoskr 0.2.3 → 0.3.0

package/README.md

@@ -1,5 +1,15 @@
 # @theaiinc/yggdrasil-ratatoskr
 
+<p align="center">
+  <a href="https://github.com/theaiinc/yggdrasil"><img alt="GitHub Repo" src="https://img.shields.io/badge/github-theaiinc%2Fyggdrasil-181717?style=flat-square&logo=github"/></a>
+  <a href="https://www.npmjs.com/package/@theaiinc/yggdrasil-ratatoskr"><img alt="npm" src="https://img.shields.io/npm/v/@theaiinc/yggdrasil-ratatoskr?style=flat-square&logo=npm"/></a>
+  <a href="https://github.com/theaiinc/yggdrasil/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/theaiinc/yggdrasil?style=flat-square"/></a>
+</p>
+
+<p align="center">
+  <img src="./ratatoskr.svg" alt="Ratatoskr" width="300" />
+</p>
+
 Lightweight runner daemon for Yggdrasil — registers, heartbeats, and executes tasks with configurable capability presets.
 
 Ratatoskr runs alongside any agent (Docker container, laptop, server) and continuously informs [Yggdrasil](https://www.npmjs.com/package/@theaiinc/yggdrasil) about runner availability, capabilities, health, and task execution. It can execute tasks itself using its built-in LLM, shell, web, code, and file handlers.
@@ -43,6 +53,8 @@ Built-in presets (available by name):
 | `python` | Python script execution | `python` | — |
 | `node_runtime` | Node.js script execution | `node` | — |
 | `github_cli` | GitHub CLI operations | `github` | `shell` |
+| `computer_use` | Desktop automation via Realm ubuntu engine | `computer_use` | `llm` |
+| `android` | Android emulator automation via Realm VM engine | `android` | `llm` |
 
 ```typescript
 import { Ratatoskr } from '@theaiinc/yggdrasil-ratatoskr';
@@ -202,6 +214,8 @@ All built-in handlers return structured metadata:
 | `python` | `{ stdout, stderr, code }` |
 | `node` | `{ stdout, stderr, code }` |
 | `github` | `{ stdout, stderr, code }` |
+| `computer_use` | `{ goal, realmId, engine, iterations, actions }` |
+| `android` | `{ goal, realmId, engine, iterations, actions }` |
 
 > **Cost tracking**: Cost is computed server-side by the orchestration layer (e.g. Oasis api-gateway's `PricingService`). Ratatoskr reports the actual model used and token counts — the server applies per-model pricing.
 
@@ -250,6 +264,31 @@ The agent has access to: `shell`, `read_file`, `write_file`, `web_search`, `web_
 | `LLM_BASE_URL` | `http://host.docker.internal:1234/v1` | OpenAI-compatible base URL |
 | `LLM_API_KEY` | `''` | API key (optional for local LM Studio) |
 | `AGENT_MAX_TOOL_ITERATIONS` | `25` | Max tool call cycles per agent task |
+| `REALM_URL` | `http://localhost:8542` | Realm API server URL (for `computer_use` / `android`) |
+| `REALM_ID` | `''` | Existing realm to reuse (optional) |
+| `REALM_AVD` | `Pixel_9_Pro` | Android Virtual Device name (for `android`) |
+| `CU_MAX_ITERATIONS` | `50` | Max screenshot-action cycles per automation task |
+
+## Realm Integration
+
+`computer_use` and `android` capabilities talk to [@theaiinc/realm-api](https://github.com/theaiinc/theaiincrealm) over the universal `/api/v1/realms/:id/*` interface:
+
+| Preset | Realm Engine | Use Case |
+|--------|-------------|----------|
+| `computer_use` | `ubuntu` | Desktop automation (XFCE + xdotool) |
+| `android` | `vm` | Android emulator automation (ADB + Realm Agent APK) |
+
+```typescript
+const ratatoskr = new Ratatoskr({
+  yggdrasilUrl: 'http://localhost:3000',
+  capabilities: ['android', 'llm'],
+});
+
+// Task metadata for an Android automation goal:
+// { type: 'android', metadata: { goal: 'Open Settings and enable Wi-Fi' } }
+```
+
+Start the Realm API server first (`pnpm realm:api` in theaiincrealm repo). For Android, use the `wip/android-vm` branch which registers `VMEngine`.
 
 ## Services
 

package/dist/src/handlers/android-handler.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Android handler — session-based mobile automation via Realm VM engine (Android emulator + ADB).
+ *
+ * Creates a "phone-use" session: observe → decide → act → repeat.
+ * Streaming is an internal Realm implementation detail.
+ *
+ * Requires @theaiinc/realm-api with VMEngine registered and an Android SDK/AVD
+ * configured on the host running the Realm server.
+ */
+export declare const androidHandler: import("../index.js").TaskHandler;
+//# sourceMappingURL=android-handler.d.ts.map

package/dist/src/handlers/android-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"android-handler.d.ts","sourceRoot":"","sources":["../../../src/handlers/android-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAYH,eAAO,MAAM,cAAc,mCAE1B,CAAC"}

package/dist/src/handlers/android-handler.js

@@ -0,0 +1,22 @@
+/**
+ * Android handler — session-based mobile automation via Realm VM engine (Android emulator + ADB).
+ *
+ * Creates a "phone-use" session: observe → decide → act → repeat.
+ * Streaming is an internal Realm implementation detail.
+ *
+ * Requires @theaiinc/realm-api with VMEngine registered and an Android SDK/AVD
+ * configured on the host running the Realm server.
+ */
+import { createAutomationHandler, buildAutomationConfig } from './automation-loop.js';
+function androidEnvironment() {
+    const env = {};
+    if (process.env.REALM_AVD)
+        env.REALM_AVD = process.env.REALM_AVD;
+    if (process.env.ANDROID_HOME)
+        env.ANDROID_HOME = process.env.ANDROID_HOME;
+    if (process.env.REALM_HTTP_PROXY)
+        env.REALM_HTTP_PROXY = process.env.REALM_HTTP_PROXY;
+    return env;
+}
+export const androidHandler = createAutomationHandler(buildAutomationConfig('vm', androidEnvironment()));
+//# sourceMappingURL=android-handler.js.map

package/dist/src/handlers/android-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"android-handler.js","sourceRoot":"","sources":["../../../src/handlers/android-handler.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAEtF,SAAS,kBAAkB;IACzB,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS;QAAE,GAAG,CAAC,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC;IACjE,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY;QAAE,GAAG,CAAC,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAAE,GAAG,CAAC,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IACtF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAG,uBAAuB,CACnD,qBAAqB,CAAC,IAAI,EAAE,kBAAkB,EAAE,CAAC,CAClD,CAAC"}

package/dist/src/handlers/automation-loop.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Shared automation loop for session-based Computer Use (desktop + Android).
+ *
+ * Flow: create session → observe → decide action → send input → observe → repeat.
+ *
+ * Cognition interacts through sessions:
+ *   startSession({ type: "computer-use" })
+ *   observe(sessionId)  →  SessionObservation
+ *   input(sessionId, { type: "mouse", params: { x, y } })
+ *
+ * NOT through streaming:
+ *   requestScreenStream()  ✗
+ *   startStream()          ✗
+ *   stopStream()           ✗
+ */
+import type { TaskHandler } from '../types/index.js';
+import { type RealmEngineType } from './realm-client.js';
+export interface AutomationConfig {
+    engine: RealmEngineType;
+    realmUrl: string;
+    realmApiKey?: string | undefined;
+    realmId?: string | undefined;
+    realmName?: string | undefined;
+    environment?: Record<string, string> | undefined;
+    maxIterations: number;
+}
+export interface AutomationAction {
+    action: string;
+    params: Record<string, unknown>;
+}
+export declare function createAutomationHandler(config: AutomationConfig): TaskHandler;
+export declare function buildAutomationConfig(engine: RealmEngineType, extraEnv?: Record<string, string>): AutomationConfig;
+//# sourceMappingURL=automation-loop.d.ts.map

package/dist/src/handlers/automation-loop.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"automation-loop.d.ts","sourceRoot":"","sources":["../../../src/handlers/automation-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAc,MAAM,mBAAmB,CAAC;AAEjE,OAAO,EAGL,KAAK,eAAe,EAGrB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,eAAe,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IACjD,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAiED,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,gBAAgB,GAAG,WAAW,CA6G7E;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAChC,gBAAgB,CAWlB"}

package/dist/src/handlers/automation-loop.js

@@ -0,0 +1,189 @@
+/**
+ * Shared automation loop for session-based Computer Use (desktop + Android).
+ *
+ * Flow: create session → observe → decide action → send input → observe → repeat.
+ *
+ * Cognition interacts through sessions:
+ *   startSession({ type: "computer-use" })
+ *   observe(sessionId)  →  SessionObservation
+ *   input(sessionId, { type: "mouse", params: { x, y } })
+ *
+ * NOT through streaming:
+ *   requestScreenStream()  ✗
+ *   startStream()          ✗
+ *   stopStream()           ✗
+ */
+import { RealmClient, loadRealmUrl, loadRealmApiKey, } from './realm-client.js';
+async function decideAction(goal, observation, previousActions) {
+    // Placeholder — real implementation will use a vision-capable LLM.
+    void goal;
+    void observation;
+    void previousActions;
+    return { action: 'wait', params: { ms: 500 } };
+}
+function isGoalComplete(result) {
+    const data = result.data;
+    if (data && typeof data === 'object' && 'goal_complete' in data) {
+        return Boolean(data.goal_complete);
+    }
+    return false;
+}
+async function executeSessionAction(client, sessionId, action, params) {
+    switch (action) {
+        case 'click':
+        case 'tap': {
+            const input = {
+                type: action === 'click' ? 'mouse' : 'touch',
+                params: { x: params.x, y: params.y },
+            };
+            const result = await client.input(sessionId, input);
+            return {
+                success: result.success,
+                ...(result.error !== undefined ? { error: result.error } : {}),
+            };
+        }
+        case 'type': {
+            const input = {
+                type: 'keyboard',
+                params: { text: params.text },
+            };
+            const result = await client.input(sessionId, input);
+            return {
+                success: result.success,
+                ...(result.error !== undefined ? { error: result.error } : {}),
+            };
+        }
+        case 'navigate':
+            // Navigate is a Realm input action.
+            return { success: false, error: 'Navigate not yet supported in session mode' };
+        case 'exec':
+            return { success: false, error: 'Exec not yet supported in session mode' };
+        case 'wait':
+            await new Promise((r) => setTimeout(r, params.ms ?? 500));
+            return { success: true };
+        default:
+            return { success: false, error: `Unknown action: ${action}` };
+    }
+}
+export function createAutomationHandler(config) {
+    return async (task) => {
+        const goal = task.metadata?.goal || '';
+        if (!goal) {
+            return { status: 'failed', metadata: { error: 'No goal specified for automation task' } };
+        }
+        const realmIdOverride = task.metadata?.realmId || config.realmId;
+        const clientConfig = {
+            baseUrl: config.realmUrl,
+            engine: task.metadata?.engine || config.engine,
+        };
+        if (config.realmApiKey)
+            clientConfig.apiKey = config.realmApiKey;
+        if (realmIdOverride)
+            clientConfig.realmId = realmIdOverride;
+        if (config.realmName)
+            clientConfig.realmName = config.realmName;
+        if (config.environment)
+            clientConfig.environment = config.environment;
+        const client = new RealmClient(clientConfig);
+        const previousActions = [];
+        let iteration = 0;
+        let sessionId;
+        try {
+            // Create a session for this automation task.
+            // Cognition requests: startSession({ type: "computer-use" })
+            const sessionType = config.engine === 'vm' ? 'phone-use' : 'computer-use';
+            const session = await client.createSession({
+                type: sessionType,
+                ...(realmIdOverride !== undefined ? { realmId: realmIdOverride } : {}),
+                ...({ metadata: { goal, engine: config.engine } }),
+            });
+            sessionId = session.sessionId;
+            while (iteration < config.maxIterations) {
+                iteration++;
+                // Observe the current state via the session.
+                // Realm decides how observation is delivered (screenshot, a11y tree, etc.).
+                const observation = await client.observe(sessionId);
+                const decision = await decideAction(goal, observation, previousActions);
+                const result = await executeSessionAction(client, sessionId, decision.action, decision.params);
+                if (!result.success) {
+                    return {
+                        status: 'failed',
+                        metadata: {
+                            error: result.error || `Action failed: ${decision.action}`,
+                            goal,
+                            realmId: session.descriptor?.realmId,
+                            sessionId,
+                            engine: config.engine,
+                            iterations: iteration,
+                            lastAction: decision.action,
+                        },
+                    };
+                }
+                previousActions.push(`${decision.action}(${JSON.stringify(decision.params)})`);
+                if (isGoalComplete(result)) {
+                    return {
+                        status: 'completed',
+                        metadata: {
+                            goal,
+                            realmId: session.descriptor?.realmId,
+                            sessionId,
+                            engine: config.engine,
+                            iterations: iteration,
+                            actions: previousActions,
+                        },
+                    };
+                }
+                if (decision.action === 'wait') {
+                    const ms = decision.params.ms ?? 500;
+                    await new Promise((r) => setTimeout(r, ms));
+                }
+            }
+            return {
+                status: 'completed',
+                metadata: {
+                    goal,
+                    realmId: (await client.getSession(sessionId))?.realmId,
+                    sessionId,
+                    engine: config.engine,
+                    iterations: iteration,
+                    actions: previousActions,
+                    note: 'Max iterations reached',
+                },
+            };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            return {
+                status: 'failed',
+                metadata: {
+                    error: `Automation loop error at iteration ${iteration}: ${message}`,
+                    goal,
+                    engine: config.engine,
+                    iterations: iteration,
+                },
+            };
+        }
+        finally {
+            if (sessionId) {
+                const destroy = task.metadata?.destroyRealm === true || process.env.REALM_DESTROY_ON_COMPLETE === 'true';
+                await client.terminateSession(sessionId, destroy);
+            }
+        }
+    };
+}
+export function buildAutomationConfig(engine, extraEnv) {
+    const config = {
+        engine,
+        realmUrl: loadRealmUrl(),
+        maxIterations: parseInt(process.env.CU_MAX_ITERATIONS || '50', 10),
+    };
+    const apiKey = loadRealmApiKey();
+    if (apiKey)
+        config.realmApiKey = apiKey;
+    if (process.env.REALM_ID)
+        config.realmId = process.env.REALM_ID;
+    if (extraEnv && Object.keys(extraEnv).length > 0)
+        config.environment = extraEnv;
+    return config;
+}
+//# sourceMappingURL=automation-loop.js.map

package/dist/src/handlers/automation-loop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"automation-loop.js","sourceRoot":"","sources":["../../../src/handlers/automation-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,EACL,WAAW,EAGX,YAAY,EACZ,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAiB3B,KAAK,UAAU,YAAY,CACzB,IAAY,EACZ,WAA+B,EAC/B,eAAyB;IAEzB,mEAAmE;IACnE,KAAK,IAAI,CAAC;IACV,KAAK,WAAW,CAAC;IACjB,KAAK,eAAe,CAAC;IACrB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,cAAc,CAAC,MAA0B;IAChD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;QAChE,OAAO,OAAO,CAAE,IAAgC,CAAC,aAAa,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,MAAmB,EACnB,SAAiB,EACjB,MAAc,EACd,MAA+B;IAE/B,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,OAAO,CAAC;QACb,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,MAAM,KAAK,GAAiB;gBAC1B,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO;gBAC5C,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,EAAE;aACrC,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,OAAO;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/D,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,CAAC,CAAC,CAAC;YACZ,MAAM,KAAK,GAAiB;gBAC1B,IAAI,EAAE,UAAU;gBAChB,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,IAAc,EAAE;aACxC,CAAC;YACF,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACpD,OAAO;gBACL,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC/D,CAAC;QACJ,CAAC;QACD,KAAK,UAAU;YACb,oCAAoC;YACpC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,4CAA4C,EAAE,CAAC;QACjF,KAAK,MAAM;YACT,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wCAAwC,EAAE,CAAC;QAC7E,KAAK,MAAM;YACT,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAG,MAAM,CAAC,EAAa,IAAI,GAAG,CAAC,CAAC,CAAC;YACtE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QAC3B;YACE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mBAAmB,MAAM,EAAE,EAAE,CAAC;IAClE,CAAC;AACH,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,MAAwB;IAC9D,OAAO,KAAK,EAAE,IAAgB,EAAE,EAAE;QAChC,MAAM,IAAI,GAAI,IAAI,CAAC,QAAQ,EAAE,IAAe,IAAI,EAAE,CAAC;QACnD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,uCAAuC,EAAE,EAAE,CAAC;QAC5F,CAAC;QAED,MAAM,eAAe,GAAI,IAAI,CAAC,QAAQ,EAAE,OAAkB,IAAI,MAAM,CAAC,OAAO,CAAC;QAC7E,MAAM,YAAY,GAAsB;YACtC,OAAO,EAAE,MAAM,CAAC,QAAQ;YACxB,MAAM,EAAG,IAAI,CAAC,QAAQ,EAAE,MAA0B,IAAI,MAAM,CAAC,MAAM;SACpE,CAAC;QACF,IAAI,MAAM,CAAC,WAAW;YAAE,YAAY,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC;QACjE,IAAI,eAAe;YAAE,YAAY,CAAC,OAAO,GAAG,eAAe,CAAC;QAC5D,IAAI,MAAM,CAAC,SAAS;YAAE,YAAY,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QAChE,IAAI,MAAM,CAAC,WAAW;YAAE,YAAY,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtE,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,YAAY,CAAC,CAAC;QAE7C,MAAM,eAAe,GAAa,EAAE,CAAC;QACrC,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,SAA6B,CAAC;QAElC,IAAI,CAAC;YACH,6CAA6C;YAC7C,6DAA6D;YAC7D,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,cAAc,CAAC;YAC1E,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;gBACzC,IAAI,EAAE,WAAW;gBACjB,GAAG,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtE,GAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;aACnD,CAAC,CAAC;YACH,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;YAE9B,OAAO,SAAS,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;gBACxC,SAAS,EAAE,CAAC;gBAEZ,6CAA6C;gBAC7C,4EAA4E;gBAC5E,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACpD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,eAAe,CAAC,CAAC;gBACxE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAE/F,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBACpB,OAAO;wBACL,MAAM,EAAE,QAAQ;wBAChB,QAAQ,EAAE;4BACR,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,kBAAkB,QAAQ,CAAC,MAAM,EAAE;4BAC1D,IAAI;4BACJ,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO;4BACpC,SAAS;4BACT,MAAM,EAAE,MAAM,CAAC,MAAM;4BACrB,UAAU,EAAE,SAAS;4BACrB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,eAAe,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAE/E,IAAI,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC3B,OAAO;wBACL,MAAM,EAAE,WAAW;wBACnB,QAAQ,EAAE;4BACR,IAAI;4BACJ,OAAO,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO;4BACpC,SAAS;4BACT,MAAM,EAAE,MAAM,CAAC,MAAM;4BACrB,UAAU,EAAE,SAAS;4BACrB,OAAO,EAAE,eAAe;yBACzB;qBACF,CAAC;gBACJ,CAAC;gBAED,IAAI,QAAQ,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;oBAC/B,MAAM,EAAE,GAAI,QAAQ,CAAC,MAAM,CAAC,EAAa,IAAI,GAAG,CAAC;oBACjD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;YAED,OAAO;gBACL,MAAM,EAAE,WAAW;gBACnB,QAAQ,EAAE;oBACR,IAAI;oBACJ,OAAO,EAAE,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO;oBACtD,SAAS;oBACT,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE,SAAS;oBACrB,OAAO,EAAE,eAAe;oBACxB,IAAI,EAAE,wBAAwB;iBAC/B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,OAAO;gBACL,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE;oBACR,KAAK,EAAE,sCAAsC,SAAS,KAAK,OAAO,EAAE;oBACpE,IAAI;oBACJ,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE,SAAS;iBACtB;aACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,SAAS,EAAE,CAAC;gBACd,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,MAAM,CAAC;gBACzG,MAAM,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,MAAuB,EACvB,QAAiC;IAEjC,MAAM,MAAM,GAAqB;QAC/B,MAAM;QACN,QAAQ,EAAE,YAAY,EAAE;QACxB,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,IAAI,EAAE,EAAE,CAAC;KACnE,CAAC;IACF,MAAM,MAAM,GAAG,eAAe,EAAE,CAAC;IACjC,IAAI,MAAM;QAAE,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC;IACxC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ;QAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;IAChE,IAAI,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;IAChF,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/src/handlers/cu-handler.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Computer Use handler — session-based desktop automation via Realm ubuntu engine.
+ *
+ * Creates a "computer-use" session: observe → decide → act → repeat.
+ * Streaming is an internal Realm implementation detail.
+ */
+export declare const computerUseHandler: import("../index.js").TaskHandler;
+//# sourceMappingURL=cu-handler.d.ts.map

package/dist/src/handlers/cu-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cu-handler.d.ts","sourceRoot":"","sources":["../../../src/handlers/cu-handler.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,eAAO,MAAM,kBAAkB,mCAE9B,CAAC"}

package/dist/src/handlers/cu-handler.js

@@ -0,0 +1,9 @@
+/**
+ * Computer Use handler — session-based desktop automation via Realm ubuntu engine.
+ *
+ * Creates a "computer-use" session: observe → decide → act → repeat.
+ * Streaming is an internal Realm implementation detail.
+ */
+import { createAutomationHandler, buildAutomationConfig } from './automation-loop.js';
+export const computerUseHandler = createAutomationHandler(buildAutomationConfig('ubuntu'));
+//# sourceMappingURL=cu-handler.js.map

package/dist/src/handlers/cu-handler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cu-handler.js","sourceRoot":"","sources":["../../../src/handlers/cu-handler.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAEtF,MAAM,CAAC,MAAM,kBAAkB,GAAG,uBAAuB,CACvD,qBAAqB,CAAC,QAAQ,CAAC,CAChC,CAAC"}

package/dist/src/handlers/realm-client.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Streamlined Realm API client — sessions replace direct capture/action calls.
+ *
+ * This client talks to @theaiinc/realm-api HTTP server but presents a
+ * session-based interface. Consumers interact through sessions:
+ *
+ *   const client = new RealmClient({ baseUrl, engine: 'ubuntu' });
+ *   const session = await client.createSession({ type: 'computer-use' });
+ *   const observation = await client.observe(session.sessionId);
+ *   const result = await client.input(session.sessionId, { type: 'mouse', params: { x: 100, y: 200 } });
+ *
+ * Consumers must NOT depend on transport details (WebRTC, screenshots, etc.).
+ */
+import type { CreateSessionRequest, CreateSessionResponse, SessionDescriptor, SessionObservation, SessionInput, SessionInputResult } from '../types/index.js';
+export type RealmEngineType = 'ubuntu' | 'vm' | 'container' | 'browser';
+export interface RealmActionResult {
+    success: boolean;
+    data?: unknown;
+    error?: string;
+    durationMs?: number;
+    timestamp?: string;
+}
+export interface RealmClientConfig {
+    baseUrl: string;
+    apiKey?: string | undefined;
+    realmId?: string | undefined;
+    engine: RealmEngineType;
+    realmName?: string | undefined;
+    environment?: Record<string, string> | undefined;
+}
+export declare class RealmClient {
+    private readonly http;
+    private readonly config;
+    private readonly realmUrl;
+    private readonly sessions;
+    constructor(config: RealmClientConfig);
+    /**
+     * Create an interaction session.
+     * This replaces the old ensureRealm() + capture() pattern.
+     */
+    createSession(request: CreateSessionRequest): Promise<CreateSessionResponse>;
+    /**
+     * Get the current session descriptor.
+     */
+    getSession(sessionId: string): Promise<SessionDescriptor>;
+    /**
+     * Observe the current state of a session.
+     * Realm decides how observation is delivered.
+     */
+    observe(sessionId: string): Promise<SessionObservation>;
+    /**
+     * Send an input action to a session.
+     */
+    input(sessionId: string, input: SessionInput): Promise<SessionInputResult>;
+    /**
+     * Terminate a session.
+     */
+    terminateSession(sessionId: string, destroyRealm?: boolean): Promise<void>;
+    /**
+     * Legacy alias for backward compatibility — delegates to createSession.
+     * @deprecated Use createSession({ type: "computer-use" }) instead.
+     */
+    ensureRealm(): Promise<string>;
+    private startIfNeeded;
+    private getSessionInternal;
+    private getCapabilitiesForType;
+}
+export declare function loadRealmUrl(): string;
+export declare function loadRealmApiKey(): string | undefined;
+//# sourceMappingURL=realm-client.d.ts.map

package/dist/src/handlers/realm-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realm-client.d.ts","sourceRoot":"","sources":["../../../src/handlers/realm-client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,KAAK,EACV,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAInB,MAAM,mBAAmB,CAAC;AAE3B,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,IAAI,GAAG,WAAW,GAAG,SAAS,CAAC;AAExE,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,MAAM,EAAE,eAAe,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;CAClD;AAaD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAgB;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;gBAEnD,MAAM,EAAE,iBAAiB;IAYrC;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,oBAAoB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAyClF;;OAEG;IACG,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAkB/D;;;OAGG;IACG,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAe7D;;OAEG;IACG,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA8BhF;;OAEG;IACG,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,YAAY,UAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAY9E;;;OAGG;IACG,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;YAiBtB,aAAa;IAQ3B,OAAO,CAAC,kBAAkB;IAS1B,OAAO,CAAC,sBAAsB;CAY/B;AAED,wBAAgB,YAAY,IAAI,MAAM,CAMrC;AAED,wBAAgB,eAAe,IAAI,MAAM,GAAG,SAAS,CAEpD"}

package/dist/src/handlers/realm-client.js

@@ -0,0 +1,199 @@
+/**
+ * Streamlined Realm API client — sessions replace direct capture/action calls.
+ *
+ * This client talks to @theaiinc/realm-api HTTP server but presents a
+ * session-based interface. Consumers interact through sessions:
+ *
+ *   const client = new RealmClient({ baseUrl, engine: 'ubuntu' });
+ *   const session = await client.createSession({ type: 'computer-use' });
+ *   const observation = await client.observe(session.sessionId);
+ *   const result = await client.input(session.sessionId, { type: 'mouse', params: { x: 100, y: 200 } });
+ *
+ * Consumers must NOT depend on transport details (WebRTC, screenshots, etc.).
+ */
+import axios from 'axios';
+import { nanoid } from 'nanoid';
+export class RealmClient {
+    http;
+    config;
+    realmUrl;
+    sessions = new Map();
+    constructor(config) {
+        this.config = config;
+        this.realmUrl = config.baseUrl.replace(/\/$/, '');
+        const headers = { 'Content-Type': 'application/json' };
+        if (config.apiKey)
+            headers['x-api-key'] = config.apiKey;
+        this.http = axios.create({
+            baseURL: this.realmUrl,
+            headers,
+            timeout: 60_000,
+        });
+    }
+    /**
+     * Create an interaction session.
+     * This replaces the old ensureRealm() + capture() pattern.
+     */
+    async createSession(request) {
+        const realmId = this.config.realmId || (await this.ensureRealm());
+        const sessionId = `session-${nanoid(12)}`;
+        const now = new Date().toISOString();
+        const capabilities = request.capabilities ?? this.getCapabilitiesForType(request.type);
+        const descriptor = {
+            id: sessionId,
+            type: request.type,
+            state: 'creating',
+            observationEndpoint: `${this.realmUrl}/api/v1/realms/${realmId}/capture`,
+            inputEndpoint: `${this.realmUrl}/api/v1/realms/${realmId}`,
+            capabilities,
+            observationMethod: 'screenshot',
+            realmId,
+            ...(request.ownerId !== undefined ? { ownerId: request.ownerId } : {}),
+            ...(request.participantIds !== undefined ? { participantIds: request.participantIds } : {}),
+            createdAt: now,
+            updatedAt: now,
+            ...(request.metadata !== undefined ? { metadata: request.metadata } : {}),
+        };
+        this.sessions.set(sessionId, {
+            id: sessionId,
+            type: request.type,
+            realmId,
+            state: 'creating',
+            capabilities,
+            ...(request.ownerId !== undefined ? { ownerId: request.ownerId } : {}),
+            ...(request.participantIds !== undefined ? { participantIds: request.participantIds } : {}),
+            createdAt: new Date(),
+        });
+        // Mark active after setup
+        descriptor.state = 'active';
+        this.sessions.get(sessionId).state = 'active';
+        return { sessionId, descriptor };
+    }
+    /**
+     * Get the current session descriptor.
+     */
+    async getSession(sessionId) {
+        const internal = this.getSessionInternal(sessionId);
+        return {
+            id: internal.id,
+            type: internal.type,
+            state: internal.state,
+            observationEndpoint: `${this.realmUrl}/api/v1/realms/${internal.realmId}/capture`,
+            inputEndpoint: `${this.realmUrl}/api/v1/realms/${internal.realmId}`,
+            capabilities: internal.capabilities,
+            observationMethod: 'screenshot',
+            realmId: internal.realmId,
+            ...(internal.ownerId !== undefined ? { ownerId: internal.ownerId } : {}),
+            ...(internal.participantIds !== undefined ? { participantIds: internal.participantIds } : {}),
+            createdAt: internal.createdAt.toISOString(),
+            updatedAt: new Date().toISOString(),
+        };
+    }
+    /**
+     * Observe the current state of a session.
+     * Realm decides how observation is delivered.
+     */
+    async observe(sessionId) {
+        const session = this.getSessionInternal(sessionId);
+        const timestamp = new Date().toISOString();
+        const { data } = await this.http.get(`/api/v1/realms/${session.realmId}/capture`);
+        return {
+            screenshot: data.screenshot,
+            ...(data.piiRedacted !== undefined ? { piiRedacted: data.piiRedacted } : {}),
+            timestamp,
+        };
+    }
+    /**
+     * Send an input action to a session.
+     */
+    async input(sessionId, input) {
+        const session = this.getSessionInternal(sessionId);
+        switch (input.type) {
+            case 'mouse':
+            case 'touch': {
+                const { data } = await this.http.post(`/api/v1/realms/${session.realmId}/click`, { x: input.params.x, y: input.params.y });
+                return {
+                    success: data.success,
+                    ...(data.error !== undefined ? { error: data.error } : {}),
+                };
+            }
+            case 'keyboard': {
+                const { data } = await this.http.post(`/api/v1/realms/${session.realmId}/type`, { text: input.params.text });
+                return {
+                    success: data.success,
+                    ...(data.error !== undefined ? { error: data.error } : {}),
+                };
+            }
+            default:
+                return { success: false, error: `Unsupported input type: ${input.type}` };
+        }
+    }
+    /**
+     * Terminate a session.
+     */
+    async terminateSession(sessionId, destroyRealm = false) {
+        const session = this.getSessionInternal(sessionId);
+        session.state = 'terminated';
+        if (destroyRealm && !this.config.realmId) {
+            try {
+                await this.http.delete(`/api/v1/realms/${session.realmId}`);
+            }
+            catch {
+                // best-effort
+            }
+        }
+    }
+    /**
+     * Legacy alias for backward compatibility — delegates to createSession.
+     * @deprecated Use createSession({ type: "computer-use" }) instead.
+     */
+    async ensureRealm() {
+        if (this.config.realmId) {
+            await this.startIfNeeded(this.config.realmId);
+            return this.config.realmId;
+        }
+        const name = this.config.realmName || `ratatoskr-${this.config.engine}-${Date.now()}`;
+        const { data } = await this.http.post('/api/v1/realms', {
+            name,
+            engine: this.config.engine,
+            environment: this.config.environment,
+        });
+        const realmId = data.id;
+        await this.startIfNeeded(realmId);
+        return realmId;
+    }
+    async startIfNeeded(realmId) {
+        const { data } = await this.http.get(`/api/v1/realms/${realmId}`);
+        if (data.realm?.session?.state === 'running')
+            return;
+        await this.http.post(`/api/v1/realms/${realmId}/start`);
+    }
+    getSessionInternal(sessionId) {
+        const session = this.sessions.get(sessionId);
+        if (!session)
+            throw new Error(`Session not found: ${sessionId}`);
+        if (session.state === 'terminated' || session.state === 'completed') {
+            throw new Error(`Session is ${session.state}: ${sessionId}`);
+        }
+        return session;
+    }
+    getCapabilitiesForType(type) {
+        switch (type) {
+            case 'computer-use':
+                return ['mouse', 'keyboard', 'scroll', 'clipboard'];
+            case 'phone-use':
+                return ['touch', 'keyboard', 'scroll'];
+            default:
+                return ['mouse', 'keyboard'];
+        }
+    }
+}
+export function loadRealmUrl() {
+    return (process.env.REALM_URL ||
+        process.env.CU_REALM_URL ||
+        'http://localhost:8542');
+}
+export function loadRealmApiKey() {
+    return process.env.REALM_API_KEY || process.env.CU_REALM_API_KEY || undefined;
+}
+//# sourceMappingURL=realm-client.js.map