@agimon-ai/doompi-computer-use 0.0.1-alpha.0

package/src/types/computerUse.ts

@@ -0,0 +1,117 @@
+export const COMPUTER_USE_PHASES = [
+  'inactive',
+  'requesting',
+  'awaiting_confirmation',
+  'activating',
+  'active',
+  'stopping',
+  'failed',
+] as const;
+
+export type ComputerUsePhase = (typeof COMPUTER_USE_PHASES)[number];
+
+export interface ComputerUseSessionIdentity {
+  readonly sessionId: string;
+  readonly runtimeId: string;
+  readonly modeEpoch: string;
+}
+
+export interface ComputerUseTarget {
+  readonly bundleId: string;
+  readonly applicationName: string;
+  readonly processId: number;
+  readonly windowId: string;
+  readonly windowTitle: string;
+}
+
+export interface ComputerUseGrant {
+  readonly runId: string;
+  readonly grantId: string;
+  readonly hostGeneration: string;
+  readonly targetGeneration: string;
+  readonly expiresAt: number;
+}
+
+export interface ComputerUseElement {
+  readonly ref: string;
+  readonly role: string;
+  readonly label?: string;
+  readonly enabled: boolean;
+  readonly secure: boolean;
+  readonly actions: readonly ComputerUseActionKind[];
+}
+
+export interface ComputerUseObservation {
+  readonly runId: string;
+  readonly snapshotId: string;
+  readonly targetGeneration: string;
+  readonly applicationName: string;
+  readonly bundleId: string;
+  readonly windowTitle: string;
+  readonly elements: readonly ComputerUseElement[];
+  readonly screenshot: {
+    readonly mimeType: 'image/png';
+    readonly data: string;
+  };
+}
+
+export type ComputerUseActionKind = 'press' | 'set_value' | 'scroll';
+
+export type ComputerUseAction =
+  | {
+      readonly kind: 'press';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+    }
+  | {
+      readonly kind: 'set_value';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+      readonly value: string;
+    }
+  | {
+      readonly kind: 'scroll';
+      readonly snapshotId: string;
+      readonly elementRef: string;
+      readonly direction: 'up' | 'down' | 'left' | 'right';
+      readonly amount: 'line' | 'page';
+    };
+
+export type ComputerUseFailureCode =
+  | 'busy_in_another_session'
+  | 'desktop_unavailable'
+  | 'permission_required_on_mac'
+  | 'confirmation_denied'
+  | 'confirmation_expired'
+  | 'target_invalid'
+  | 'target_lost'
+  | 'recording_failed'
+  | 'trace_failed'
+  | 'grant_expired'
+  | 'stale_request'
+  | 'outcome_uncertain'
+  | 'internal_error';
+
+export interface ComputerUseState {
+  readonly revision: number;
+  readonly phase: ComputerUsePhase;
+  readonly identity: ComputerUseSessionIdentity;
+  readonly requestId?: string;
+  readonly target?: ComputerUseTarget;
+  readonly grant?: ComputerUseGrant;
+  readonly failure?: {
+    readonly code: ComputerUseFailureCode;
+    readonly message: string;
+  };
+}
+
+export type ComputerUseStateEvent =
+  | { readonly type: 'request'; readonly requestId: string; readonly target: ComputerUseTarget }
+  | { readonly type: 'await_confirmation'; readonly requestId: string }
+  | { readonly type: 'confirm'; readonly requestId: string }
+  | { readonly type: 'deny'; readonly requestId: string }
+  | { readonly type: 'activated'; readonly requestId: string; readonly grant: ComputerUseGrant }
+  | { readonly type: 'stop' }
+  | { readonly type: 'stopped' }
+  | { readonly type: 'fail'; readonly code: ComputerUseFailureCode; readonly message: string }
+  | { readonly type: 'reset' };

package/src/types/computerUseApi.ts

@@ -0,0 +1,78 @@
+import type { DoomApiCaller } from '@agimon-ai/doompi-extension-contracts/package-api';
+
+export const API_BASE_PATH = 'computer-use';
+export const computerUseChannelType = 'computer_use_state';
+export const COMPUTER_USE_MODE_ID = 'computer-use';
+export const COMPUTER_USE_STATUS_KEY = 'doom-computer-use';
+export const COMPUTER_USE_MODE_STATUS_KEY = 'doom-computer-use-mode';
+export const SESSION_QUERY_PARAM = 'session';
+export const COMPUTER_USE_MAX_DURATION_MS = 1_800 * 1_000;
+export const COMPUTER_USE_DEFAULT_DURATION_MS = 300 * 1_000;
+export const COMPUTER_USE_CONFIRMATION_WINDOW_MS = 2 * 60 * 1_000;
+
+export function activationUrl(sessionId: string): string {
+  return `/api/plugin/${API_BASE_PATH}/activate?${SESSION_QUERY_PARAM}=${encodeURIComponent(sessionId)}`;
+}
+
+export const COMPUTER_USE_ROUTES = {
+  activate: '/activate',
+  agentState: '/agent/state',
+  agentObserve: '/agent/observe',
+  agentAction: '/agent/action',
+  agentStop: '/agent/stop',
+  hubState: '/hub/state',
+  hubActivation: '/hub/activation',
+  hubAuthorization: '/hub/authorization',
+  hubNext: '/hub/next',
+  hubComplete: '/hub/complete',
+  hubStop: '/hub/stop',
+} as const;
+
+export const COMPUTER_USE_WAKE_LIMIT = 1_000_000;
+
+export interface ComputerUseArtifactView {
+  readonly artifactId: string;
+  readonly status: 'ready' | 'failed';
+  readonly downloadUrl?: string;
+  readonly previewUrl?: string;
+  readonly actionCount?: number;
+  readonly completedAt?: string;
+}
+
+export interface ComputerUseSessionView {
+  readonly sessionId: string;
+  readonly revision: number;
+  readonly wake: number;
+  readonly phase: 'inactive' | 'awaiting_confirmation' | 'activating' | 'active' | 'stopping' | 'failed';
+  readonly requestId?: string;
+  readonly target?: Record<string, unknown>;
+  readonly durationMs?: number;
+  readonly expiresAt?: number;
+  readonly failure?: { readonly code: string; readonly message: string };
+  readonly artifact?: ComputerUseArtifactView;
+}
+
+export interface ComputerUseActivationRequest {
+  readonly requestId: string;
+  readonly target: Record<string, unknown>;
+  readonly durationSeconds: number;
+  readonly createdAt: number;
+  readonly confirmationExpiresAt: number;
+  readonly caller: DoomApiCaller;
+}
+
+export interface ComputerUseBrokerRequest {
+  readonly id: string;
+  readonly operation: 'observe' | 'act';
+  readonly payload?: unknown;
+  readonly grantId: string;
+  readonly sequence?: number;
+}
+
+export interface ComputerUseChannelPayload {
+  readonly state: ComputerUseSessionView;
+  readonly targets: readonly Record<string, unknown>[];
+  readonly busy?: true;
+}
+
+export type ComputerUseBrowserCommand = { readonly action: 'status' | 'targets' | 'stop' | 'artifacts' };

package/src/web/ComputerActionToolCard.tsx

@@ -0,0 +1,51 @@
+// @scaffold-generated
+import {
+  collapseLines,
+  MessageItem,
+  MessageItemBody,
+  MessageItemHeader,
+  MessageItemStatus,
+  MessageLines,
+  toolTone,
+} from '@agimon-ai/doompi-web-components';
+import type { ToolMessageRenderProps } from '@agimon-ai/doompi-web-contracts';
+import {
+  computerActionCallSummary,
+  computerActionCollapsedLines,
+  computerActionResultLines,
+} from './computerActionToolRender.ts';
+
+/**
+ * The computer_action tool's timeline item: Perform one constrained semantic action in the authorized application window. The shell owns
+ * the frame, the outcome tone, the status badge, and the expand toggle; this
+ * card supplies the header summary and the body lines, and says whether it
+ * hides any so the toggle appears only when it reveals something.
+ */
+export function ComputerActionToolCard({ toolName, args, result, running, isError }: ToolMessageRenderProps) {
+  const lines = computerActionResultLines(result);
+  const collapsed = collapseLines(lines, computerActionCollapsedLines, false);
+  return (
+    <MessageItem tone={toolTone({ running, isError })} expandable={collapsed.hidden > 0}>
+      {({ expanded }) => {
+        const view = expanded ? collapseLines(lines, computerActionCollapsedLines, true) : collapsed;
+        return (
+          <>
+            <MessageItemHeader title={toolName}>
+              <span data-testid={`tool-call-${toolName}`} className="min-w-0 flex-1 truncate text-doom-text">
+                {computerActionCallSummary(args)}
+              </span>
+            </MessageItemHeader>
+            {view.shown.length > 0 || running || isError ? (
+              <MessageItemBody data-testid={`tool-result-${toolName}`} className="flex flex-col gap-1">
+                {view.shown.length > 0 ? <MessageLines lines={view.shown} /> : null}
+                {view.hidden > 0 ? <MessageItemStatus expands>{view.hidden} more line(s)</MessageItemStatus> : null}
+                {running ? <MessageItemStatus tone="running">running</MessageItemStatus> : null}
+                {isError ? <MessageItemStatus tone="error">failed</MessageItemStatus> : null}
+              </MessageItemBody>
+            ) : null}
+          </>
+        );
+      }}
+    </MessageItem>
+  );
+}

package/src/web/ComputerStateToolCard.tsx

@@ -0,0 +1,51 @@
+// @scaffold-generated
+import {
+  collapseLines,
+  MessageItem,
+  MessageItemBody,
+  MessageItemHeader,
+  MessageItemStatus,
+  MessageLines,
+  toolTone,
+} from '@agimon-ai/doompi-web-components';
+import type { ToolMessageRenderProps } from '@agimon-ai/doompi-web-contracts';
+import {
+  computerStateCallSummary,
+  computerStateCollapsedLines,
+  computerStateResultLines,
+} from './computerStateToolRender.ts';
+
+/**
+ * The computer_state tool's timeline item: Observe the authorized application window and semantic accessibility state. The shell owns
+ * the frame, the outcome tone, the status badge, and the expand toggle; this
+ * card supplies the header summary and the body lines, and says whether it
+ * hides any so the toggle appears only when it reveals something.
+ */
+export function ComputerStateToolCard({ toolName, args, result, running, isError }: ToolMessageRenderProps) {
+  const lines = computerStateResultLines(result);
+  const collapsed = collapseLines(lines, computerStateCollapsedLines, false);
+  return (
+    <MessageItem tone={toolTone({ running, isError })} expandable={collapsed.hidden > 0}>
+      {({ expanded }) => {
+        const view = expanded ? collapseLines(lines, computerStateCollapsedLines, true) : collapsed;
+        return (
+          <>
+            <MessageItemHeader title={toolName}>
+              <span data-testid={`tool-call-${toolName}`} className="min-w-0 flex-1 truncate text-doom-text">
+                {computerStateCallSummary(args)}
+              </span>
+            </MessageItemHeader>
+            {view.shown.length > 0 || running || isError ? (
+              <MessageItemBody data-testid={`tool-result-${toolName}`} className="flex flex-col gap-1">
+                {view.shown.length > 0 ? <MessageLines lines={view.shown} /> : null}
+                {view.hidden > 0 ? <MessageItemStatus expands>{view.hidden} more line(s)</MessageItemStatus> : null}
+                {running ? <MessageItemStatus tone="running">running</MessageItemStatus> : null}
+                {isError ? <MessageItemStatus tone="error">failed</MessageItemStatus> : null}
+              </MessageItemBody>
+            ) : null}
+          </>
+        );
+      }}
+    </MessageItem>
+  );
+}

package/src/web/ComputerUsePanel.tsx

@@ -0,0 +1,136 @@
+import type { WebPluginSlotProps } from '@agimon-ai/doompi-web-contracts';
+import { useStore } from '@tanstack/react-store';
+import { useEffect, useState } from 'react';
+import { activationUrl, COMPUTER_USE_DEFAULT_DURATION_MS, computerUseChannelType } from '../types/computerUseApi.ts';
+import { computerUse } from './computerUseStore.ts';
+
+function text(value: unknown, fallback: string): string {
+  return typeof value === 'string' || typeof value === 'number' ? String(value) : fallback;
+}
+
+export function ComputerUsePanel({ sessionId, sendSessionFrame }: WebPluginSlotProps) {
+  const session = useStore(computerUse.store, (state) => computerUse.select(state, sessionId));
+  const [selected, setSelected] = useState(0);
+  const [confirmationTarget, setConfirmationTarget] = useState<Record<string, unknown>>();
+  const [error, setError] = useState<string>();
+  const send = (payload: Record<string, unknown>) => {
+    if (sessionId !== null) sendSessionFrame(sessionId, { type: computerUseChannelType, payload });
+  };
+
+  useEffect(() => {
+    if (sessionId === null) return;
+    sendSessionFrame(sessionId, { type: computerUseChannelType, payload: { action: 'status' } });
+    sendSessionFrame(sessionId, { type: computerUseChannelType, payload: { action: 'targets' } });
+  }, [sendSessionFrame, sessionId]);
+
+  if (sessionId === null) return <p className="px-1 text-[10px] text-doom-faint">Select a session.</p>;
+  const state = session.state;
+  const target = session.targets[selected];
+  const artifact = state.artifact;
+  const confirm = async () => {
+    if (confirmationTarget === undefined) return;
+    setError(undefined);
+    const response = await fetch(activationUrl(sessionId), {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ target: confirmationTarget, durationMs: COMPUTER_USE_DEFAULT_DURATION_MS }),
+    });
+    if (!response.ok) {
+      const value = (await response.json()) as { error?: unknown };
+      setError(typeof value.error === 'string' ? value.error : `Activation failed with HTTP ${response.status}.`);
+      return;
+    }
+    setConfirmationTarget(undefined);
+    send({ action: 'status' });
+  };
+
+  return (
+    <div data-testid="computer-use-panel" className="flex flex-col gap-2 text-[10px] text-doom-text">
+      <header className="flex items-center justify-between gap-2">
+        <span className="text-doom-faint">{state.phase.replaceAll('_', ' ')}</span>
+        <button type="button" className="rounded bg-doom-panel px-2 py-1" onClick={() => send({ action: 'targets' })}>
+          Refresh targets
+        </button>
+      </header>
+      {session.busy ? (
+        <p className="rounded border border-doom-warning px-2 py-1">Computer use is busy in another session</p>
+      ) : null}
+      {state.failure ? <p className="text-doom-error">{state.failure.message}</p> : null}
+      {error ? <p className="text-doom-error">{error}</p> : null}
+
+      {(state.phase === 'inactive' || state.phase === 'failed') && confirmationTarget === undefined ? (
+        <div className="flex flex-col gap-2">
+          <label htmlFor="computer-use-target">Application window</label>
+          <select
+            id="computer-use-target"
+            value={selected}
+            onChange={(event) => setSelected(Number(event.target.value))}
+            className="rounded bg-doom-panel px-2 py-1"
+          >
+            {session.targets.map((item, index) => (
+              <option key={text(item.windowId, String(index))} value={index}>
+                {text(item.applicationName, text(item.windowTitle, text(item.windowId, 'Window')))}
+              </option>
+            ))}
+          </select>
+          <button
+            type="button"
+            disabled={!target || Boolean(session.busy)}
+            className="rounded bg-doom-accent px-2 py-1 disabled:opacity-50"
+            onClick={() => target !== undefined && setConfirmationTarget(Object.freeze({ ...target }))}
+          >
+            Request activation
+          </button>
+        </div>
+      ) : null}
+
+      {confirmationTarget !== undefined ? (
+        <div className="rounded border border-doom-accent p-3">
+          <strong>Confirm computer control</strong>
+          <p className="my-2 text-doom-text">
+            {text(confirmationTarget.applicationName, 'Application')}:{' '}
+            {text(confirmationTarget.windowTitle, text(confirmationTarget.windowId, 'Window'))}
+          </p>
+          <p className="my-2 text-doom-faint">Confirm a 5 minute session for this exact application window.</p>
+          <div className="flex gap-2">
+            <button type="button" className="rounded bg-doom-accent px-2 py-1" onClick={() => void confirm()}>
+              Confirm
+            </button>
+            <button
+              type="button"
+              className="rounded bg-doom-panel px-2 py-1"
+              onClick={() => setConfirmationTarget(undefined)}
+            >
+              Cancel
+            </button>
+          </div>
+        </div>
+      ) : null}
+
+      {state.phase === 'awaiting_confirmation' || state.phase === 'activating' ? (
+        <p>Desktop activation is pending.</p>
+      ) : null}
+      {state.phase === 'active' || state.phase === 'stopping' ? (
+        <button
+          type="button"
+          disabled={state.phase === 'stopping'}
+          className="rounded bg-doom-panel px-2 py-1"
+          onClick={() => send({ action: 'stop' })}
+        >
+          Stop computer use
+        </button>
+      ) : null}
+      {artifact ? (
+        <div data-testid="computer-use-artifact" className="rounded bg-doom-panel p-2">
+          <strong>Completed artifact</strong>
+          <p>ID: {artifact.artifactId}</p>
+          <p>Status: {artifact.status}</p>
+          {artifact.completedAt ? <p>Completed: {artifact.completedAt}</p> : null}
+          {artifact.actionCount !== undefined ? <p>Actions: {artifact.actionCount}</p> : null}
+          {artifact.previewUrl ? <a href={artifact.previewUrl}>Preview</a> : null}
+          {artifact.downloadUrl ? <a href={artifact.downloadUrl}>Download</a> : null}
+        </div>
+      ) : null}
+    </div>
+  );
+}

package/src/web/computerActionToolRender.ts

@@ -0,0 +1,20 @@
+import type { MessageLine } from '@agimon-ai/doompi-web-components';
+import { type ToolResultView, toolResultTextLines } from '@agimon-ai/doompi-web-contracts';
+
+export const computerActionToolName = 'computer_action' as const;
+export const computerActionCollapsedLines = 8;
+
+export function computerActionCallSummary(args: Readonly<Record<string, unknown>>): string {
+  const kind = typeof args.kind === 'string' ? args.kind : '';
+  const ref = typeof args.elementRef === 'string' ? args.elementRef : '';
+  return [kind, ref].filter(Boolean).join(' ');
+}
+
+/**
+ * The body as toned lines: what the result's details say, or its text. Pure,
+ * so a unit test can pin it without a DOM; the item decides how many to show.
+ */
+export function computerActionResultLines(result: ToolResultView | null): MessageLine[] {
+  if (result === null) return [];
+  return toolResultTextLines(result.content).map((text) => ({ text, tone: 'dim' }));
+}

package/src/web/computerStateToolRender.ts

@@ -0,0 +1,18 @@
+import type { MessageLine } from '@agimon-ai/doompi-web-components';
+import { type ToolResultView, toolResultTextLines } from '@agimon-ai/doompi-web-contracts';
+
+export const computerStateToolName = 'computer_state' as const;
+export const computerStateCollapsedLines = 8;
+
+export function computerStateCallSummary(_args: Readonly<Record<string, unknown>>): string {
+  return 'authorized window';
+}
+
+/**
+ * The body as toned lines: what the result's details say, or its text. Pure,
+ * so a unit test can pin it without a DOM; the item decides how many to show.
+ */
+export function computerStateResultLines(result: ToolResultView | null): MessageLine[] {
+  if (result === null) return [];
+  return toolResultTextLines(result.content).map((text) => ({ text, tone: 'dim' }));
+}

package/src/web/computerUseStore.ts

@@ -0,0 +1,26 @@
+import { defineSessionStore } from '@agimon-ai/doompi-web-contracts';
+import { computerUseChannelType } from '../types/computerUseApi.ts';
+import type { ComputerUseChannelPayload } from '../types/computerUseApi.ts';
+
+export interface ComputerUseSession extends ComputerUseChannelPayload {}
+
+const initialState: ComputerUseSession = {
+  state: { sessionId: '', revision: 0, wake: 0, phase: 'inactive' },
+  targets: [],
+};
+
+export const computerUse = defineSessionStore<ComputerUseSession>(initialState);
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+export const computerUseChannel = computerUse.channel<ComputerUseChannelPayload>({
+  channel: computerUseChannelType,
+  parse(input) {
+    if (!isRecord(input) || !isRecord(input.state) || !Array.isArray(input.targets)) return null;
+    if (typeof input.state.phase !== 'string' || typeof input.state.revision !== 'number') return null;
+    return input as unknown as ComputerUseChannelPayload;
+  },
+  reduce: (_current, payload) => payload,
+});

package/src/web/index.ts

@@ -0,0 +1,41 @@
+import { defineWebPlugin } from '@agimon-ai/doompi-web-contracts';
+import {
+  COMPUTER_USE_MODE_ID,
+  COMPUTER_USE_MODE_STATUS_KEY,
+  COMPUTER_USE_STATUS_KEY,
+} from '../types/computerUseApi.ts';
+import { ComputerActionToolCard } from './ComputerActionToolCard.tsx';
+import { computerActionToolName } from './computerActionToolRender.ts';
+import { ComputerStateToolCard } from './ComputerStateToolCard.tsx';
+import { computerStateToolName } from './computerStateToolRender.ts';
+import { ComputerUsePanel } from './ComputerUsePanel.tsx';
+import { computerUseChannel } from './computerUseStore.ts';
+
+export const webPlugin = defineWebPlugin({
+  id: 'computer-use',
+  minorModes: [
+    {
+      name: 'computer use',
+      modeId: COMPUTER_USE_MODE_ID,
+      keys: 'c e',
+      statusKey: COMPUTER_USE_MODE_STATUS_KEY,
+      order: 70,
+    },
+  ],
+  activityGroups: [
+    {
+      name: 'computer-use',
+      keys: 'c e',
+      order: 70,
+      statusKey: COMPUTER_USE_STATUS_KEY,
+      hideWhenEmpty: true,
+      marksBackgroundWork: false,
+    },
+  ],
+  activitySections: [{ id: 'computer-use', component: ComputerUsePanel }],
+  channels: [computerUseChannel],
+  toolRenderers: [
+    { tools: [computerStateToolName], message: ComputerStateToolCard },
+    { tools: [computerActionToolName], message: ComputerActionToolCard },
+  ],
+});

package/src/web/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "lib": ["ES2023", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "jsx": "react-jsx",
+    "strict": true,
+    "noImplicitAny": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "noEmit": true,
+    "resolveJsonModule": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "allowImportingTsExtensions": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": [
+    ".",
+    "../types/computerUse.ts",
+    "../types/computerUseApi.ts",
+    "../exports/webClient.ts",
+    "../../tests/web"
+  ]
+}