@windowkit/agent-chat 0.1.0 → 0.4.0

package/src/core/types.ts

@@ -58,7 +58,14 @@ export type ToolOutput =
   | { type: 'terminal'; text: string; exitCode?: number }
   /** A file's change. `oldText` null for a new file. */
   | { type: 'diff'; path: string; oldText: string | null; newText: string }
-  | { type: 'image'; mime: string; data: string /* base64 */ }
+  /** `width` and `height` in pixels, where the harness says. */
+  | {
+      type: 'image';
+      mime: string;
+      data: string /* base64 */;
+      width?: number;
+      height?: number;
+    }
   /** What an app tool's `run` returned, as it returned it. */
   | { type: 'value'; value: unknown };
 
@@ -161,6 +168,49 @@ export interface DataPart {
   value: unknown;
 }
 
+/** `DataPart.name`s a driver of this package writes, and their values. */
+export interface DataValues {
+  /** A pull request a command opened, edited or merged. */
+  'pull-request': PullRequestData;
+  /** An agent session started elsewhere: a cloud session, a workflow. */
+  session: SessionLinkData;
+  /** A page the agent published. */
+  artifact: ArtifactData;
+  /** Files the agent sent the user. */
+  attachments: AttachmentsData;
+}
+
+export interface PullRequestData {
+  number: number;
+  url?: string;
+  /** `'created'`, `'edited'`, `'merged'`, or what else the harness says. */
+  action: string;
+}
+
+export interface SessionLinkData {
+  url: string;
+  /** What the session was started to do. */
+  title?: string;
+}
+
+export interface ArtifactData {
+  url: string;
+  title?: string;
+  /** The local file it was published from. */
+  path?: string;
+}
+
+export interface AttachmentsData {
+  caption?: string;
+  files: readonly {
+    path: string;
+    /** Bytes. */
+    size?: number;
+    mime?: string;
+    isImage: boolean;
+  }[];
+}
+
 export type Part =
   | TextPart
   | ReasoningPart

package/src/drivers/claude/index.ts

@@ -36,6 +36,7 @@ import type {
   ToolCall,
 } from '../../core/types.js';
 import { InputQueue } from './input.js';
+import { TASK_TOOLS, TaskFold, dataOf, imagesOf } from './results.js';
 import { incomingOf, originOf, peerEnvelope } from './peers.js';
 import { shapeFromJsonSchema, zodShapeOf } from './schema.js';
 import {
@@ -171,7 +172,11 @@ async function startClaude(
   }
 
   // --- tool calls, as the session sees them --------------------------------
-  const seen = new Map<string, { name: string; input: unknown }>();
+  const seen = new Map<
+    string,
+    { name: string; input: unknown; parent?: string }
+  >();
+  const tasks = new TaskFold();
   const denied = new Set<string>();
   const hidden = new Set<string>();
 
@@ -271,6 +276,9 @@ async function startClaude(
     // already the default for the models that think; `driverOptions.thinking`
     // overrides it, for an older model or for none.
     thinking: { type: 'adaptive', display: 'summarized' },
+    // a reply the user might send next, after each turn: drawn as a chip
+    // that fills the composer
+    promptSuggestions: true,
     hooks: { PreToolUse: [{ hooks: [preToolUse] }] },
     canUseTool,
     env: {
@@ -345,6 +353,11 @@ async function startClaude(
       }
       case 'result':
         return onResult(m);
+      case 'prompt_suggestion':
+        if (typeof m.suggestion === 'string') {
+          host.emit({ type: 'suggestion', text: m.suggestion });
+        }
+        return;
     }
   };
 
@@ -407,7 +420,11 @@ async function startClaude(
           const id = String(block.id);
           const name = String(block.name);
           blocks.set(index, { kind: 'tool', id });
-          seen.set(id, { name, input: undefined });
+          seen.set(id, {
+            name,
+            input: undefined,
+            ...(parent ? { parent } : {}),
+          });
           if (HIDDEN_TOOLS.has(name)) {
             hidden.add(id);
             return;
@@ -486,7 +503,11 @@ async function startClaude(
       } else if (block.type === 'tool_use') {
         const id = String(block.id);
         const name = String(block.name);
-        seen.set(id, { name, input: block.input });
+        seen.set(id, {
+          name,
+          input: block.input,
+          ...(parent ? { parent } : {}),
+        });
         if (name === 'TodoWrite') {
           hidden.add(id);
           host.emit({
@@ -519,12 +540,29 @@ async function startClaude(
   const onToolResults = (m: SdkMessage) => {
     const msg = m.message as { content?: unknown } | undefined;
     if (!Array.isArray(msg?.content)) return;
-    for (const block of msg.content as Record<string, unknown>[]) {
-      if (block.type !== 'tool_result') continue;
+    const results = (msg.content as Record<string, unknown>[]).filter(
+      (b) => b.type === 'tool_result',
+    );
+    // the tool's own output object, which is one per message: a message
+    // with several results says nothing of which it belongs to
+    const structured = results.length === 1 ? m.tool_use_result : undefined;
+    for (const block of results) {
       const id = String(block.tool_use_id);
-      if (hidden.has(id)) continue;
       const call = seen.get(id);
       const isError = block.is_error === true;
+      if (
+        call &&
+        TASK_TOOLS.has(call.name) &&
+        !isError &&
+        tasks.apply(call.name, call.input, structured)
+      ) {
+        host.emit({
+          type: 'plan',
+          entries: tasks.entries(),
+          ...(call.parent ? { parent: call.parent } : {}),
+        });
+      }
+      if (hidden.has(id)) continue;
       const text = resultText(block.content);
       const name = call?.name ?? '';
       const app = isAppTool(aliases[name] ?? name);
@@ -538,8 +576,25 @@ async function startClaude(
         state: isError ? 'error' : 'done',
         ...(isError ? { error: text } : {}),
         // an app tool's outputs are the session's, from the value `run` returned
-        ...(app ? {} : { output: outputOf(name, call?.input, text, isError) }),
+        ...(app
+          ? {}
+          : {
+              output: [
+                ...outputOf(name, call?.input, text, isError),
+                ...imagesOf(block.content, structured),
+              ],
+            }),
       });
+      if (isError || app) continue;
+      for (const d of dataOf(name, call?.input, structured)) {
+        host.emit({
+          type: 'data',
+          id: `${id}:${d.key}`,
+          name: d.name,
+          value: d.value,
+          ...(call?.parent ? { parent: call.parent } : {}),
+        });
+      }
     }
   };
 

package/src/drivers/claude/results.ts

@@ -0,0 +1,252 @@
+// What Claude Code's structured tool results say beyond their text.
+//
+// A tool's result reaches the client twice: as the `tool_result` block the
+// model reads, and as the user message's `tool_use_result`, the tool's own
+// output object (typed `unknown` by the SDK; the shapes here are its
+// `sdk-tools.d.ts`). The text is what a card shows. The object is where the
+// rest is: the task list's changes, the pull request a command opened, the
+// page the agent published, the files it sent, the pixels of an image.
+
+import type {
+  ArtifactData,
+  AttachmentsData,
+  PlanEntry,
+  PullRequestData,
+  SessionLinkData,
+  ToolOutput,
+} from '../../core/types.js';
+
+type Obj = Record<string, unknown>;
+
+const obj = (v: unknown): Obj | undefined =>
+  v && typeof v === 'object' && !Array.isArray(v) ? (v as Obj) : undefined;
+const str = (v: unknown): string | undefined =>
+  typeof v === 'string' && v !== '' ? v : undefined;
+const num = (v: unknown): number | undefined =>
+  typeof v === 'number' && Number.isFinite(v) ? v : undefined;
+
+/** The tools whose calls are the plan, not cards. */
+export const TASK_TOOLS: ReadonlySet<string> = new Set([
+  'TaskCreate',
+  'TaskUpdate',
+  'TaskList',
+  'TaskGet',
+]);
+
+type Status = PlanEntry['status'];
+const statusOf = (v: unknown): Status | undefined =>
+  v === 'pending' || v === 'in_progress' || v === 'completed' ? v : undefined;
+
+/**
+ * Claude Code's task list, as the fold of every `TaskCreate` and
+ * `TaskUpdate` this session saw, corrected by what `TaskList` and `TaskGet`
+ * report. It lives as long as the session: a task made in one turn is
+ * updated in the next.
+ */
+export class TaskFold {
+  readonly #tasks = new Map<string, { subject: string; status: Status }>();
+
+  /** Fold one finished call in. True if the list changed. */
+  apply(name: string, input: unknown, result: unknown): boolean {
+    const i = obj(input) ?? {};
+    const r = obj(result) ?? {};
+    switch (name) {
+      case 'TaskCreate': {
+        const task = obj(r.task);
+        const id = str(task?.id);
+        const subject = str(task?.subject) ?? str(i.subject);
+        if (!id || !subject) return false;
+        this.#tasks.set(id, { subject, status: 'pending' });
+        return true;
+      }
+      case 'TaskUpdate': {
+        if (r.success === false) return false;
+        const id = str(r.taskId) ?? str(i.taskId);
+        if (!id) return false;
+        if (i.status === 'deleted') return this.#tasks.delete(id);
+        const known = this.#tasks.get(id);
+        const subject = str(i.subject) ?? known?.subject;
+        // a task from before this session, named by nothing here
+        if (!subject) return false;
+        const status =
+          statusOf(obj(r.statusChange)?.to) ??
+          statusOf(i.status) ??
+          known?.status ??
+          'pending';
+        if (known && known.subject === subject && known.status === status) {
+          return false;
+        }
+        this.#tasks.set(id, { subject, status });
+        return true;
+      }
+      case 'TaskList': {
+        if (!Array.isArray(r.tasks)) return false;
+        const before = JSON.stringify(this.entries());
+        this.#tasks.clear();
+        for (const t of r.tasks as unknown[]) {
+          const task = obj(t);
+          const id = str(task?.id);
+          const subject = str(task?.subject);
+          if (id && subject) {
+            this.#tasks.set(id, {
+              subject,
+              status: statusOf(task?.status) ?? 'pending',
+            });
+          }
+        }
+        return JSON.stringify(this.entries()) !== before;
+      }
+      case 'TaskGet': {
+        const task = obj(r.task);
+        const id = str(task?.id);
+        const subject = str(task?.subject);
+        if (!id || !subject) return false;
+        const status = statusOf(task?.status) ?? 'pending';
+        const known = this.#tasks.get(id);
+        if (known && known.subject === subject && known.status === status) {
+          return false;
+        }
+        this.#tasks.set(id, { subject, status });
+        return true;
+      }
+    }
+    return false;
+  }
+
+  entries(): PlanEntry[] {
+    return [...this.#tasks.values()].map((t) => ({
+      text: t.subject,
+      status: t.status,
+    }));
+  }
+}
+
+/** A data part a finished call's result is worth, besides its card. */
+export type ResultData =
+  | { name: 'pull-request'; key: string; value: PullRequestData }
+  | { name: 'session'; key: string; value: SessionLinkData }
+  | { name: 'artifact'; key: string; value: ArtifactData }
+  | { name: 'attachments'; key: string; value: AttachmentsData };
+
+export function dataOf(
+  name: string,
+  input: unknown,
+  result: unknown,
+): ResultData[] {
+  const r = obj(result);
+  if (!r) return [];
+  const i = obj(input) ?? {};
+  switch (name) {
+    case 'Bash': {
+      const pr = obj(obj(r.gitOperation)?.pr);
+      const number = num(pr?.number);
+      if (!pr || number === undefined) return [];
+      const url = str(pr.url);
+      return [
+        {
+          name: 'pull-request',
+          key: 'pr',
+          value: {
+            number,
+            ...(url ? { url } : {}),
+            action: str(pr.action) ?? 'created',
+          },
+        },
+      ];
+    }
+    case 'Agent':
+    case 'Task':
+    case 'Workflow': {
+      const url = str(r.sessionUrl);
+      if (!url) return [];
+      const title = str(r.description) ?? str(i.description);
+      return [
+        {
+          name: 'session',
+          key: 'session',
+          value: { url, ...(title ? { title } : {}) },
+        },
+      ];
+    }
+    case 'Artifact': {
+      // a publish: list, read and the rest return other shapes
+      const url = str(r.url);
+      const path = str(r.path);
+      if (!url || !path) return [];
+      const title = str(r.title);
+      return [
+        {
+          name: 'artifact',
+          key: 'artifact',
+          value: { url, path, ...(title ? { title } : {}) },
+        },
+      ];
+    }
+    case 'SendUserFile': {
+      if (!Array.isArray(r.attachments)) return [];
+      const files = (r.attachments as unknown[]).flatMap((a) => {
+        const f = obj(a);
+        const path = str(f?.path);
+        if (!f || !path) return [];
+        const size = num(f.size);
+        const mime = str(f.media_type);
+        return [
+          {
+            path,
+            ...(size !== undefined ? { size } : {}),
+            ...(mime ? { mime } : {}),
+            isImage: f.isImage === true || !!mime?.startsWith('image/'),
+          },
+        ];
+      });
+      if (!files.length) return [];
+      const caption = str(r.caption) ?? str(i.caption);
+      return [
+        {
+          name: 'attachments',
+          key: 'files',
+          value: { files, ...(caption ? { caption } : {}) },
+        },
+      ];
+    }
+  }
+  return [];
+}
+
+/**
+ * The images a result carries: `image` blocks in the `tool_result` (an MCP
+ * tool's screenshot), or else Read's own of an image file.
+ */
+export function imagesOf(content: unknown, result: unknown): ToolOutput[] {
+  const out: ToolOutput[] = [];
+  if (Array.isArray(content)) {
+    for (const c of content as unknown[]) {
+      const block = obj(c);
+      if (block?.type !== 'image') continue;
+      const source = obj(block.source);
+      const data = str(source?.data);
+      if (source?.type !== 'base64' || !data) continue;
+      out.push({
+        type: 'image',
+        mime: str(source.media_type) ?? 'image/png',
+        data,
+      });
+    }
+  }
+  if (out.length) return out;
+  const r = obj(result);
+  const file = obj(r?.file);
+  const data = str(file?.base64);
+  if (r?.type !== 'image' || !file || !data) return [];
+  const dims = obj(file.dimensions);
+  const width = num(dims?.displayWidth) ?? num(dims?.originalWidth);
+  const height = num(dims?.displayHeight) ?? num(dims?.originalHeight);
+  return [
+    {
+      type: 'image',
+      mime: str(file.type) ?? 'image/png',
+      data,
+      ...(width !== undefined && height !== undefined ? { width, height } : {}),
+    },
+  ];
+}

package/src/drivers/claude/tools.ts

@@ -39,9 +39,15 @@ const KINDS: Readonly<Record<string, ToolKind>> = {
   ExitPlanMode: 'think',
 };
 
-/** The tools this driver draws as something other than a tool card. */
+/** The tools this driver draws as something other than a tool card: the
+ *  plan (`TodoWrite`, and the task tools Claude Code plans with now) and the
+ *  question card. */
 export const HIDDEN_TOOLS: ReadonlySet<string> = new Set([
   'TodoWrite',
+  'TaskCreate',
+  'TaskUpdate',
+  'TaskList',
+  'TaskGet',
   'AskUserQuestion',
 ]);
 

package/src/view/AgentChat.tsx

@@ -7,7 +7,13 @@
 // `onToolCall`). The top rung lifts the session out (`useAgent`, then
 // `session`) with the same options, moved rather than rewritten.
 
-import { useMemo, useRef, type ReactElement, type ReactNode } from 'react';
+import {
+  useMemo,
+  useRef,
+  useState,
+  type ReactElement,
+  type ReactNode,
+} from 'react';
 import type { StyleProp } from 'react-x11';
 
 import {
@@ -21,6 +27,7 @@ import { Composer, type ComposerOptions } from './Composer.js';
 import {
   ChatContext,
   componentMap,
+  composerChannel,
   fenceMap,
   type ChatContextValue,
 } from './context.js';
@@ -126,6 +133,7 @@ function ChatView({
   const look = useLook();
   const { fences, components, renderTool, codeActions, onLink, fontSize } =
     view;
+  const [composer] = useState(composerChannel);
   const context = useMemo<ChatContextValue>(
     () => ({
       session,
@@ -135,8 +143,18 @@ function ChatView({
       codeActions: codeActions ?? [copyCode],
       onLink,
       fontSize,
+      composer,
     }),
-    [session, fences, components, renderTool, codeActions, onLink, fontSize],
+    [
+      session,
+      fences,
+      components,
+      renderTool,
+      codeActions,
+      onLink,
+      fontSize,
+      composer,
+    ],
   );
 
   if (snap.status === 'unavailable' && view.fallback !== undefined) {

package/src/view/ArtifactCard.tsx

@@ -0,0 +1,18 @@
+// A page the agent published: the Artifact tool's `url`, `title` and the
+// file it came from.
+
+import type { ReactElement } from 'react';
+
+import type { ArtifactData } from '../core/types.js';
+import { LinkCard, shortUrl } from './LinkCard.js';
+
+export function ArtifactCard({ value }: { value: ArtifactData }): ReactElement {
+  return (
+    <LinkCard
+      kind="Published"
+      title={value.title ?? value.path ?? shortUrl(value.url)}
+      detail={[value.path, shortUrl(value.url)].filter(Boolean).join(' · ')}
+      href={value.url}
+    />
+  );
+}

package/src/view/AttachmentCard.tsx

@@ -0,0 +1,103 @@
+// Files the agent sent the user (`SendUserFile`): an image is drawn, read
+// from where the agent left it; any other file is a row with its size and,
+// where the app takes links, an Open button with its `file:` URL.
+
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import type { ReactElement } from 'react';
+import { Button } from 'react-x11';
+
+import type { AttachmentsData } from '../core/types.js';
+import { useChat } from './context.js';
+import { ImageView } from './ImageView.js';
+import { useLook } from './look.js';
+import { useSessionSnapshot } from './use-agent.js';
+
+export function AttachmentCard({
+  value,
+}: {
+  value: AttachmentsData;
+}): ReactElement {
+  const { session, onLink } = useChat();
+  const cwd = useSessionSnapshot(session).info.cwd;
+  const look = useLook();
+  return (
+    <box
+      style={{
+        flexDirection: 'column',
+        gap: look.gap,
+        padding: look.inset,
+        borderWidth: 1,
+        borderColor: look.border,
+        borderRadius: look.radius,
+      }}
+      aria-label="Attachments"
+    >
+      {value.caption ? (
+        <text style={{ color: look.text }}>{value.caption}</text>
+      ) : null}
+      {value.files.map((f, i) => {
+        const name = path.basename(f.path);
+        if (f.isImage) {
+          return (
+            <box key={i} style={{ flexDirection: 'column', gap: 2 }}>
+              <ImageView
+                source={cwd ? { path: f.path, cwd } : { path: f.path }}
+                alt={name}
+              />
+              <text style={{ color: look.muted, fontSize: look.smallSize }}>
+                {[name, sizeOf(f.size)].filter(Boolean).join(' · ')}
+              </text>
+            </box>
+          );
+        }
+        const full = cwd ? path.resolve(cwd, f.path) : f.path;
+        return (
+          <box
+            key={i}
+            style={{
+              flexDirection: 'row',
+              alignItems: 'center',
+              gap: look.gap,
+            }}
+          >
+            <text style={{ color: look.text, flexShrink: 1, maxLines: 1 }}>
+              {name}
+            </text>
+            <text
+              style={{
+                color: look.muted,
+                fontSize: look.smallSize,
+                flexGrow: 1,
+              }}
+            >
+              {sizeOf(f.size)}
+            </text>
+            {onLink && path.isAbsolute(full) ? (
+              <Button
+                size="small"
+                variant="ghost"
+                label="Open"
+                onPress={() => onLink(pathToFileURL(full).href)}
+              />
+            ) : null}
+          </box>
+        );
+      })}
+    </box>
+  );
+}
+
+/** `2 KB`, `1.4 MB`. */
+export function sizeOf(bytes: number | undefined): string {
+  if (bytes === undefined) return '';
+  if (bytes < 1024) return `${bytes} B`;
+  const units = ['KB', 'MB', 'GB'];
+  let n = bytes / 1024;
+  let u = 0;
+  while (n >= 1024 && u < units.length - 1) {
+    n /= 1024;
+    u++;
+  }
+  return `${n < 10 ? n.toFixed(1).replace(/\.0$/, '') : Math.round(n)} ${units[u]}`;
+}