experimental-a2 0.18.1 → 0.19.0

package/examples/playground/app/a0/context-menu.tsx

@@ -1,5 +1,5 @@
 'use client'
-import type { KeyboardEvent, ReactNode } from 'react'
+import type { KeyboardEvent, ReactNode, RefObject } from 'react'
 import {
   useEffect,
   useEffectEvent,
@@ -7,27 +7,38 @@ import {
   useRef,
   useState,
 } from 'react'
+import { Check } from 'lucide-react'
 
 export type MenuItem = {
   label: string
   onSelect: () => void
   danger?: boolean
+  /** One of a set of choices; a checked one is the current choice. */
+  checked?: boolean
 }
 
+const ITEMS = '[role="menuitem"], [role="menuitemradio"]'
+
 // Keeps the menu this far inside the window's edges.
 const MARGIN = 8
 
 /**
- * A menu at a point, as a right-click opens one. It takes focus, moves with
- * the arrow keys, and closes on Escape, a press outside, or a pick; Escape
- * and a pick hand focus back through `onClose`.
+ * A menu at a point, as a right-click opens one. The point is the menu's top
+ * left corner, or its bottom right for a button near the window's foot. It
+ * takes focus, moves with the arrow keys, and closes on Escape, a press
+ * outside, or a pick; Escape and a pick hand focus back through `onClose`.
  */
 export function ContextMenu({
   at,
+  corner = 'top-left',
+  trigger,
   items,
   onClose,
 }: {
   at: { x: number; y: number }
+  corner?: 'top-left' | 'bottom-right'
+  /** The button that opens the menu; a press on it is its own, not outside. */
+  trigger?: RefObject<HTMLElement | null>
   items: MenuItem[]
   onClose: (closed: { returnFocus: boolean }) => void
 }): ReactNode {
@@ -39,19 +50,26 @@ export function ContextMenu({
     const element = menu.current
     if (!element) return
     const { width, height } = element.getBoundingClientRect()
+    const x = corner === 'top-left' ? at.x : at.x - width
+    const y = corner === 'top-left' ? at.y : at.y - height
     setPosition({
-      x: Math.max(MARGIN, Math.min(at.x, innerWidth - width - MARGIN)),
-      y: Math.max(MARGIN, Math.min(at.y, innerHeight - height - MARGIN)),
+      x: Math.max(MARGIN, Math.min(x, innerWidth - width - MARGIN)),
+      y: Math.max(MARGIN, Math.min(y, innerHeight - height - MARGIN)),
     })
-    element.querySelector<HTMLElement>('[role="menuitem"]')?.focus()
-  }, [at])
+    element.querySelector<HTMLElement>(ITEMS)?.focus()
+  }, [at, corner])
 
   const dismiss = useEffectEvent((returnFocus: boolean) =>
     onClose({ returnFocus }),
   )
   useEffect(() => {
     const outside = (event: PointerEvent): void => {
-      if (!menu.current?.contains(event.target as Node)) dismiss(false)
+      const target = event.target as Node
+      if (
+        !menu.current?.contains(target) &&
+        !trigger?.current?.contains(target)
+      )
+        dismiss(false)
     }
     const away = (): void => dismiss(false)
     document.addEventListener('pointerdown', outside, true)
@@ -66,7 +84,7 @@ export function ContextMenu({
 
   const move = (event: KeyboardEvent<HTMLDivElement>): void => {
     const buttons = [
-      ...event.currentTarget.querySelectorAll<HTMLElement>('[role="menuitem"]'),
+      ...event.currentTarget.querySelectorAll<HTMLElement>(ITEMS),
     ]
     const focused = buttons.indexOf(document.activeElement as HTMLElement)
     const next =
@@ -101,13 +119,22 @@ export function ContextMenu({
         <button
           key={item.label}
           type="button"
-          role="menuitem"
+          role={item.checked === undefined ? 'menuitem' : 'menuitemradio'}
+          aria-checked={item.checked}
           data-danger={item.danger || undefined}
           onClick={() => {
             onClose({ returnFocus: true })
             item.onSelect()
           }}
         >
+          {item.checked !== undefined && (
+            <Check
+              size={12}
+              className="a0-menu-check"
+              visibility={item.checked ? 'visible' : 'hidden'}
+              aria-hidden
+            />
+          )}
           {item.label}
         </button>
       ))}

package/examples/playground/app/a0/model.test.ts

@@ -71,3 +71,31 @@ it("reads the app's harness from its URL, the AI SDK's by default", () => {
     'a0-pi',
   ])
 })
+
+const sendWithFile = (url: string) =>
+  appAgent.events['ai.control.requested']['~standard'].validate({
+    action: 'send',
+    message: {
+      id: 'u1',
+      role: 'user',
+      parts: [
+        { type: 'file', mediaType: 'image/png', url },
+        { type: 'text', text: 'Like this.' },
+      ],
+    },
+  })
+
+it("only takes a user's files from the playground's Blob store", () => {
+  expect(
+    sendWithFile(
+      'https://store.public.blob.vercel-storage.com/a0/app/shot-x1.png',
+    ),
+  ).not.toHaveProperty('issues')
+  for (const url of [
+    'data:image/png;base64,iVBORw0KGgo=',
+    'http://store.public.blob.vercel-storage.com/shot.png',
+    'http://169.254.169.254/latest/meta-data',
+    'https://example.com/shot.png',
+  ])
+    expect(sendWithFile(url)).toHaveProperty('issues')
+})

package/examples/playground/app/a0/model.ts

@@ -35,6 +35,35 @@ export const defaultModelChoice: ModelChoice = {
 
 export type AppMessage = UIMessage
 
+/** A file the user attached, uploaded to the playground's public Blob store. */
+export function isAttachmentUrl(value: string): boolean {
+  try {
+    const { protocol, hostname } = new URL(value)
+    return (
+      protocol === 'https:' &&
+      hostname.endsWith('.public.blob.vercel-storage.com')
+    )
+  } catch {
+    return false
+  }
+}
+
+/**
+ * A user's files must be attachments. Whatever hands them to the model
+ * fetches them by URL, so the URL can't point anywhere else, and a message
+ * never holds a file's bytes.
+ */
+const messageSchema = z
+  .custom<AppMessage>()
+  .refine(
+    (message) =>
+      message.role !== 'user' ||
+      message.parts.every(
+        (part) => part.type !== 'file' || isAttachmentUrl(part.url),
+      ),
+    "A message's files must be uploaded attachments",
+  )
+
 /**
  * The loop an app's agent runs: the AI SDK's, FX's, or Pi's. An
  * app picks it before its first message and keeps it.
@@ -56,6 +85,7 @@ export const harnessFrom = (value: unknown): Harness =>
 const appContract = (name: string) =>
   agentContract({
     name,
+    messageSchema,
     events: { 'model.chosen': modelChoiceSchema },
     reducer: {
       name: 'a0.model-choice.v1',

package/examples/playground/app/a0/server.ts

@@ -5,6 +5,7 @@ import { handlerContext } from 'experimental-a2/ai'
 import { fx } from 'experimental-a2/ai/fx'
 import { pi } from 'experimental-a2/ai/pi'
 import { agent } from 'experimental-a2/ai/server'
+import { otel } from 'experimental-a2/otel'
 import { createServer } from 'experimental-a2/server'
 import { vercelQueues } from 'experimental-a2/scheduler-vercel'
 import { z } from 'zod'
@@ -33,6 +34,7 @@ const instructions = (
 - If the preview stops answering, read /tmp/next.log, then start next dev with \`${START_DEV}\`; it does nothing while one is already running.
 - next.config allows the preview's *.vercel.run host (allowedDevOrigins); keep that when you edit it.
 - Background anything long-running; a bash call stops after its timeout.
+- The user can attach files. Images come with their message; any other file is a "name: URL" line in it. Download one with curl when you need its contents.
 - Keep replies short and in plain text, without Markdown: what you changed.`
 
 /** A tool's result, with how long it took; the chat shows it. */
@@ -118,10 +120,13 @@ function chosenModel({
   return { model, reasoning, speed: fast ? 'fast' : 'standard' } as const
 }
 
+const telemetry = otel()
+
 const shared = (harness: Harness) => ({
   ...(store ? { store } : {}),
   ...(appSchedulers[harness] ? { scheduler: appSchedulers[harness] } : {}),
   contract: harnessAgents[harness],
+  telemetry,
   maxSteps: 80,
 })
 

package/examples/playground/app/a0/uploads/route.ts

@@ -0,0 +1,29 @@
+import { handleUpload, type HandleUploadBody } from '@vercel/blob/client'
+import { isAppId } from '../apps'
+
+/** Tokens for the browser to upload an app's attachments straight to Blob. */
+export async function POST(request: Request): Promise<Response> {
+  const body = (await request.json()) as HandleUploadBody
+  try {
+    return Response.json(
+      await handleUpload({
+        request,
+        body,
+        onBeforeGenerateToken: async (pathname, appId) => {
+          if (
+            appId === null ||
+            !isAppId(appId) ||
+            !pathname.startsWith(`a0/${appId}/`)
+          )
+            throw new Error("An attachment goes in its app's folder")
+          return { addRandomSuffix: true }
+        },
+      }),
+    )
+  } catch (cause) {
+    return Response.json(
+      { error: cause instanceof Error ? cause.message : String(cause) },
+      { status: 400 },
+    )
+  }
+}

package/examples/playground/app/agent/[agentId]/agent-client.tsx

@@ -393,10 +393,11 @@ export function AgentClient({
             >
               Stop
             </button>
+            {/* With no turn to steer, steering would only send. */}
             <button
               type="button"
               className="secondary"
-              disabled={!canSend || !prompt.trim()}
+              disabled={!canSend || !prompt.trim() || !state.active}
               onClick={() => void send(undefined, true)}
             >
               Steer

package/examples/playground/app/agent/[agentId]/agent-queue.test.tsx

@@ -14,6 +14,7 @@ const queue = (edit: Edit) =>
       items={[
         {
           id: 'queued',
+          role: 'user',
           message: {
             id: 'queued',
             role: 'user',

package/examples/playground/app/agent/model.test.ts

@@ -29,9 +29,18 @@ it('folds compaction settings beside the AI state', () => {
   )
   state = reducer.fold(
     state,
-    event(2, 'ai.message.created', {
-      message: { id: 'user', role: 'user', parts: [] },
-    }),
+    // A send's id is its message's.
+    {
+      ...event(2, 'ai.control.requested', {
+        action: 'send',
+        message: { id: 'user', role: 'user', parts: [] },
+      }),
+      id: 'user',
+    },
+  )
+  state = reducer.fold(
+    state,
+    event(3, 'ai.message.created', { messageId: 'user', role: 'user' }),
   )
   expect(state.compactionSettings).toEqual({ mode: 'disabled' })
   expect(state.compactionSettingsId).toBe('event-1')

package/examples/playground/app/chat/[chatId]/agent-state.test.ts

@@ -19,11 +19,7 @@ function running() {
     active: {
       turnId: 'turn',
       phase: 'generating',
-      input: {
-        id: 'input',
-        generate: true,
-        message: { id: 'input', role: 'user', parts: [] },
-      },
+      input: { id: 'input', role: 'user', generate: true },
     },
   }
   const append = (event: { type: string; payload: unknown; id?: string }) => {

package/examples/playground/next.config.ts

@@ -6,13 +6,16 @@ import type { NextConfig } from 'next'
 
 const nextConfig: NextConfig = {
   // libfx loads a native binary for the platform, falling back to its
-  // WebAssembly, by paths the bundler cannot follow; Node loads both from
-  // node_modules at runtime.
-  serverExternalPackages: ['libfx'],
+  // WebAssembly, and photon, which Pi resizes images with, loads its
+  // WebAssembly, by paths the bundler cannot follow; Node loads them from
+  // node_modules at runtime. Pi stays bundled (externalized, it makes
+  // Turbopack panic on its WebAssembly), so it finds photon through this
+  // app: package.json depends on photon at the version Pi pins, and the two
+  // move together.
+  serverExternalPackages: ['libfx', '@silvia-odwyer/photon-node'],
   turbopack: {
     // WebAssembly Pi's code reaches that the bundler cannot trace.
     resolveAlias: {
-      '@silvia-odwyer/photon-node': './app/a0/no-photon.cjs',
       'quickjs-wasi': './app/a0/no-quickjs.cjs',
       'quickjs-wasi/quickjs.wasm': './app/a0/no-quickjs.cjs',
     },

package/examples/playground/package.json

@@ -22,13 +22,15 @@
     "@earendil-works/pi-ai": "1.0.3",
     "@earendil-works/pi-coding-agent": "1.0.2",
     "@opentelemetry/api": "^1.9.0",
+    "@silvia-odwyer/photon-node": "0.3.4",
+    "@vercel/blob": "^2.8.0",
     "@vercel/functions": "^3.9.3",
     "@vercel/otel": "^2.1.0",
     "@vercel/queue": "^0.2.0",
     "@vercel/sandbox": "^3.0.0",
     "ai": "^7.0.58",
     "codemirror": "^6.0.2",
-    "experimental-a2": "0.18.1",
+    "experimental-a2": "0.19.0",
     "geist": "^1.7.2",
     "ioredis": "^5.9.0",
     "libfx": "0.0.12",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "experimental-a2",
-  "version": "0.18.1",
+  "version": "0.19.0",
   "description": "Syncs your immutable event log, from server to browser: one contract, handlers, derived state, and a live client per session.",
   "license": "MIT",
   "type": "module",

package/src/ai-agent.ts

@@ -1282,6 +1282,10 @@ export function createHandlers<
         throw new TypeError(
           `a harness runs tools without approval; '${name}' sets needsApproval`,
         )
+      if (tool.type === 'provider')
+        throw new TypeError(
+          `a harness runs only tools A2 executes; '${name}' is provider-defined`,
+        )
     }
   }
 

package/src/ai-client-state.ts

@@ -1,6 +1,11 @@
 import type { UIMessage } from 'ai'
 import type { AIState } from './ai.ts'
-import type { ControlCommand, InboxItem } from './ai-control.ts'
+import {
+  steeringEnd,
+  steersOn,
+  type ControlCommand,
+  type InboxItem,
+} from './ai-control.ts'
 import {
   interruptGenerationProjection,
   settleAbandonedCompaction,
@@ -92,10 +97,10 @@ export function projectAIClientState<M extends UIMessage>(
   const admit = (): void => {
     if (turnId !== null || paused || items.length === 0) return
     let count = 0
-    for (const item of items) {
+    for (const [index, item] of items.entries()) {
       count += 1
       show(item.message)
-      if (item.generate) {
+      if (item.generate && !steersOn(item, items.slice(index + 1))) {
         turnId = item.id
         starting = true
         suspended = false
@@ -118,18 +123,21 @@ export function projectAIClientState<M extends UIMessage>(
             ? { afterStepOf: command.turnId }
             : {}),
           id: command.message.id,
+          role: command.message.role,
           message: command.message,
           generate:
             command.message.role === 'user' &&
             (command.action === 'steer' || command.generate !== false),
         }
-        items = command.action === 'steer' ? [item, ...items] : [...items, item]
+        if (command.action === 'steer') {
+          const at = steeringEnd(items, command.turnId)
+          items = [...items.slice(0, at), item, ...items.slice(at)]
+        } else items = [...items, item]
         break
       }
       case 'edit':
         items = items.map((item) =>
-          item.id === command.message.id &&
-          item.message.role === command.message.role
+          item.id === command.message.id && item.role === command.message.role
             ? { ...item, message: command.message }
             : item,
         )
@@ -147,20 +155,24 @@ export function projectAIClientState<M extends UIMessage>(
       case 'send-now': {
         if (turnId !== command.turnId) break
         const selected = items.find((item) => item.id === command.inputId)
-        if (!selected || selected.message.role !== 'user') break
+        if (!selected || selected.role !== 'user') break
+        const { afterStepOf: _, ...promoted } = selected
         items = items.map((item) =>
           item.id === selected.id
             ? {
-                ...item,
+                ...promoted,
                 generate: true,
                 ...(turnId === null ? {} : { afterStepOf: turnId }),
               }
             : item,
         )
+        const others = items.filter((item) => item.id !== selected.id)
         items = moveItem({
           items,
           inputId: command.inputId,
-          beforeId: items[0]?.id ?? null,
+          beforeId:
+            others[turnId === null ? 0 : steeringEnd(others, turnId)]?.id ??
+            null,
         })
         break
       }

package/src/ai-context-schema.ts

@@ -1,6 +1,7 @@
 import { assertMessageBase, restoreMessageBase } from './ai-message-base.ts'
 import type { UIMessage } from 'ai'
 import { events } from './ai.ts'
+import { isUIMessage } from './ai-message-shape.ts'
 import type { AIContextState } from './ai-context.ts'
 import type { StandardSchemaV1 } from './standard-schema.ts'
 import { validateSync } from './validate.ts'
@@ -14,13 +15,34 @@ const id = (value: unknown): value is string =>
 const nullableId = (value: unknown): boolean => value === null || id(value)
 const matches = (schema: StandardSchemaV1<unknown>, value: unknown): boolean =>
   validateSync(schema, value, 'AI context checkpoint').issues === undefined
-const isMessage = (value: unknown): value is UIMessage =>
-  matches(events['ai.message.created'], { message: value })
 const indexes = (value: unknown, messages: readonly UIMessage[]): boolean =>
   record(value) &&
   Object.values(value).every(index) &&
   messages.every((message) => Object.hasOwn(value, message.id))
 
+const inputs = (value: unknown): boolean =>
+  record(value) &&
+  Object.keys(value).length === 2 &&
+  Array.isArray(value['commands']) &&
+  value['commands'].every(
+    (held) =>
+      record(held) &&
+      Object.keys(held).every((key) =>
+        ['commandId', 'inputId', 'message'].includes(key),
+      ) &&
+      id(held['commandId']) &&
+      id(held['inputId']) &&
+      (held['message'] === undefined || isUIMessage(held['message'])),
+  ) &&
+  Array.isArray(value['waiting']) &&
+  value['waiting'].every(
+    (input) =>
+      record(input) &&
+      Object.keys(input).length === 2 &&
+      id(input['inputId']) &&
+      isUIMessage(input['message']),
+  )
+
 const compaction = (value: unknown): boolean =>
   value === null ||
   (record(value) &&
@@ -109,7 +131,7 @@ const restorableBase = (options: {
   current: UIMessage | undefined
 }): boolean => {
   assertMessageBase(options.base)
-  return isMessage(
+  return isUIMessage(
     restoreMessageBase({ current: options.current, base: options.base }),
   )
 }
@@ -230,6 +252,7 @@ export const contextSchema = <M extends UIMessage>(
         'compaction',
         'modelMetadata',
         'closed',
+        'inputs',
         'authority',
         'generation',
         'activeProjection',
@@ -249,9 +272,10 @@ export const contextSchema = <M extends UIMessage>(
         const current = value['activeProjection']
         if (
           !Array.isArray(messages) ||
-          !messages.every(isMessage) ||
+          !messages.every(isUIMessage) ||
           !compaction(value['compaction']) ||
           typeof value['closed'] !== 'boolean' ||
+          !inputs(value['inputs']) ||
           !authority(value['authority']) ||
           !record(metadata) ||
           !Object.entries(metadata).every(

package/src/ai-context.ts

@@ -19,7 +19,12 @@ import type {
   ToolCalledPayload,
   ToolResultPayload,
 } from './ai.ts'
-import type { ControlCommit } from './ai-control.ts'
+import {
+  type ControlCommand,
+  type ControlCommit,
+  type ControlReceipt,
+  unexplainedAdmission,
+} from './ai-control.ts'
 import { contextSchema } from './ai-context-schema.ts'
 import type { Contract, ContractEvent, EventDefs } from './contract.ts'
 import {
@@ -49,8 +54,22 @@ export type AIModelContext<M extends UIMessage = UIMessage> = {
   modelMetadata: Record<string, ModelMetadataState>
 }
 
+/**
+ * Inputs' content between their command and their admission: each input's
+ * content is stored only in the command that brought it in. Lists, not
+ * records: clients choose the ids, and a record would read an id such as
+ * `constructor` or `__proto__` from its prototype.
+ */
+export type AIContextInputs<M extends UIMessage> = {
+  /** Undecided input commands; a remove holds no message. */
+  commands: { commandId: string; inputId: string; message?: M }[]
+  /** Queued inputs' current content. */
+  waiting: { inputId: string; message: M }[]
+}
+
 export type AIContextState<M extends UIMessage> = AIModelContext<M> & {
   closed: boolean
+  inputs: AIContextInputs<M>
   authority: Pick<
     ControlCommit['view'],
     | 'requestId'
@@ -211,6 +230,7 @@ export const createContextReducer = <M extends UIMessage, D extends EventDefs>({
     compaction: null,
     modelMetadata: {},
     closed: false,
+    inputs: { commands: [], waiting: [] },
     authority: null,
     generation: null,
     activeProjection: null,
@@ -222,7 +242,7 @@ export const createContextReducer = <M extends UIMessage, D extends EventDefs>({
     // The application state rides in this checkpoint so every generation
     // reads it without another state read.
     name:
-      app === undefined ? 'a2.ai.context.v6' : `${app.name}+a2.ai.context.v6`,
+      app === undefined ? 'a2.ai.context.v7' : `${app.name}+a2.ai.context.v7`,
     initialState,
     stateSchema: contextSchema<M>(app),
   }).fold((state, event) => {
@@ -265,19 +285,79 @@ const reduceContext = <M extends UIMessage, D extends EventDefs>(
       }
       return view.closed ? endGeneration(next) : next
     }
+    case 'ai.control.requested': {
+      const command = event.payload as ControlCommand<M>
+      const held =
+        command.action === 'send' ||
+        command.action === 'steer' ||
+        command.action === 'edit'
+          ? {
+              commandId: event.id,
+              inputId: command.message.id,
+              message: command.message,
+            }
+          : command.action === 'remove'
+            ? { commandId: event.id, inputId: command.inputId }
+            : undefined
+      if (held === undefined) return state
+      return {
+        ...state,
+        inputs: {
+          ...state.inputs,
+          commands: [...state.inputs.commands, held],
+        },
+      }
+    }
+    case 'ai.control.decided': {
+      const receipt = event.payload as ControlReceipt
+      const held = receipt.work
+        ? undefined
+        : state.inputs.commands.find(
+            (command) => command.commandId === receipt.commandId,
+          )
+      if (held === undefined) return state
+      const commands = state.inputs.commands.filter(
+        (command) => command !== held,
+      )
+      if (receipt.outcome !== 'applied')
+        return { ...state, inputs: { ...state.inputs, commands } }
+      const waiting = state.inputs.waiting.filter(
+        (input) => input.inputId !== held.inputId,
+      )
+      if (held.message !== undefined)
+        waiting.push({ inputId: held.inputId, message: held.message })
+      return { ...state, inputs: { commands, waiting } }
+    }
     case 'ai.session.closed':
       return endGeneration({
         ...state,
         closed: true,
+        inputs: { commands: [], waiting: [] },
         compaction:
           state.compaction?.status === 'running'
             ? (state.activeProjection?.baseCompaction ?? null)
             : state.compaction,
       })
     case 'ai.message.created': {
-      const { message } = event.payload as MessageCreatedPayload<M>
+      const { messageId } = event.payload as MessageCreatedPayload
+      // A send or steer admitted with its own receipt is still undecided
+      // here; its command id is the input's.
+      const message =
+        state.inputs.waiting.find((input) => input.inputId === messageId)
+          ?.message ??
+        state.inputs.commands.find((command) => command.commandId === messageId)
+          ?.message
+      if (message === undefined) throw unexplainedAdmission(messageId)
       return {
         ...state,
+        inputs: {
+          commands: state.inputs.commands.filter(
+            (command) => command.commandId !== messageId,
+          ),
+          waiting: state.inputs.waiting.filter(
+            (input) => input.inputId !== messageId,
+          ),
+        },
         messages: upsertMessage(state.messages, message),
         activeProjection:
           state.activeProjection?.responseMessageId === message.id