experimental-a2 0.18.1 → 0.19.0

package/src/ai-control-server.ts

@@ -6,6 +6,7 @@ import type { UIMessage } from 'ai'
 import type {
   AIEventDefs,
   GenerationRequestedPayload,
+  MessageCreatedPayload,
   MessageInterruptedPayload,
 } from './ai.ts'
 import {
@@ -15,6 +16,9 @@ import {
 } from './ai-coordinator.ts'
 import {
   initialControlState,
+  STEERED_REASON,
+  steeringEnd,
+  steersOn,
   type AIControlState,
   type ControlCommand,
   type ControlReceipt,
@@ -67,7 +71,7 @@ export type AIControlRuntime<
     events: void | AppendInput<D> | readonly AppendInput<D>[]
   }): Promise<AppendInput<D>>
   cancelled(options: { error: unknown; signal: AbortSignal }): boolean
-  reducer: Reducer<D, AIControlState<M>>
+  reducer: Reducer<D, AIControlState>
   coordinator: Reducer<D, AICoordinatorState>
 }
 
@@ -81,8 +85,8 @@ export const createControlRuntime = <
   const telemetry = options.telemetry ?? CONSOLE_TELEMETRY
   const reducer = makeReducer({
     contract: options.contract,
-    name: 'a2.ai.control.v5',
-    initialState: initialControlState<M>(),
+    name: 'a2.ai.control.v6',
+    initialState: initialControlState(),
   }).fold((state, event) => {
     if (event.type === 'ai.control.requested') {
       const command = event.payload as ControlCommand<M>
@@ -141,7 +145,7 @@ export const createControlRuntime = <
           (await aiEventId('a2.ai.fact', ctx.event.id, String(facts.length))),
       }
       facts.push(fact)
-      state.coordinator = foldCoordinator<M, D>(state.coordinator, {
+      state.coordinator = foldCoordinator<D>(state.coordinator, {
         ...ctx.event,
         ...fact,
       } as ContractEvent<D>)
@@ -173,25 +177,52 @@ export const createControlRuntime = <
       })
       state.active = null
     }
+    const decide = async (): Promise<AppendInput<D>[]> => {
+      const receipt: ControlReceipt = {
+        commandId: ctx.event.id,
+        ...(ctx.event.type === 'ai.work.reported'
+          ? { work: true as const }
+          : {}),
+        outcome: reason === undefined ? 'applied' : 'rejected',
+        ...(reason === undefined ? {} : { reason }),
+      }
+      return [
+        ...(facts as AppendInput<D>[]),
+        {
+          type: 'ai.control.committed',
+          id: await aiEventId('a2.ai.control.committed', ctx.event.id),
+          payload: controlCommit({ before, after: state, moves }),
+        } as AppendInput<D>,
+        ...jobs,
+        {
+          type: 'ai.control.decided',
+          id: await aiEventId('a2.ai.control.decided', ctx.event.id),
+          payload: receipt,
+        } as AppendInput<D>,
+      ]
+    }
     const insert = ({
       message,
       generate = true,
-      first = false,
       afterStepOf,
     }: {
       message: M
       generate?: boolean
-      first?: boolean
       afterStepOf?: string
     }): void => {
       const candidate = {
         id: message.id,
-        message,
+        role: message.role,
         generate: generate && message.role === 'user',
         ...(afterStepOf === undefined ? {} : { afterStepOf }),
       }
-      if (first) state.inbox.items.unshift(candidate)
-      else state.inbox.items.push(candidate)
+      if (afterStepOf === undefined) state.inbox.items.push(candidate)
+      else
+        state.inbox.items.splice(
+          steeringEnd(state.inbox.items, afterStepOf),
+          0,
+          candidate,
+        )
     }
     const moveInput = ({ from, to }: { from: number; to: number }): void => {
       if (from === to) return
@@ -213,7 +244,7 @@ export const createControlRuntime = <
         ...request,
         control: { turnId: active.turnId, version: active.version },
       }
-      state.coordinator = foldCoordinator<M, D>(state.coordinator, {
+      state.coordinator = foldCoordinator<D>(state.coordinator, {
         ...ctx.event,
         type: 'ai.generation.requested',
         id,
@@ -247,10 +278,11 @@ export const createControlRuntime = <
               (candidate) => candidate.id === command.message.id,
             )
             if (item === undefined) reason = 'not-pending'
-            else if (item.message.role !== command.message.role)
+            else if (item.role !== command.message.role)
               reason = 'role-mismatch'
-            else item.message = command.message
-            break
+            // Its content takes effect at its receipt. An edit never runs
+            // admission, so no input's admission shares a batch with it.
+            return decide()
           }
           case 'remove': {
             const position = state.inbox.items.findIndex(
@@ -294,12 +326,18 @@ export const createControlRuntime = <
             if (!selected) reason = 'not-pending'
             else if ((state.active?.turnId ?? null) !== command.turnId)
               reason = 'stale-turn'
-            else if (selected.message.role !== 'user') reason = 'not-user-input'
+            else if (selected.role !== 'user') reason = 'not-user-input'
             else {
               selected.generate = true
               if (state.active) selected.afterStepOf = state.active.turnId
               else delete selected.afterStepOf
-              moveInput({ from: position, to: 0 })
+              const others = state.inbox.items.filter(
+                (item) => item !== selected,
+              )
+              moveInput({
+                from: position,
+                to: state.active ? steeringEnd(others, state.active.turnId) : 0,
+              })
             }
             break
           }
@@ -315,7 +353,6 @@ export const createControlRuntime = <
             insert({
               message: command.message,
               generate: true,
-              first: true,
               afterStepOf: command.turnId,
             })
             break
@@ -586,15 +623,19 @@ export const createControlRuntime = <
           !call.terminal &&
           !approvalBlocksExecution(call),
       )
+      // A paused turn keeps its steering until Resume, so every steer sent
+      // while it's paused targets it and they enter together.
       const handoffReady =
         handoff &&
         workSettled &&
         !externalPending &&
-        (active.suspended ||
-          response.failure !== undefined ||
-          response.calls.every(
-            (call) => callReadyForModel(call) || approvalBlocksExecution(call),
-          ))
+        (active.suspended
+          ? !state.inbox.paused
+          : response.failure !== undefined ||
+            response.calls.every(
+              (call) =>
+                callReadyForModel(call) || approvalBlocksExecution(call),
+            ))
       const readyForModel =
         !active.suspended &&
         response.failure === undefined &&
@@ -606,7 +647,7 @@ export const createControlRuntime = <
         response.completion !== undefined &&
         response.completion.finishReason !== 'tool-calls'
       if (handoffReady && !naturalCompletion) {
-        await end({ lastSeenIndex: undefined, reason: 'steered after step' })
+        await end({ lastSeenIndex: undefined, reason: STEERED_REASON })
       } else if (readyForModel && (!handoff || handoffReady)) {
         const resume =
           active.model !== null && active.model.version !== active.version
@@ -669,10 +710,15 @@ export const createControlRuntime = <
     ) {
       while (state.inbox.items.length > 0) {
         const item = state.inbox.items.shift()!
+        if (steersOn(item, state.inbox.items)) item.generate = false
         delete item.afterStepOf
         await emit({
           type: 'ai.message.created',
-          payload: { message: item.message, generate: item.generate },
+          payload: {
+            messageId: item.id,
+            role: item.role,
+            ...(item.generate ? {} : { generate: false as const }),
+          } satisfies MessageCreatedPayload,
           id: await aiEventId(
             'a2.ai.admitted',
             options.contract.name,
@@ -706,26 +752,7 @@ export const createControlRuntime = <
         break
       }
     }
-    const receipt: ControlReceipt = {
-      commandId: ctx.event.id,
-      ...(ctx.event.type === 'ai.work.reported' ? { work: true as const } : {}),
-      outcome: reason === undefined ? 'applied' : 'rejected',
-      ...(reason === undefined ? {} : { reason }),
-    }
-    return [
-      ...(facts as AppendInput<D>[]),
-      {
-        type: 'ai.control.committed',
-        id: await aiEventId('a2.ai.control.committed', ctx.event.id),
-        payload: controlCommit({ before, after: state, moves }),
-      } as AppendInput<D>,
-      ...jobs,
-      {
-        type: 'ai.control.decided',
-        id: await aiEventId('a2.ai.control.decided', ctx.event.id),
-        payload: receipt,
-      } as AppendInput<D>,
-    ]
+    return decide()
   }
 
   const report = async ({

package/src/ai-control-state.ts

@@ -3,7 +3,6 @@ import type {
   ControlCommit,
   ControlChange,
 } from './ai-control.ts'
-import type { UIMessage } from 'ai'
 
 type RecordValue = Record<string, unknown> | unknown[]
 const container = (value: unknown): value is RecordValue =>
@@ -131,9 +130,9 @@ export const applyControlChanges = <T>({
   return next
 }
 
-export const controlCommit = <M extends UIMessage>(options: {
-  before: AIControlState<M>
-  after: AIControlState<M>
+export const controlCommit = (options: {
+  before: AIControlState
+  after: AIControlState
   moves: ControlChange[]
 }): ControlCommit => {
   const active = options.after.active

package/src/ai-control.ts

@@ -12,13 +12,22 @@ import type {
   ToolResultPayload,
 } from './ai.ts'
 
-export type InboxItem<M extends UIMessage = UIMessage> = {
+/**
+ * An input in the queue or the active turn. Its content stays in the
+ * `ai.control.requested` command that brought it, and in no other event.
+ */
+export type InputRef = {
   id: string
-  message: M
+  role: UIMessage['role']
   generate: boolean
   afterStepOf?: string
 }
 
+/** A queued input with its content, as `AIState` shows it. */
+export type InboxItem<M extends UIMessage = UIMessage> = InputRef & {
+  message: M
+}
+
 export type ControlCommand<M extends UIMessage = UIMessage> =
   | { action: 'send'; message: M; generate?: boolean }
   | { action: 'edit'; message: M }
@@ -55,9 +64,9 @@ export type ControlReceipt = {
     | 'not-user-input'
 }
 
-export type ActiveTurn<M extends UIMessage = UIMessage> = {
+export type ActiveTurn = {
   turnId: string
-  input: InboxItem<M>
+  input: InputRef
   phase:
     | 'starting'
     | 'generating'
@@ -76,11 +85,11 @@ export type ControlWork = {
   facts?: string[]
 }
 
-export type AIControlState<M extends UIMessage = UIMessage> = {
+export type AIControlState = {
   pendingStops: Record<string, string>
-  inbox: { paused: boolean; items: InboxItem<M>[] }
+  inbox: { paused: boolean; items: InputRef[] }
   active:
-    | (ActiveTurn<M> & {
+    | (ActiveTurn & {
         version: number
         suspended: boolean
         model: ControlWork | null
@@ -107,9 +116,49 @@ export type ToolExecutionRequest = {
   generation: GenerationStartedPayload
 }
 
-export const initialControlState = <
-  M extends UIMessage,
->(): AIControlState<M> => ({
+/**
+ * The `ai.message.interrupted` reason A2 writes when steering ends a response
+ * after a step; a harness's replay continues the turn on it. Reserved, so a
+ * client's interruption can't carry it.
+ */
+export const STEERED_REASON = 'steered after step'
+
+/**
+ * A fold met an admission no command explains. Only A2's control lane writes
+ * `ai.message.created`, after the command that brought its input in.
+ */
+export const unexplainedAdmission = (messageId: string): TypeError =>
+  new TypeError(
+    `ai.message.created for ${messageId} follows no command that brought it in; only A2's control lane writes it`,
+  )
+
+/**
+ * Where new steering for a turn goes in the inbox: right after the turn's
+ * earlier steering, or first, so it hands off in the order it was sent.
+ */
+export const steeringEnd = (
+  items: readonly InputRef[],
+  turnId: string,
+): number => items.findLastIndex((item) => item.afterStepOf === turnId) + 1
+
+/**
+ * Whether an input steering a turn enters as context for later steering: the
+ * run of inputs steering one turn is admitted together, and only its last
+ * generating input starts the next turn.
+ */
+export const steersOn = (
+  item: InputRef,
+  rest: readonly InputRef[],
+): boolean => {
+  if (item.afterStepOf === undefined) return false
+  for (const next of rest) {
+    if (next.afterStepOf !== item.afterStepOf) return false
+    if (next.generate) return true
+  }
+  return false
+}
+
+export const initialControlState = (): AIControlState => ({
   pendingStops: {},
   inbox: { paused: false, items: [] },
   active: null,

package/src/ai-coordinator.ts

@@ -107,7 +107,7 @@ const updateResponse = (
     ? state
     : { ...state, response: update(state.response) }
 
-export const foldCoordinator = <M extends UIMessage, D extends EventDefs>(
+export const foldCoordinator = <D extends EventDefs>(
   state: AICoordinatorState,
   event: ContractEvent<D>,
 ): AICoordinatorState => {
@@ -120,15 +120,16 @@ export const foldCoordinator = <M extends UIMessage, D extends EventDefs>(
       return rest
     }
     case 'ai.message.created': {
-      const payload = event.payload as MessageCreatedPayload<M>
-      const message = payload.message
-      if (message.role !== 'user') return state
+      const payload = event.payload as MessageCreatedPayload
+      if (payload.role !== 'user') return state
       return {
         ...state,
         queued: [
-          ...state.queued.filter((item) => item.messageId !== message.id),
+          ...state.queued.filter(
+            (item) => item.messageId !== payload.messageId,
+          ),
           {
-            messageId: message.id,
+            messageId: payload.messageId,
             generate: payload.generate !== false,
           },
         ],

package/src/ai-fx.ts

@@ -12,7 +12,7 @@ import {
   type ToolSet,
   type UIMessageChunk,
 } from 'ai'
-import type { FxAgent, FxHostTool, FxTurn } from 'libfx'
+import type { FxAgent, FxHostTool, FxPromptBlock, FxTurn } from 'libfx'
 import {
   type Deferred,
   defineHarness,
@@ -22,6 +22,8 @@ import {
   type GatewayCredential,
   type Harness,
   type HarnessGeneration,
+  harnessImages,
+  type HarnessInput,
   type HarnessRead,
   type HarnessStep,
   type HarnessTurn,
@@ -48,6 +50,8 @@ export function fx(): Harness<'fx', never> {
 
 const SNAPSHOT_VERSION = 1
 const CATALOG_TTL_MS = 60 * 60 * 1000
+/** The Gateway model catalog libfx reads for each new kernel. */
+const CATALOG_PATH = '/coding-agent/v1/models'
 
 type StreamPart = { type: string } & Record<string, unknown>
 /** A recorded tool call, with the input the kernel reads from its text. */
@@ -62,7 +66,7 @@ type RecordedCall = {
 
 /** One FX turn to replay: its prompt, recorded calls, and how it ended. */
 type TurnPlan = {
-  prompt: string
+  prompt: string | FxPromptBlock[]
   calls: RecordedCall[]
   /** Steering to queue while serving the call at that index. */
   steers: Map<number, string[]>
@@ -188,7 +192,7 @@ async function runStep(step: HarnessStep): Promise<HarnessGeneration> {
     try {
       checkpoint = await agent!.checkpoint()
     } catch {
-      // Over libfx's bound: later steps replay from the previous snapshot.
+      // Later steps replay from the previous snapshot.
       return
     }
     await saveHarness({
@@ -206,8 +210,10 @@ async function runStep(step: HarnessStep): Promise<HarnessGeneration> {
     init: RequestInit = {},
   ): Promise<Response> => {
     const url = input instanceof Request ? input.url : String(input)
-    if ((init.method ?? 'GET').toUpperCase() === 'GET')
+    const method = (init.method ?? 'GET').toUpperCase()
+    if (method === 'GET' && new URL(url).pathname === CATALOG_PATH)
       return catalog(url, credential, withAuth(init.headers, credential))
+    if (method === 'GET') return fetch(input, init)
     const state = turn!
     if (state.failure) throw state.failure
     const body = decodeBody(init.body)
@@ -328,7 +334,8 @@ async function runStep(step: HarnessStep): Promise<HarnessGeneration> {
     return fail(state, diverged(`no recorded result for ${toolName}`))
   })
 
-  let turns = read.turns.map(turnPlan)
+  const plans = (turns: HarnessTurn[]): Promise<TurnPlan[]> =>
+    Promise.all(turns.map((recorded) => turnPlan(recorded, step.signal)))
   let results = read.results
   let controller: ReadableStreamDefaultController<UIMessageChunk> | undefined
   let closed = false
@@ -364,11 +371,10 @@ async function runStep(step: HarnessStep): Promise<HarnessGeneration> {
             if (read.saved === undefined) throw error
             // An unusable checkpoint: replay the whole log instead.
             read = await replayAll()
-            turns = read.turns.map(turnPlan)
             results = read.results
             agent = await create(undefined)
           }
-          await run(streamController, turns)
+          await run(streamController, await plans(read.turns))
           closed = true
           streamController.close()
         } catch (error) {
@@ -396,11 +402,14 @@ async function runStep(step: HarnessStep): Promise<HarnessGeneration> {
 
 // ------------------------------------------------------------ the plan
 
-function turnPlan(turn: HarnessTurn): TurnPlan {
+async function turnPlan(
+  turn: HarnessTurn,
+  signal: AbortSignal,
+): Promise<TurnPlan> {
   const calls: RecordedCall[] = []
   const steers = new Map<number, string[]>()
-  const steer = (at: number, texts: string[] | undefined) => {
-    if (texts !== undefined) steers.set(Math.max(0, at), texts)
+  const steer = (at: number, inputs: HarnessInput[] | undefined) => {
+    if (inputs !== undefined) steers.set(Math.max(0, at), inputs.map(steerText))
   }
   steer(0, turn.generations.length === 0 ? turn.steers.get(0) : undefined)
   turn.generations.forEach((generation, index) => {
@@ -408,7 +417,38 @@ function turnPlan(turn: HarnessTurn): TurnPlan {
     // A generation's step is its last call.
     steer(calls.length - 1, turn.steers.get(index))
   })
-  return { prompt: turn.prompt, calls, steers, end: turn.end }
+  const images = await harnessImages({
+    harness: 'fx()',
+    input: turn.prompt,
+    signal,
+  })
+  return {
+    prompt:
+      images.length === 0
+        ? turn.prompt.text
+        : [
+            ...images.map((image): FxPromptBlock => ({
+              type: 'image',
+              data: image.data,
+              mimeType: image.mediaType,
+            })),
+            ...(turn.prompt.text === ''
+              ? []
+              : [{ type: 'text' as const, text: turn.prompt.text }]),
+          ],
+    calls,
+    steers,
+    end: turn.end,
+  }
+}
+
+/** libfx steers with text alone, so a steer's files fail the step. */
+function steerText(input: HarnessInput): string {
+  if (input.files.length > 0)
+    throw new Error(
+      'fx() steers with text only: send files in a message of their own',
+    )
+  return input.text
 }
 
 /** A completed generation's model calls: its compactions, then its step. */
@@ -764,15 +804,13 @@ async function kernelTools(
   execute: (toolName: string, input: unknown) => unknown,
 ): Promise<FxHostTool[]> {
   return Promise.all(
-    Object.entries(tools)
-      .filter(([, tool]) => tool.type !== 'provider')
-      .map(async ([toolName, tool]) => ({
-        name: toolName,
-        description:
-          typeof tool.description === 'string' ? tool.description : toolName,
-        inputSchema: (await asSchema(tool.inputSchema).jsonSchema) as object,
-        execute: (input: unknown) => execute(toolName, input),
-      })),
+    Object.entries(tools).map(async ([toolName, tool]) => ({
+      name: toolName,
+      description:
+        typeof tool.description === 'string' ? tool.description : toolName,
+      inputSchema: (await asSchema(tool.inputSchema).jsonSchema) as object,
+      execute: (input: unknown) => execute(toolName, input),
+    })),
   )
 }