broapp 0.2.0 → 0.3.0

package/src/ai/host/run.ts

@@ -17,24 +17,54 @@ import { jsonSchema, stepCountIs, streamText, tool } from 'ai';
 import type { ModelMessage, ToolSet } from 'ai';
 
 import type { HostLogger, StreamSink } from '../../host/app.ts';
-import { fromTransportError, PublicError } from '../../shared/errors.ts';
+import type { PendingApprovals } from '../../host/approvals.ts';
+import type { ApprovalQuestion, Approver } from '../../host/gate.ts';
+import type { Effect } from '../../shared/contract.ts';
+import { fromTransportError, isPublicError, publicError } from '../../shared/errors.ts';
+import type { ToolPermission } from '../shared/types.ts';
 import type { ChatEvent, StreamChatParams } from './run-types.ts';
 
 import { AdapterError } from './adapter.ts';
+import type { AdapterConfig, ProviderAdapter } from './adapter.ts';
 import type { Registry } from './registry.ts';
-import type { AiContextProviders, AiTool, Confirmations, ContextDocument } from './tool.ts';
+import type { AiContextProviders, AiTool, ContextDocument } from './tool.ts';
 
 /** What the run loop needs from the `Ai` that owns it. */
 export interface RunDeps {
   readonly registry: Registry;
-  readonly app: { readonly name: string; readonly purpose: string; readonly terminology?: readonly string[] };
+  readonly app: {
+    readonly name: string;
+    readonly purpose: string;
+    readonly terminology?: readonly string[];
+    readonly instructions?: string;
+  };
   readonly context: AiContextProviders;
   readonly tools: Record<string, AiTool>;
   readonly contextBudgetChars: number;
   readonly maxSteps: number;
   readonly confirmTimeoutMs: number;
-  readonly confirmations: Confirmations;
+  readonly approvals: PendingApprovals;
   readonly logger: HostLogger;
+  /**
+   * Called once when a turn ends, however it ends.
+   *
+   * The run identifier is chosen by the browser and used as the prefix of every
+   * request identifier the turn produces, so this is what lets something
+   * outside the AI layer — Autoapp's run store — close the record the gate has
+   * been writing steps into.
+   */
+  readonly onRunEnd?: (runId: string, status: 'succeeded' | 'failed' | 'cancelled', summary: string) => void;
+}
+
+/**
+ * What the browser is told about a tool before it runs.
+ *
+ * The browser's vocabulary is still `read` and `confirm`, because that is what
+ * it shows a person; the gate's vocabulary is the effect. The mapping is here,
+ * in one place, so the two never drift into meaning different things.
+ */
+function permissionOf(effect: Effect): ToolPermission {
+  return effect === 'read' ? 'read' : 'confirm';
 }
 
 /** How many records a search may contribute to one turn. */
@@ -103,6 +133,12 @@ export function buildSystemPrompt(deps: RunDeps, documents: readonly ContextDocu
     `You are the assistant built into "${deps.app.name}". ${deps.app.purpose}`,
   ];
   if (terms.length > 0) lines.push(`Terms used in this application: ${terms.join(', ')}`);
+  // Verbatim, and before the rules: an application that needs standing
+  // instructions needs them read as part of what it is, not as an afterthought
+  // among the documents.
+  if (deps.app.instructions !== undefined && deps.app.instructions !== '') {
+    lines.push('', deps.app.instructions);
+  }
   lines.push(
     '',
     '# Rules',
@@ -147,16 +183,72 @@ async function assembleContext(
   return fitToBudget(documents, deps.contextBudgetChars);
 }
 
-/** A message the model may see, without whatever a caller invented. */
+/**
+ * A message the model may see, without whatever a caller invented.
+ *
+ * Images ride on the message they arrived with and nowhere else. History turns
+ * are strings by contract, so an earlier turn's picture is already a
+ * `[image: name]` line the browser put there — the alternative, resending
+ * every image on every turn, would cost the user the same upload again on each
+ * question.
+ */
 function toModelMessages(params: StreamChatParams): ModelMessage[] {
   const messages: ModelMessage[] = params.history.map((turn) => ({
     role: turn.role,
     content: turn.content,
   }));
-  messages.push({ role: 'user', content: params.message });
+  const files = params.files ?? [];
+  if (files.length === 0) {
+    messages.push({ role: 'user', content: params.message });
+    return messages;
+  }
+  messages.push({
+    role: 'user',
+    content: [
+      { type: 'text', text: params.message },
+      // `data` is a base64 string, which `FilePart` accepts as `DataContent`.
+      ...files.map((file) => ({
+        type: 'file' as const,
+        mediaType: file.mediaType,
+        data: file.data,
+        filename: file.name,
+      })),
+    ],
+  });
   return messages;
 }
 
+/** Base64 characters allowed across every image on one message. */
+const MAX_FILE_CHARS_PER_TURN = 6_000_000;
+
+/**
+ * Whether the model chosen in Settings can read an image.
+ *
+ * The capability is on the adapter's model list, which is fetched from the
+ * provider — so this asks for that list once per turn, and only when the turn
+ * actually carries an image. A model the list does not mention is assumed to
+ * see: a custom server's list is often incomplete, and a provider that cannot
+ * read the image will say so far more precisely than a guess here would.
+ */
+async function modelCanSee(
+  resolved: { adapter: ProviderAdapter; config: AdapterConfig; modelId: string },
+  signal: AbortSignal,
+  logger: HostLogger,
+): Promise<boolean> {
+  try {
+    const models = await resolved.adapter.models(resolved.config, signal);
+    const found = models.find((model) => model.modelId === resolved.modelId);
+    return found === undefined ? true : found.capabilities.vision;
+  } catch (cause) {
+    // A listing that failed says nothing about the model. Refusing here would
+    // turn a provider hiccup into "your model cannot see", which is a lie.
+    logger.warn(
+      `[broapp] ai could not list models to check vision: ${String(cause instanceof Error ? cause.message : cause)}`,
+    );
+    return true;
+  }
+}
+
 /**
  * A message safe to show a user.
  *
@@ -165,15 +257,82 @@ function toModelMessages(params: StreamChatParams): ModelMessage[] {
  * URL, or an echo of the prompt.
  */
 function safeMessage(cause: unknown, logger: HostLogger): string {
-  if (cause instanceof AdapterError || cause instanceof PublicError) return cause.message;
+  if (cause instanceof AdapterError || isPublicError(cause)) return cause.message;
   logger.error(
     `[broapp] ai.chat provider error: ${String(cause instanceof Error ? (cause.stack ?? cause.message) : cause)}`,
   );
   return 'The AI provider returned an error.';
 }
 
-/** Build the AI SDK tool set, wrapping each tool in the permission dance. */
-function buildTools(params: StreamChatParams, deps: RunDeps, sink: StreamSink<ChatEvent>): ToolSet {
+/**
+ * The approver for one run.
+ *
+ * The gate decides that a person has to be asked; this is how the asking
+ * reaches them. The `confirm` event goes out on the same stream the browser is
+ * already watching, and the answer comes back on `ai.chatConfirm`, which hands
+ * it to the same approval table. The run's own deadline is applied here rather
+ * than left to the gate's, because how long a chat turn should wait for a
+ * click is a property of the chat, not of the application.
+ */
+function createRunApprover(
+  deps: RunDeps,
+  sink: StreamSink<ChatEvent>,
+  callIdOf: (requestId: string) => string,
+): Approver {
+  return {
+    async ask(question: ApprovalQuestion, signal: AbortSignal): Promise<boolean> {
+      await sink.emit({
+        type: 'confirm',
+        callId: callIdOf(question.requestId),
+        tool: question.route,
+        input: question.input,
+        requestId: question.requestId,
+        releaseId: question.releaseId,
+        argumentsHash: question.argumentsHash,
+        // The narrower of the gate's window and the turn's, because the turn's
+        // is what actually stops the waiting below.
+        expiresAt: Math.min(question.expiresAt, Date.now() + deps.confirmTimeoutMs),
+      });
+      // A question nobody answers is a denial. The gate has a deadline of its
+      // own, but it belongs to the application; this one belongs to the turn.
+      const waiting = new AbortController();
+      const timer = setTimeout(
+        () => waiting.abort(new Error('the question timed out')),
+        deps.confirmTimeoutMs,
+      );
+      const relay = (): void => waiting.abort(new Error('the run was cancelled'));
+      signal.addEventListener('abort', relay, { once: true });
+      if (signal.aborted) relay();
+      try {
+        return await deps.approvals.ask(question, waiting.signal);
+      } finally {
+        clearTimeout(timer);
+        signal.removeEventListener('abort', relay);
+      }
+    },
+  };
+}
+
+/**
+ * True when a tool call failed because nobody allowed it.
+ *
+ * The gate throws a `PublicError` with code `rejected`; a tool that reached it
+ * through `HostApp.invoke` has had that turned into the marked bridge error the
+ * browser would have seen. Both are the same answer — the user said no — and
+ * both have to become an ordinary tool result rather than a failure.
+ */
+function wasDeclined(cause: unknown): boolean {
+  if (isPublicError(cause)) return cause.code === 'rejected';
+  return fromTransportError(cause).code === 'rejected';
+}
+
+/** Build the AI SDK tool set, each call carrying the run's envelope to the gate. */
+function buildTools(
+  params: StreamChatParams,
+  deps: RunDeps,
+  sink: StreamSink<ChatEvent>,
+  approver: Approver,
+): ToolSet {
   const tools: ToolSet = {};
   for (const [name, definition] of Object.entries(deps.tools)) {
     tools[name] = tool({
@@ -186,18 +345,27 @@ function buildTools(params: StreamChatParams, deps: RunDeps, sink: StreamSink<Ch
           callId,
           tool: name,
           input,
-          permission: definition.permission,
+          permission: permissionOf(definition.effect),
         });
 
-        if (definition.permission === 'confirm') {
-          await sink.emit({ type: 'confirm', callId, tool: name, input });
-          const approved = await deps.confirmations.wait(
-            params.runId,
-            callId,
-            deps.confirmTimeoutMs,
+        let output: unknown;
+        try {
+          // The envelope is built here, from what the run loop knows. Nothing
+          // the model produced is read when it is filled in, which is what
+          // stops a model from calling a tool as the user.
+          output = await definition.execute(
+            input,
+            {
+              requestId: `${params.runId}:${callId}`,
+              channel: 'ai',
+              caller: `ai:${params.runId}`,
+              signal: sink.signal,
+              approver,
+            },
             sink.signal,
           );
-          if (!approved) {
+        } catch (cause) {
+          if (wasDeclined(cause)) {
             // A refusal is an ordinary result, not a failure: the model has to
             // be told, so it can say something rather than retry.
             await sink.emit({
@@ -209,12 +377,6 @@ function buildTools(params: StreamChatParams, deps: RunDeps, sink: StreamSink<Ch
             });
             return DECLINED;
           }
-        }
-
-        let output: unknown;
-        try {
-          output = await definition.execute(input, sink.signal);
-        } catch (cause) {
           // One tool failing is not the turn failing. The model gets the
           // reason and can carry on or explain.
           output = { error: safeToolMessage(cause, name, deps.logger) };
@@ -237,7 +399,7 @@ function buildTools(params: StreamChatParams, deps: RunDeps, sink: StreamSink<Ch
  * failure and is logged rather than shown.
  */
 function safeToolMessage(cause: unknown, name: string, logger: HostLogger): string {
-  if (cause instanceof PublicError) return cause.message;
+  if (isPublicError(cause)) return cause.message;
   const reduced = fromTransportError(cause);
   if (reduced.code !== 'internal') return reduced.message;
   logger.error(
@@ -246,17 +408,71 @@ function safeToolMessage(cause: unknown, name: string, logger: HostLogger): stri
   return 'The tool failed.';
 }
 
-/** Run one `ai.chat` turn. */
+/** How much of the person's message stands in for the whole turn. */
+const SUMMARY_CHARS = 200;
+
+/** Run one `ai.chat` turn, and tell whoever is listening how it ended. */
 export async function runChat(
   params: StreamChatParams,
   sink: StreamSink<ChatEvent>,
   deps: RunDeps,
+): Promise<void> {
+  // Reported exactly once, whatever happens: a turn that threw, a turn the
+  // browser cancelled and a turn that finished all have to close their record,
+  // or a run store is left with something that looks like it is still running.
+  let ended = false;
+  const end = (status: 'succeeded' | 'failed' | 'cancelled'): void => {
+    if (ended) return;
+    ended = true;
+    deps.onRunEnd?.(params.runId, status, params.message.slice(0, SUMMARY_CHARS));
+  };
+  try {
+    await runTurn(params, sink, deps, end);
+    end(sink.signal.aborted ? 'cancelled' : 'succeeded');
+  } catch (cause) {
+    end(sink.signal.aborted ? 'cancelled' : 'failed');
+    throw cause;
+  }
+}
+
+/** The turn itself. */
+async function runTurn(
+  params: StreamChatParams,
+  sink: StreamSink<ChatEvent>,
+  deps: RunDeps,
+  end: (status: 'succeeded' | 'failed' | 'cancelled') => void,
 ): Promise<void> {
   // Throws a PublicError when nothing is configured. `runStream` in host/app.ts
   // turns that into the right thing on the wire, so it is not caught here.
-  const resolved = await deps.registry.resolve();
+  // The turn's own model, when a conversation has one. `resolve` applies it
+  // after the provider and key checks, so the vision check below and the model
+  // instance built later both follow it without a second code path.
+  const resolved = await deps.registry.resolve({ modelId: params.modelId });
+
+  // Both checks come before anything is emitted, so a turn that cannot carry
+  // its images fails as a whole rather than half-answering.
+  const files = params.files ?? [];
+  if (files.length > 0) {
+    const characters = files.reduce((total, file) => total + file.data.length, 0);
+    if (characters > MAX_FILE_CHARS_PER_TURN) {
+      throw publicError.invalidInput('Images on one message are limited to about 4 MB together.');
+    }
+    if (!(await modelCanSee(resolved, sink.signal, deps.logger))) {
+      throw publicError.rejected(
+        'The chosen model cannot read images. Pick one that can in Settings.',
+      );
+    }
+  }
+
   const documents = await assembleContext(params, deps, sink.signal);
 
+  // One approver per run. The request identifier the gate will use is
+  // `<runId>:<callId>`, so the call a `confirm` event names can be recovered
+  // from it — which is what keeps `ai.chatConfirm`'s wire shape unchanged.
+  const approver = createRunApprover(deps, sink, (requestId) =>
+    requestId.startsWith(`${params.runId}:`) ? requestId.slice(params.runId.length + 1) : requestId,
+  );
+
   const result = streamText({
     // Always a model *instance*. A string here would be resolved by the AI
     // SDK's gateway, over the global fetch, to a Vercel host — see
@@ -264,7 +480,7 @@ export async function runChat(
     model: resolved.adapter.model(resolved.config, resolved.modelId),
     system: buildSystemPrompt(deps, documents),
     messages: toModelMessages(params),
-    tools: buildTools(params, deps, sink),
+    tools: buildTools(params, deps, sink, approver),
     stopWhen: stepCountIs(deps.maxSteps),
     abortSignal: sink.signal,
     // The default handler prints the error; this layer reports it as an event
@@ -294,6 +510,9 @@ export async function runChat(
           code: 'provider',
           message: safeMessage(part.error, deps.logger),
         });
+        // The stream ends here rather than at `done`, so the turn's outcome is
+        // settled here too.
+        end('failed');
         return;
       case 'tool-error': {
         // `execute` never throws, so this means the SDK failed before the tool
@@ -309,6 +528,7 @@ export async function runChat(
         break;
       }
       case 'abort':
+        end('cancelled');
         return;
       default:
         // tool-call, tool-result, text-start, finish-step, reasoning, source,