@opengeni/react 0.48.0 → 0.57.0

package/src/timeline/tool-renderers.tsx

@@ -26,15 +26,17 @@ import {
   Share2Icon,
   TargetIcon,
   TerminalIcon,
+  VideoIcon,
   WrenchIcon,
 } from "lucide-react";
-import { useEffect, useState, type ReactNode } from "react";
+import { useEffect, useRef, useState, type ReactNode } from "react";
 import { stringifyPayload, tryParseJson } from "../lib/format";
 import { useTimelineComputeLabel } from "./compute-label";
 import {
   applyPatchOpsFromToolItem,
   controlCaret,
   execTruncated,
+  generatedImageReceipt,
   isExecSessionLostBanner,
   looksBinary,
   mediaPreviewFact,
@@ -664,11 +666,14 @@ function ComputerCallRenderer({ item, loadRetainedScreenshot }: ToolRendererProp
       );
     }
     return (
-      <RetainedScreenshotDisclosure
+      <RetainedSessionImageDisclosure
         artifact={retained}
         load={loadRetainedScreenshot}
-        verb={verb}
-        countSuffix={countSuffix}
+        title={`${verb}${countSuffix}`}
+        caption={`${verb}${countSuffix}`}
+        noun="screenshot"
+        icon={<CameraIcon className={ICON_SIZE} />}
+        lightboxLabel="Screenshot"
         batched={batched}
         failed={isFailed}
         cancelled={isCancelled}
@@ -756,30 +761,26 @@ function ComputerCallRenderer({ item, loadRetainedScreenshot }: ToolRendererProp
   );
 }
 
-function RetainedScreenshotDisclosure({
-  artifact,
-  load,
-  verb,
-  countSuffix,
-  batched,
-  failed,
-  cancelled,
-}: {
-  artifact: RetainedArtifactReference;
-  load: ToolRendererProps["loadRetainedScreenshot"];
-  verb: string;
-  countSuffix: string;
-  batched: string | null;
-  failed: boolean;
-  cancelled: boolean;
-}) {
-  const [state, setState] = useState<
-    | { kind: "loading" }
-    | { kind: "ready"; url: string }
-    | { kind: "unavailable"; label: string }
-    | { kind: "error"; message: string }
-  >({ kind: "loading" });
-
+type RetainedImageState =
+  | { kind: "loading" }
+  | { kind: "ready"; url: string }
+  | { kind: "unavailable"; label: string }
+  | { kind: "error"; message: string };
+
+function useRetainedImageObjectUrl(
+  artifact: RetainedArtifactReference,
+  load: ToolRendererProps["loadRetainedArtifact"],
+): RetainedImageState {
+  const [state, setState] = useState<RetainedImageState>({ kind: "loading" });
+  // Function outputs are commonly serialized JSON. Parsing them creates a new
+  // object on every render, so depend on the immutable wire value rather than
+  // object identity; otherwise the loader can refetch after its own setState.
+  const artifactValue = retainedArtifactValue(artifact);
+  const stableArtifactRef = useRef({ value: artifactValue, artifact });
+  if (stableArtifactRef.current.value !== artifactValue) {
+    stableArtifactRef.current = { value: artifactValue, artifact };
+  }
+  const stableArtifact = stableArtifactRef.current.artifact;
   useEffect(() => {
     if (!load) {
       setState({ kind: "unavailable", label: "retrieval is not configured" });
@@ -788,14 +789,24 @@ function RetainedScreenshotDisclosure({
     const controller = new AbortController();
     let objectUrl: string | null = null;
     setState({ kind: "loading" });
-    void load(artifact, controller.signal)
-      .then((bytes) => {
+    void load(stableArtifact, controller.signal)
+      .then((source) => {
         if (controller.signal.aborted) return;
-        if (!bytes) {
+        if (!source) {
           setState({ kind: "unavailable", label: "bytes are unavailable" });
           return;
         }
-        objectUrl = URL.createObjectURL(new Blob([Uint8Array.from(bytes)], { type: "image/png" }));
+        if (!(source instanceof Uint8Array)) {
+          if (!source.url.trim()) {
+            setState({ kind: "unavailable", label: "URL is unavailable" });
+            return;
+          }
+          setState({ kind: "ready", url: source.url });
+          return;
+        }
+        objectUrl = URL.createObjectURL(
+          new Blob([source as unknown as BlobPart], { type: stableArtifact.contentType }),
+        );
         setState({ kind: "ready", url: objectUrl });
       })
       .catch((error: unknown) => {
@@ -809,38 +820,86 @@ function RetainedScreenshotDisclosure({
             ? { kind: "unavailable", label: "deleted" }
             : status === 410
               ? { kind: "unavailable", label: "expired or unavailable" }
-              : {
-                  kind: "error",
-                  message: "retrieval failed",
-                },
+              : { kind: "error", message: "retrieval failed" },
         );
       });
     return () => {
       controller.abort();
       if (objectUrl) URL.revokeObjectURL(objectUrl);
     };
-  }, [artifact, load]);
+  }, [stableArtifact, load]);
+  return state;
+}
+
+function retainedArtifactValue(artifact: RetainedArtifactReference): string {
+  return [
+    artifact.artifactId,
+    artifact.kind,
+    artifact.contentType,
+    artifact.originalBytes,
+    artifact.sha256,
+    artifact.retainedAt,
+    artifact.dimensions?.width ?? "",
+    artifact.dimensions?.height ?? "",
+    artifact.retention.policy,
+    artifact.retention.expiresAt ?? "",
+    artifact.retrieval.method,
+    artifact.retrieval.path,
+    artifact.retrieval.acceptRanges,
+    artifact.retrieval.maxRangeBytes,
+  ].join("\0");
+}
+
+function RetainedSessionImageDisclosure({
+  artifact,
+  load,
+  title,
+  caption,
+  noun,
+  icon,
+  lightboxLabel,
+  batched,
+  failed,
+  cancelled,
+}: {
+  artifact: RetainedArtifactReference;
+  load: ToolRendererProps["loadRetainedScreenshot"];
+  title: string;
+  caption: string;
+  noun: "image" | "screenshot";
+  icon: ReactNode;
+  lightboxLabel: string;
+  batched: string | null;
+  failed: boolean;
+  cancelled: boolean;
+}) {
+  const state = useRetainedImageObjectUrl(artifact, load);
 
-  const caption = `${verb}${countSuffix}`;
   return (
     <ActivityDisclosure
-      icon={<CameraIcon className={ICON_SIZE} />}
+      icon={icon}
       iconTone={failed ? "failed" : state.kind === "ready" ? "accent" : "muted"}
-      title={caption}
+      title={title}
       failed={failed}
       cancelled={cancelled}
       preview={
         state.kind === "loading"
-          ? "loading retained screenshot…"
+          ? `loading retained ${noun}…`
           : state.kind === "error"
-            ? "screenshot retrieval failed"
+            ? `${noun} retrieval failed`
             : state.kind === "unavailable"
-              ? `screenshot ${state.label}`
+              ? `${noun} ${state.label}`
               : undefined
       }
       media={
         state.kind === "ready" ? (
-          <Thumbnail src={state.url} caption={caption} />
+          <Thumbnail
+            src={state.url}
+            caption={caption}
+            alt={caption}
+            expandLabel={`Expand ${noun}`}
+            lightboxLabel={lightboxLabel}
+          />
         ) : state.kind === "loading" ? (
           <MediaSkeleton />
         ) : (
@@ -849,19 +908,171 @@ function RetainedScreenshotDisclosure({
       }
     >
       {state.kind === "ready" ? (
-        <ScreenshotFigure src={state.url} caption={caption} />
+        <ScreenshotFigure
+          src={state.url}
+          caption={caption}
+          alt={caption}
+          expandLabel={`Expand ${noun}`}
+          lightboxLabel={lightboxLabel}
+        />
       ) : state.kind === "loading" ? (
-        <BodyNote>Loading the retained screenshot…</BodyNote>
+        <BodyNote>Loading the retained {noun}…</BodyNote>
       ) : state.kind === "unavailable" ? (
-        <BodyNote>Screenshot {state.label}.</BodyNote>
+        <BodyNote>
+          {noun === "screenshot" ? "Screenshot" : "Image"} {state.label}.
+        </BodyNote>
       ) : (
-        <BodyNote tone="error">Screenshot retrieval failed: {state.message}</BodyNote>
+        <BodyNote tone="error">
+          {noun === "screenshot" ? "Screenshot" : "Image"} retrieval failed: {state.message}
+        </BodyNote>
       )}
       {batched ? <BodyNote>batched: {batched}</BodyNote> : null}
     </ActivityDisclosure>
   );
 }
 
+function GeneratedImageRenderer({ item, loadRetainedArtifact }: ToolRendererProps) {
+  const args = parseToolArgs(item.arguments);
+  const prompt = typeof args.prompt === "string" ? args.prompt : "";
+  const raw =
+    item.raw && typeof item.raw === "object" && !Array.isArray(item.raw)
+      ? (item.raw as Record<string, unknown>)
+      : null;
+  // Function tools settle through agent.toolCall.output; OpenAI's hosted image
+  // call is already complete on agent.toolCall.created and carries the compact
+  // receipt in raw.output. Both paths deliberately converge on one renderer.
+  const receipt = generatedImageReceipt(item.output) ?? generatedImageReceipt(raw?.output);
+  if (item.status === "running") {
+    return (
+      <ActivityDisclosure
+        icon={<ImageIcon className={ICON_SIZE} />}
+        iconTone="running"
+        title="Generating image"
+        running
+        preview={<RunningPreview>{truncatePreview(prompt, 72) || "creating…"}</RunningPreview>}
+        media={<MediaSkeleton />}
+      >
+        {prompt ? <BodyNote>{prompt}</BodyNote> : null}
+      </ActivityDisclosure>
+    );
+  }
+  if (!receipt) return <GenericRenderer item={item} />;
+  return (
+    <GeneratedImageDisclosure
+      receipt={receipt}
+      load={loadRetainedArtifact}
+      prompt={prompt}
+      failed={item.status === "failed"}
+      cancelled={item.status === "cancelled"}
+    />
+  );
+}
+
+function GeneratedVideoRenderer({ item }: ToolRendererProps) {
+  const args = parseToolArgs(item.arguments);
+  const prompt = typeof args.prompt === "string" ? args.prompt : "";
+  const parsedOutput = typeof item.output === "string" ? tryParseJson(item.output) : item.output;
+  const accepted =
+    parsedOutput &&
+    typeof parsedOutput === "object" &&
+    !Array.isArray(parsedOutput) &&
+    (parsedOutput as Record<string, unknown>).status === "accepted";
+  const failed = item.status === "failed";
+  const cancelled = item.status === "cancelled";
+  const running = item.status === "running";
+  return (
+    <ActivityDisclosure
+      icon={<VideoIcon className={ICON_SIZE} />}
+      iconTone={failed ? "failed" : running ? "running" : accepted ? "accent" : "muted"}
+      title={
+        running ? "Starting video generation" : accepted ? "Generating video" : "Generate video"
+      }
+      running={running}
+      failed={failed}
+      cancelled={cancelled}
+      preview={truncatePreview(prompt, 88) || (accepted ? "request accepted" : undefined)}
+    >
+      {prompt ? <BodyNote>{prompt}</BodyNote> : null}
+      {accepted ? (
+        <BodyNote>The video will appear here when it is ready.</BodyNote>
+      ) : item.output !== undefined ? (
+        <PayloadBlock label="Output" value={item.output} />
+      ) : null}
+    </ActivityDisclosure>
+  );
+}
+function GeneratedImageDisclosure({
+  receipt,
+  load,
+  prompt,
+  failed,
+  cancelled,
+}: {
+  receipt: NonNullable<ReturnType<typeof generatedImageReceipt>>;
+  load: ToolRendererProps["loadRetainedArtifact"];
+  prompt: string;
+  failed: boolean;
+  cancelled: boolean;
+}) {
+  const state = useRetainedImageObjectUrl(receipt.artifact, load);
+  const dimensions = receipt.artifact.dimensions!;
+  const title = failed ? "Image generation failed" : "Generated image";
+  const caption = prompt || `Generated image · ${dimensions.width}×${dimensions.height}`;
+  return (
+    <ActivityDisclosure
+      icon={<ImageIcon className={ICON_SIZE} />}
+      iconTone={failed ? "failed" : state.kind === "ready" ? "accent" : "muted"}
+      title={title}
+      defaultOpen={!failed && !cancelled}
+      failed={failed}
+      cancelled={cancelled}
+      preview={
+        state.kind === "loading"
+          ? "loading image…"
+          : state.kind === "error"
+            ? "image retrieval failed"
+            : state.kind === "unavailable"
+              ? `image ${state.label}`
+              : truncatePreview(prompt, 88) || `${dimensions.width}×${dimensions.height}`
+      }
+      media={
+        state.kind === "ready" ? (
+          <Thumbnail
+            src={state.url}
+            caption={caption}
+            alt={caption}
+            expandLabel="Expand generated image"
+            lightboxLabel="Generated image"
+          />
+        ) : state.kind === "loading" ? (
+          <MediaSkeleton />
+        ) : (
+          <MediaEmpty />
+        )
+      }
+    >
+      {state.kind === "ready" ? (
+        <ScreenshotFigure
+          src={state.url}
+          caption={caption}
+          alt={caption}
+          expandLabel="Expand generated image"
+          lightboxLabel="Generated image"
+        />
+      ) : state.kind === "loading" ? (
+        <BodyNote>Loading the generated image…</BodyNote>
+      ) : state.kind === "unavailable" ? (
+        <BodyNote>Image {state.label}.</BodyNote>
+      ) : (
+        <BodyNote tone="error">Image retrieval failed.</BodyNote>
+      )}
+      <BodyNote>
+        {dimensions.width}×{dimensions.height} · {receipt.sandboxPath}
+      </BodyNote>
+    </ActivityDisclosure>
+  );
+}
+
 /* ---- web_search ------------------------------------------------------------ */
 
 type WebSearchResult = { title: string; domain: string; snippet: string };
@@ -1013,11 +1224,12 @@ const VIEW_IMAGE_ERRORS = [
   "unable to read image",
 ];
 
-function ViewImageRenderer({ item }: ToolRendererProps) {
+function ViewImageRenderer({ item, loadRetainedScreenshot }: ToolRendererProps) {
   const args = parseToolArgs(item.arguments);
   const path = typeof args.path === "string" ? args.path : "";
   const out = item.output;
   const text = typeof out === "string" ? out : "";
+  const retained = retainedScreenshotMetadata(out);
   const omittedMedia = mediaPreviewFact(out);
 
   if (item.status === "running") {
@@ -1038,6 +1250,45 @@ function ViewImageRenderer({ item }: ToolRendererProps) {
   const viewFailed = item.status === "failed";
   const viewCancelled = item.status === "cancelled";
 
+  if (retained) {
+    const title = `Viewed ${basename(path)}`;
+    if (!retained.available) {
+      const state =
+        retained.reason === "expired" || retained.reason === "deleted"
+          ? retained.reason
+          : "unavailable";
+      return (
+        <ActivityDisclosure
+          icon={<ImageIcon className={ICON_SIZE} />}
+          iconTone={viewFailed ? "failed" : "muted"}
+          title={`${title} · ${state}`}
+          failed={viewFailed}
+          cancelled={viewCancelled}
+          preview={`image ${state}`}
+          media={<MediaEmpty />}
+        >
+          <BodyNote tone={viewFailed ? "error" : undefined}>
+            Image {state}: {retained.reason.replaceAll("_", " ")}.
+          </BodyNote>
+        </ActivityDisclosure>
+      );
+    }
+    return (
+      <RetainedSessionImageDisclosure
+        artifact={retained}
+        load={loadRetainedScreenshot}
+        title={title}
+        caption={path || title}
+        noun="image"
+        icon={<ImageIcon className={ICON_SIZE} />}
+        lightboxLabel="Image"
+        batched={null}
+        failed={viewFailed}
+        cancelled={viewCancelled}
+      />
+    );
+  }
+
   const errMatch = VIEW_IMAGE_ERRORS.find((p) => text.includes(p));
   if (errMatch) {
     const tooBig = text.includes("exceeded the allowed size");
@@ -1935,7 +2186,6 @@ const BASE_ENTRIES: ToolRegistryEntry[] = [
   // truth and is consulted first by the registry.
   { match: "rawType", type: "apply_patch_call", render: ApplyPatchRenderer },
   { match: "rawType", type: "computer_call", render: ComputerCallRenderer },
-  { match: "rawType", type: "hosted_tool_call", render: WebSearchRenderer },
   { match: "rawType", type: "tool_search_call", render: ToolSearchRenderer },
   // First-party sandbox + MCP tools resolve by name (exact or MCP leaf).
   { match: "name", name: "exec_command", render: ExecRenderer },
@@ -1955,6 +2205,9 @@ const BASE_ENTRIES: ToolRegistryEntry[] = [
   { match: "name", name: "computer_keypress", render: ComputerCallRenderer },
   { match: "name", name: "computer_drag", render: ComputerCallRenderer },
   { match: "name", name: "web_search_call", render: WebSearchRenderer },
+  { match: "name", name: "image_generation_call", render: GeneratedImageRenderer },
+  { match: "name", name: "generate_image", render: GeneratedImageRenderer },
+  { match: "name", name: "generate_video", render: GeneratedVideoRenderer },
   { match: "name", name: "tool_search", render: ToolSearchRenderer },
   { match: "name", name: "view_image", render: ViewImageRenderer },
   { match: "name", name: "environment_set_variable", render: SecretSetRenderer },

package/src/timeline/types.ts

@@ -1,4 +1,12 @@
-import type { ResourceRef, SessionStatus, ToolAuthNeededPayload, ToolRef } from "@opengeni/sdk";
+import type {
+  MediaGenerationResult,
+  ResourceRef,
+  SessionStatus,
+  TimelineAnnotation,
+  TimelineAnnotationSource,
+  ToolAuthNeededPayload,
+  ToolRef,
+} from "@opengeni/sdk";
 
 /* ----------------------------------------------------------------------------
    Timeline item types
@@ -9,10 +17,19 @@ import type { ResourceRef, SessionStatus, ToolAuthNeededPayload, ToolRef } from
    and the component demo.
    -------------------------------------------------------------------------- */
 
+export type TimelineAnnotationSourceDescriptor = Omit<
+  TimelineAnnotationSource,
+  "startOffset" | "endOffset" | "contextBefore" | "contextAfter"
+> & {
+  text: string;
+};
+
 export type UserMessageItem = {
   kind: "user-message";
   id: string;
   text: string;
+  annotations?: TimelineAnnotation[] | undefined;
+  annotationSource?: TimelineAnnotationSourceDescriptor | undefined;
   presentation?: {
     kind: "realtime_voice" | "realtime_voice_handoff";
     /** Full bounded execution context sent with this visible voice request. */
@@ -23,6 +40,15 @@ export type UserMessageItem = {
   /** Tools requested for the turn this message starts. */
   tools: ToolRef[];
   occurredAt: string;
+  /** Local-only delivery projection; absent for authoritative durable events. */
+  delivery?:
+    | {
+        state: "sending" | "queued" | "failed";
+        error?: string | undefined;
+        onRetry?: (() => void) | undefined;
+        onRemove?: (() => void) | undefined;
+      }
+    | undefined;
 };
 
 export type AgentMessageItem = {
@@ -33,6 +59,8 @@ export type AgentMessageItem = {
   /** Still receiving deltas (no completed/turn-end seen yet). */
   streaming: boolean;
   occurredAt: string;
+  /** Available only after the canonical completed event lands. */
+  annotationSource?: TimelineAnnotationSourceDescriptor | undefined;
 };
 
 export type ReasoningItem = {
@@ -44,6 +72,23 @@ export type ReasoningItem = {
   occurredAt: string;
 };
 
+/**
+ * Bounded delivery metadata carried by a projected tool-output event. The
+ * canonical event remains unchanged; this fact only explains what the current
+ * timeline surface could render and whether separately retained full evidence
+ * exists.
+ */
+export type ToolCallTruncation = {
+  truncated: true;
+  surface: string;
+  reason: string;
+  omittedBytes: number | null;
+  fullEvidence: {
+    available: boolean;
+    reason: string | null;
+  };
+};
+
 export type ToolCallItem = {
   kind: "tool-call";
   id: string;
@@ -52,6 +97,7 @@ export type ToolCallItem = {
   name: string;
   arguments: unknown;
   output: unknown;
+  truncation?: ToolCallTruncation | null;
   /**
    * The provider-native tool item (`agent.toolCall.created.payload.raw`). Carries
    * `type` (e.g. `apply_patch_call`, `computer_call`, `hosted_tool_call`) and the
@@ -62,6 +108,8 @@ export type ToolCallItem = {
   raw: unknown;
   status: "running" | "complete" | "failed" | "cancelled";
   occurredAt: string;
+  /** Canonical textual projection of the settled tool output. */
+  annotationSource?: TimelineAnnotationSourceDescriptor | undefined;
 };
 
 /**
@@ -79,6 +127,8 @@ export type WorkerItem = {
   prompt: string | null;
   /** The target/spawned worker session id, when parseable from args/output. */
   workerSessionId: string | null;
+  /** Bounded structured failure retained from session_create/session_send_message. */
+  failure: { code: string; message: string } | null;
   status: "running" | "complete" | "failed" | "cancelled";
   occurredAt: string;
 };
@@ -256,10 +306,12 @@ export type MachineInputMember = {
     | "goal_continuation"
     | "agent_message"
     | "agent_steer_instruction"
-    | "child_terminal_result";
+    | "child_terminal_result"
+    | "media_generation_result";
   classification: "success" | "failure" | "action_required" | "info";
   sourceId: string;
   summary: string;
+  result?: MediaGenerationResult | undefined;
 };
 
 /**
@@ -291,7 +343,7 @@ export type AuthNeededItem = {
   /** The runtime surface that requested recovery, when the event is an MCP auth signal. */
   serverId: string | null;
   /** Durable event family that produced this notice. */
-  source?: "tool" | "credential" | undefined;
+  source?: "tool" | "credential" | "capability" | undefined;
   /** The connection's registrable domain, e.g. "linear.app". */
   providerDomain: string;
   /** The lapsed connection to reconnect, when the row survived. */
@@ -305,6 +357,8 @@ export type AuthNeededItem = {
   toolName: string | null;
   /** A pre-minted authorization URL, when the broker already produced one. */
   authorizationUrl: string | null;
+  /** Agent-selected catalog recommendation. The host still owns authorization. */
+  capability?: NonNullable<ToolAuthNeededPayload["capability"]> | null | undefined;
   occurredAt: string;
 };