@opengeni/react 0.46.1 → 0.52.1

package/src/timeline/parsers.ts

@@ -1,6 +1,14 @@
-import { normalizeMcpOutput, type GitFileDiff } from "@opengeni/sdk";
+import {
+  normalizeMcpOutput,
+  parseGeneratedImageReceipt,
+  type GeneratedImageReceipt,
+  type GitFileDiff,
+  type RetainedArtifactMetadata,
+} from "@opengeni/sdk";
 import { tryParseJson } from "../lib/format";
 
+export type { GeneratedImageReceipt };
+
 /* ----------------------------------------------------------------------------
    Pure parsers for the provider-native tool shapes that the timeline renders.
 
@@ -437,25 +445,6 @@ export function isApplyPatch(item: { name: string; raw: unknown }): boolean {
   return name === "apply_patch_call" || name === "apply_patch" || name.endsWith("__apply_patch");
 }
 
-/* --- secret redaction ------------------------------------------------------- */
-
-const SECRET_KEY = /^(value|secret|token|password|api[_-]?key|signing[_-]?key)$/i;
-
-/** Deep-redact secret-looking values so arguments never leak a key into the UI. */
-export function redactSecrets(value: unknown): unknown {
-  if (Array.isArray(value)) {
-    return value.map(redactSecrets);
-  }
-  if (value && typeof value === "object") {
-    const out: Record<string, unknown> = {};
-    for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
-      out[k] = SECRET_KEY.test(k) ? "••••" : redactSecrets(v);
-    }
-    return out;
-  }
-  return value;
-}
-
 /** Parse tool arguments that may arrive as a JSON string or an object. */
 export function parseToolArgs(args: unknown): Record<string, unknown> {
   if (args == null) {
@@ -558,6 +547,43 @@ export function screenshotDataUrl(out: unknown): string | null {
   return null;
 }
 
+/** Parse the closed retained-screenshot receipt carried by new tool events. */
+export function retainedScreenshotMetadata(out: unknown): RetainedArtifactMetadata | null {
+  if (typeof out === "string" && (out.startsWith("{") || out.startsWith("["))) {
+    const parsed = tryParseJson(out);
+    if (parsed !== undefined && parsed !== out) return retainedScreenshotMetadata(parsed);
+  }
+  if (Array.isArray(out)) {
+    for (const entry of out) {
+      const metadata = retainedScreenshotMetadata(entry);
+      if (metadata) return metadata;
+    }
+    return null;
+  }
+  if (!out || typeof out !== "object") return null;
+  const value = out as Record<string, unknown>;
+  if (typeof value.artifactId !== "string" || typeof value.available !== "boolean") return null;
+  if (!value.available) {
+    return typeof value.reason === "string" ? (value as unknown as RetainedArtifactMetadata) : null;
+  }
+  return value.kind === "computer_screenshot" &&
+    value.contentType === "image/png" &&
+    typeof value.originalBytes === "number" &&
+    typeof value.sha256 === "string" &&
+    value.dimensions !== null &&
+    typeof value.dimensions === "object" &&
+    value.retention !== null &&
+    typeof value.retention === "object" &&
+    (value.retention as Record<string, unknown>).policy === "session_screenshot"
+    ? (value as unknown as RetainedArtifactMetadata)
+    : null;
+}
+
+/** Parse the closed permanent generated-image receipt from native/function tools. */
+export function generatedImageReceipt(out: unknown): GeneratedImageReceipt | null {
+  return parseGeneratedImageReceipt(out);
+}
+
 export type TimelineMediaPreview = {
   type: "media_preview";
   mediaType: string;

package/src/timeline/projection.ts

@@ -1,5 +1,10 @@
-import type { SessionEvent, SessionStatus } from "@opengeni/sdk";
-const { default: fleetDecisionItem } = await import("./fleet-decision-projection");
+import {
+  parseMediaGenerationResult,
+  type SessionEvent,
+  type SessionStatus,
+  type TimelineAnnotation,
+} from "@opengeni/sdk";
+import fleetDecisionItem from "./fleet-decision-projection";
 import {
   CREDIT_EXHAUSTION_MESSAGE,
   humanizeFailureReason,
@@ -21,6 +26,7 @@ import type {
   TimelineItem,
   TurnEndItem,
   ToolCallItem,
+  ToolCallTruncation,
   WorkerItem,
 } from "./types";
 
@@ -61,6 +67,56 @@ const WORKER_MESSAGE_TOOL = "session_send_message";
  * render as machine-input rows.
  */
 const LANDMARK_ONLY_TOOL_LEAVES = new Set(["memory_save", "memory_correct"]);
+const TIMELINE_ANNOTATION_ANSI_SEQUENCE = new RegExp("\\u001B\\[[0-?]*[ -/]*[@-~]", "g");
+
+function safeTimelineAnnotationJson(value: unknown): string {
+  try {
+    return JSON.stringify(value) ?? "";
+  } catch {
+    return String(value ?? "");
+  }
+}
+
+function timelineAnnotationToolOutputValue(value: unknown, depth = 0): string {
+  if (depth > 8 || value === null || value === undefined) {
+    return value == null ? "" : safeTimelineAnnotationJson(value);
+  }
+  if (typeof value === "string") return value;
+  if (typeof value !== "object" || Array.isArray(value)) {
+    return safeTimelineAnnotationJson(value);
+  }
+  const record = value as Record<string, unknown>;
+  if (record.type === "text" && typeof record.text === "string") return record.text;
+  if (Array.isArray(record.content)) {
+    const text = record.content.find(
+      (part) =>
+        part !== null &&
+        typeof part === "object" &&
+        (part as { type?: unknown }).type === "text" &&
+        typeof (part as { text?: unknown }).text === "string",
+    ) as { text: string } | undefined;
+    if (text) return text.text;
+  }
+  if ("structuredContent" in record) {
+    return timelineAnnotationToolOutputValue(record.structuredContent, depth + 1);
+  }
+  if ("result" in record) return timelineAnnotationToolOutputValue(record.result, depth + 1);
+  return safeTimelineAnnotationJson(value);
+}
+
+/** Browser mirror of the server's canonical annotation source projection. */
+function timelineAnnotationToolOutputText(output: unknown): string | null {
+  const raw = timelineAnnotationToolOutputValue(output);
+  const marker = raw.indexOf("\nOutput:\n");
+  const text = (
+    marker >= 0
+      ? raw.slice(marker + "\nOutput:\n".length)
+      : raw.startsWith("Output:\n")
+        ? raw.slice("Output:\n".length)
+        : raw
+  ).replace(TIMELINE_ANNOTATION_ANSI_SEQUENCE, "");
+  return text.length > 0 ? text : null;
+}
 
 export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
   const items: TimelineItem[] = [];
@@ -134,6 +190,15 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
           kind: "user-message",
           id: event.id,
           text: voiceMessage?.text ?? stringValue(payload.text),
+          annotations: timelineAnnotations(payload.annotations),
+          annotationSource: {
+            kind: "user_message",
+            eventId: event.id,
+            eventType: "user.message",
+            sequence: event.sequence,
+            turnId,
+            text: voiceMessage?.text ?? stringValue(payload.text),
+          },
           ...(voiceMessage ? { presentation: voiceMessage.presentation } : {}),
           resources: resourceRefs(payload.resources),
           tools: toolRefs(payload.tools),
@@ -212,6 +277,17 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
           // Completion time is what the footer shows ("finished at"); keep the
           // first-delta stamp only until this event arrives.
           open.occurredAt = event.occurredAt;
+          open.annotationSource =
+            text && open.text === text
+              ? {
+                  kind: "assistant_message",
+                  eventId: event.id,
+                  eventType: "agent.message.completed",
+                  sequence: event.sequence,
+                  turnId,
+                  text,
+                }
+              : undefined;
           // The SDK can emit a hosted-tool item only after its provider-native
           // operation has completed, even though answer deltas were already
           // streamed. The completed message event is the durable ordering
@@ -231,6 +307,14 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
             text,
             streaming: false,
             occurredAt: event.occurredAt,
+            annotationSource: {
+              kind: "assistant_message",
+              eventId: event.id,
+              eventType: "agent.message.completed",
+              sequence: event.sequence,
+              turnId,
+              text,
+            },
           });
         }
         break;
@@ -312,6 +396,7 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
           name,
           arguments: args,
           output: undefined,
+          truncation: null,
           // The provider-native item drives the per-tool renderers (apply_patch
           // operation, computer_call action, web_search providerData, …).
           raw: payload.raw,
@@ -336,8 +421,30 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
         }
         // An output carrying an explicit error flag (or an MCP isError result)
         // settles the tool to "failed" so the renderer can surface it loudly.
+        const truncation = toolCallTruncation(payload);
         target.status = isErrorOutput(payload) ? "failed" : "complete";
-        target.output = payload.output;
+        target.output = Object.prototype.hasOwnProperty.call(payload, "output")
+          ? payload.output
+          : truncation && typeof payload.preview === "string"
+            ? payload.preview
+            : payload.output;
+        target.truncation = truncation;
+        const sourceText = timelineAnnotationToolOutputText(
+          Object.prototype.hasOwnProperty.call(payload, "output")
+            ? payload.output
+            : payload.preview,
+        );
+        if (sourceText !== null) {
+          target.annotationSource = {
+            kind: "tool_output",
+            eventId: event.id,
+            eventType: "agent.toolCall.output",
+            sequence: event.sequence,
+            turnId,
+            text: sourceText,
+            label: target.name,
+          };
+        }
         break;
       }
 
@@ -540,6 +647,19 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
 
       case "tool.auth_needed":
       case "credential.auth_needed": {
+        const capability = capabilityAuthorizationRequest(payload.capability);
+        if (
+          event.type === "tool.auth_needed" &&
+          !stringValue(payload.toolName) &&
+          stringValue(payload.serverId) !== "codex_apps"
+        ) {
+          // Historical optional-MCP initialize/tools-list credential misses are
+          // setup availability, not evidence of a concrete conversational tool
+          // call. Keep the raw event in Debug/audit but do not manufacture an
+          // actionable reconnect card in the transcript. Actual tools/call auth
+          // failures carry a concrete toolName and remain visible.
+          break;
+        }
         // Keep the whole structured payload — the renderer turns it into a clean
         // inline reconnect card, and the app starts the recovery flow off the
         // connectionId/resource. Losing it to a plain-text notice was the ugly
@@ -549,6 +669,12 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
           kind: "auth-needed",
           id: event.id,
           turnId,
+          serverId: typeof payload.serverId === "string" ? payload.serverId : null,
+          source: capability
+            ? "capability"
+            : event.type === "tool.auth_needed"
+              ? "tool"
+              : "credential",
           providerDomain: stringValue(payload.providerDomain),
           connectionId: typeof payload.connectionId === "string" ? payload.connectionId : null,
           reason: authNeededReason(payload.reason),
@@ -557,6 +683,7 @@ export function buildTimeline(events: SessionEvent[]): TimelineItem[] {
           toolName: typeof payload.toolName === "string" ? payload.toolName : null,
           authorizationUrl:
             typeof payload.authorizationUrl === "string" ? payload.authorizationUrl : null,
+          capability,
           occurredAt: event.occurredAt,
         });
         break;
@@ -1301,6 +1428,7 @@ function machineInputMembers(value: unknown): MachineInputBatchItem["members"] {
     "agent_message",
     "agent_steer_instruction",
     "child_terminal_result",
+    "media_generation_result",
   ]);
   const classifications = new Set<MachineInputBatchItem["members"][number]["classification"]>([
     "success",
@@ -1319,16 +1447,24 @@ function machineInputMembers(value: unknown): MachineInputBatchItem["members"] {
             member.classification as MachineInputBatchItem["members"][number]["classification"],
           ) &&
           typeof member.sourceId === "string"
-          ? [
-              {
-                id: member.id,
-                kind: member.kind as MachineInputBatchItem["members"][number]["kind"],
-                classification:
-                  member.classification as MachineInputBatchItem["members"][number]["classification"],
-                sourceId: member.sourceId,
-                summary: stringValue(member.summary),
-              },
-            ]
+          ? (() => {
+              const kind = member.kind as MachineInputBatchItem["members"][number]["kind"];
+              const result =
+                kind === "media_generation_result"
+                  ? parseMediaGenerationResult(member.result)
+                  : null;
+              return [
+                {
+                  id: member.id,
+                  kind,
+                  classification:
+                    member.classification as MachineInputBatchItem["members"][number]["classification"],
+                  sourceId: member.sourceId,
+                  summary: stringValue(member.summary),
+                  ...(result ? { result } : {}),
+                },
+              ];
+            })()
           : [];
       })
     : [];
@@ -1364,6 +1500,25 @@ function resourceRefs(value: unknown): import("@opengeni/sdk").ResourceRef[] {
   });
 }
 
+function timelineAnnotations(value: unknown): TimelineAnnotation[] {
+  if (!Array.isArray(value)) return [];
+  return value.filter((candidate): candidate is TimelineAnnotation => {
+    const annotation = asRecord(candidate);
+    const source = asRecord(annotation.source);
+    return (
+      typeof annotation.id === "string" &&
+      typeof annotation.ordinal === "number" &&
+      typeof annotation.quote === "string" &&
+      typeof annotation.note === "string" &&
+      typeof source.eventId === "string" &&
+      typeof source.eventType === "string" &&
+      typeof source.sequence === "number" &&
+      typeof source.startOffset === "number" &&
+      typeof source.endOffset === "number"
+    );
+  });
+}
+
 function toolRefs(value: unknown): import("@opengeni/sdk").ToolRef[] {
   if (!Array.isArray(value)) {
     return [];
@@ -1426,6 +1581,31 @@ function isErrorOutput(payload: Record<string, unknown>): boolean {
   );
 }
 
+function toolCallTruncation(payload: Record<string, unknown>): ToolCallTruncation | null {
+  const truncation = asRecord(payload.truncation);
+  const fullEvidence = asRecord(truncation.fullEvidence);
+  const surface = stringValue(truncation.surface);
+  const reason = stringValue(truncation.reason);
+  if (
+    truncation.truncated !== true ||
+    !surface ||
+    !reason ||
+    typeof fullEvidence.available !== "boolean"
+  ) {
+    return null;
+  }
+  return {
+    truncated: true,
+    surface,
+    reason,
+    omittedBytes: numberOrNull(truncation.omittedBytes),
+    fullEvidence: {
+      available: fullEvidence.available,
+      reason: stringValue(fullEvidence.reason) || null,
+    },
+  };
+}
+
 function findOpenCall(
   items: TimelineItem[],
   callId: string | null,
@@ -1549,6 +1729,37 @@ function authNeededReason(value: unknown): AuthNeededItem["reason"] {
     : null;
 }
 
+function capabilityAuthorizationRequest(
+  value: unknown,
+): NonNullable<AuthNeededItem["capability"]> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const record = value as Record<string, unknown>;
+  const kind = record.kind;
+  const source = record.source;
+  const action = record.action;
+  if (
+    typeof record.id !== "string" ||
+    typeof record.name !== "string" ||
+    !["pack", "mcp", "api", "skill", "plugin"].includes(String(kind)) ||
+    !["built_in", "library", "configured", "public_registry", "registry", "manual"].includes(
+      String(source),
+    ) ||
+    !["connect", "add_credentials", "enable"].includes(String(action)) ||
+    typeof record.rationale !== "string"
+  ) {
+    return null;
+  }
+  return {
+    id: record.id,
+    name: record.name,
+    kind: kind as NonNullable<AuthNeededItem["capability"]>["kind"],
+    source: source as NonNullable<AuthNeededItem["capability"]>["source"],
+    action: action as NonNullable<AuthNeededItem["capability"]>["action"],
+    rationale: record.rationale,
+    requiredVariables: stringList(record.requiredVariables),
+  };
+}
+
 function stringList(value: unknown): string[] {
   return Array.isArray(value)
     ? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
@@ -1609,6 +1820,15 @@ export function extractSessionRef(value: unknown, depth = 0): string | null {
     return null;
   }
   const record = value as Record<string, unknown>;
+  const receiptResource = asRecord(record.resource);
+  if (
+    record.receiptVersion === "mcp-mutation-receipt.v1" &&
+    receiptResource.type === "session" &&
+    typeof receiptResource.id === "string" &&
+    looksLikeId(receiptResource.id)
+  ) {
+    return receiptResource.id;
+  }
   if (typeof record.sessionId === "string" && looksLikeId(record.sessionId)) {
     return record.sessionId;
   }

package/src/timeline/registry.ts

@@ -1,3 +1,4 @@
+import type { RetainedArtifactReference, VideoArtifactPlaybackSource } from "@opengeni/sdk";
 import type { ComponentType } from "react";
 import { mcpToolLeaf } from "./tool-display-name";
 import type { ToolCallItem } from "./types";
@@ -22,8 +23,26 @@ import type { ToolCallItem } from "./types";
 
 export type ToolRendererProps = {
   item: ToolCallItem;
+  loadRetainedScreenshot?: RetainedScreenshotLoader | undefined;
+  loadRetainedArtifact?: RetainedArtifactLoader | undefined;
 };
 
+export type RetainedScreenshotLoader = (
+  artifact: RetainedArtifactReference,
+  signal: AbortSignal,
+) => Promise<Uint8Array | null>;
+
+export type RetainedArtifactLoader = (
+  artifact: RetainedArtifactReference,
+  signal: AbortSignal,
+) => Promise<Uint8Array | { url: string } | null>;
+
+/** Mint an expiring source for native browser playback without loading video bytes in JS. */
+export type VideoArtifactPlaybackLoader = (
+  artifactId: string,
+  signal: AbortSignal,
+) => Promise<VideoArtifactPlaybackSource>;
+
 export type ToolRenderer = ComponentType<ToolRendererProps>;
 
 /** A registry entry: which key it matches and the component that renders it. */

package/src/timeline/screenshot-lightbox.tsx

@@ -3,6 +3,7 @@ import { AnimatePresence, motion } from "motion/react";
 import { createContext, useCallback, useContext, useMemo, useState, type ReactNode } from "react";
 import { Dialog } from "radix-ui";
 import { cn } from "../lib/cn";
+import { usePortalTokenStyle } from "../lib/use-portal-token-style";
 
 /* ----------------------------------------------------------------------------
    Screenshot lightbox
@@ -18,7 +19,7 @@ import { cn } from "../lib/cn";
    -------------------------------------------------------------------------- */
 
 type LightboxController = {
-  open: (src: string, caption?: string) => void;
+  open: (src: string, caption?: string, source?: HTMLElement | null, label?: string) => void;
 };
 
 const LightboxContext = createContext<LightboxController | null>(null);
@@ -58,10 +59,25 @@ export function LightboxProvider({ children }: { children: ReactNode }) {
 }
 
 function LightboxRoot({ children }: { children: ReactNode }) {
-  const [state, setState] = useState<{ src: string; caption?: string } | null>(null);
+  const [state, setState] = useState<{
+    src: string;
+    caption?: string;
+    label: string;
+  } | null>(null);
+  const [tokenSource, setTokenSource] = useState<HTMLElement | null>(null);
+  const portalStyle = usePortalTokenStyle(tokenSource);
 
-  const open = useCallback((src: string, caption?: string) => {
-    setState(caption ? { src, caption } : { src });
+  const open = useCallback(
+    (src: string, caption?: string, source?: HTMLElement | null, label = "Screenshot") => {
+      setTokenSource(source ?? null);
+      setState(caption ? { src, caption, label } : { src, label });
+    },
+    [],
+  );
+
+  const close = useCallback(() => {
+    setState(null);
+    setTokenSource(null);
   }, []);
 
   const controller = useMemo<LightboxController>(() => ({ open }), [open]);
@@ -69,7 +85,7 @@ function LightboxRoot({ children }: { children: ReactNode }) {
   return (
     <LightboxContext.Provider value={controller}>
       {children}
-      <Dialog.Root open={state !== null} onOpenChange={(next) => !next && setState(null)}>
+      <Dialog.Root open={state !== null} onOpenChange={(next) => !next && close()}>
         <AnimatePresence>
           {state !== null ? (
             <Dialog.Portal forceMount>
@@ -80,12 +96,13 @@ function LightboxRoot({ children }: { children: ReactNode }) {
                   exit={{ opacity: 0 }}
                   transition={{ duration: 0.15 }}
                   className="og-root fixed inset-0 z-50 bg-black/90 backdrop-blur-md"
+                  style={portalStyle}
                 />
               </Dialog.Overlay>
               <Dialog.Content
                 asChild
                 forceMount
-                aria-label="Screenshot"
+                aria-label={state.label}
                 onOpenAutoFocus={(event) => event.preventDefault()}
               >
                 <motion.div
@@ -101,12 +118,16 @@ function LightboxRoot({ children }: { children: ReactNode }) {
                   // figcaption's "click outside to close" affordance is real.
                   onClick={(event) => {
                     if (event.target === event.currentTarget) {
-                      setState(null);
+                      close();
                     }
                   }}
                   className="og-root fixed inset-0 z-50 flex items-center justify-center p-6 sm:p-12"
+                  style={portalStyle}
                 >
-                  <Dialog.Title className="sr-only">Screenshot</Dialog.Title>
+                  <Dialog.Title className="sr-only">{state.label}</Dialog.Title>
+                  <Dialog.Description className="sr-only">
+                    {state.caption ?? `Expanded ${state.label.toLowerCase()}`}
+                  </Dialog.Description>
                   {/* The figure is one self-contained object: image, caption, and
                       its own close control. The close button anchors to a wrapper
                       sized to the IMAGE (w-fit), so it hugs the real top-right
@@ -117,7 +138,7 @@ function LightboxRoot({ children }: { children: ReactNode }) {
                       {/* A plain <img>: this SDK is framework-agnostic, with no host Image component. */}
                       <img
                         src={state.src}
-                        alt={state.caption ?? "Screenshot"}
+                        alt={state.caption ?? state.label}
                         className="min-h-0 max-h-[82vh] w-auto max-w-full rounded-og-md border border-white/10 object-contain shadow-og-lg"
                       />
                       <Dialog.Close