@larkup/workspace-app 0.0.0-stage → 0.2.0

package/src/components/chat/tools/chat-media-preview.tsx

@@ -0,0 +1,466 @@
+'use client';
+
+import { useEffect, useMemo, useRef, useState } from 'react';
+import { Image as ImageIcon, Film, AudioLines, Maximize2, X } from 'lucide-react';
+import { workspaceUrl } from '../../../host';
+
+/**
+ * Inline media preview for chat messages.
+ *
+ * Renders a compact preview of media referenced in assistant responses:
+ * - Images: thumbnail with lightbox expand
+ * - Video: compact player with timestamp range
+ * - Audio: mini player bar
+ */
+
+export function ChatMediaPreview({
+  assetId,
+  mediaType,
+  fileName,
+  mediaUrl,
+  sourceUrl,
+  startSecs,
+  endSecs,
+}: {
+  assetId: string;
+  mediaType: 'image' | 'video' | 'audio' | 'frame-preview';
+  fileName?: string;
+  mediaUrl?: string;
+  sourceUrl?: string;
+  startSecs?: number;
+  endSecs?: number;
+}) {
+  const [expanded, setExpanded] = useState(false);
+  const [imageUnavailable, setImageUnavailable] = useState(false);
+  const videoRef = useRef<HTMLVideoElement>(null);
+  // Media is rendered only from a server-validated presentMedia result. Do
+  // not invent a URL from an identifier emitted in model text: it could point
+  // to a missing or unrelated asset and creates the misleading dummy preview
+  // shown in the previous chat experience.
+  const assetUrl = mediaUrl ? workspaceUrl(mediaUrl) : undefined;
+  const playbackUrl =
+    assetUrl && startSecs !== undefined
+      ? `${assetUrl.split('#')[0]}#t=${startSecs}${endSecs !== undefined ? `,${endSecs}` : ''}`
+      : assetUrl;
+  const providerEmbedUrl = useMemo(
+    () => (mediaType === 'video' ? getProviderEmbedUrl(sourceUrl, startSecs, endSecs) : undefined),
+    [mediaType, sourceUrl, startSecs, endSecs],
+  );
+  const hasClipEnd =
+    mediaType === 'video' &&
+    Number.isFinite(startSecs) &&
+    Number.isFinite(endSecs) &&
+    (endSecs as number) > (startSecs as number);
+  useEffect(() => {
+    const video = videoRef.current;
+    if (!video || !Number.isFinite(startSecs)) return;
+    const seek = () => {
+      // Media fragments are not applied consistently after client-side
+      // navigation. Setting currentTime after metadata makes the cited moment
+      // deterministic for local uploads and downloaded provider videos.
+      video.currentTime = Math.max(0, startSecs as number);
+    };
+    if (video.readyState >= HTMLMediaElement.HAVE_METADATA) seek();
+    else video.addEventListener('loadedmetadata', seek, { once: true });
+    return () => video.removeEventListener('loadedmetadata', seek);
+  }, [playbackUrl, startSecs]);
+
+  if (!assetUrl) return null;
+
+  if (mediaType === 'image') {
+    if (imageUnavailable) {
+      return (
+        <div className="my-2 w-full max-w-2xl rounded-lg border border-border bg-card p-3">
+          <p className="text-xs text-muted-foreground">This source image is no longer available.</p>
+          <MediaCitationFooter
+            icon={<ImageIcon className="size-3 text-muted-foreground" />}
+            fileName={fileName}
+            sourceUrl={sourceUrl}
+          />
+        </div>
+      );
+    }
+    return (
+      <>
+        <div className="my-2 w-full max-w-2xl overflow-hidden rounded-lg border border-border bg-card">
+          <button
+            type="button"
+            onClick={() => setExpanded(true)}
+            className="group relative block w-full overflow-hidden"
+          >
+            <img
+              src={assetUrl}
+              alt={fileName ?? 'Image'}
+              className="max-h-105 w-full object-contain"
+              loading="lazy"
+              onError={() => setImageUnavailable(true)}
+            />
+            <div className="absolute inset-0 flex items-center justify-center bg-black/0 opacity-0 transition-all group-hover:bg-black/20 group-hover:opacity-100">
+              <Maximize2 className="size-5 text-white" />
+            </div>
+          </button>
+          <MediaCitationFooter
+            icon={<ImageIcon className="size-3 text-muted-foreground" />}
+            fileName={fileName}
+            sourceUrl={sourceUrl}
+          />
+        </div>
+
+        {expanded && (
+          <div
+            className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-8"
+            onClick={() => setExpanded(false)}
+          >
+            <button
+              type="button"
+              aria-label="Close image preview"
+              className="absolute right-4 top-4 rounded-full bg-black/50 p-2 text-white hover:bg-black/70"
+              onClick={() => setExpanded(false)}
+            >
+              <X className="size-5" />
+            </button>
+            <img
+              src={assetUrl}
+              alt={fileName ?? 'Image'}
+              className="max-h-[80vh] max-w-[90vw] rounded-lg object-contain"
+              onClick={(event) => event.stopPropagation()}
+              onError={() => setImageUnavailable(true)}
+            />
+          </div>
+        )}
+      </>
+    );
+  }
+
+  if (mediaType === 'video') {
+    return (
+      <div className="my-2 w-full max-w-2xl overflow-hidden rounded-lg border border-border bg-card">
+        {providerEmbedUrl ? (
+          <iframe
+            src={providerEmbedUrl}
+            title={fileName ? `Watch ${fileName}` : 'Video preview'}
+            className="aspect-video w-full bg-black"
+            allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share"
+            allowFullScreen
+          />
+        ) : (
+          <video
+            ref={videoRef}
+            src={playbackUrl}
+            controls
+            playsInline
+            preload="metadata"
+            className="max-h-90 w-full bg-black object-contain"
+            onTimeUpdate={(event) => {
+              // Media-fragment end times are not enforced consistently by
+              // browsers. Stop local playback at the cited evidence boundary
+              // so a supporting clip never continues into unrelated footage.
+              if (hasClipEnd && event.currentTarget.currentTime >= (endSecs as number)) {
+                event.currentTarget.pause();
+                event.currentTarget.currentTime = endSecs as number;
+              }
+            }}
+          />
+        )}
+        <MediaCitationFooter
+          icon={<Film className="size-3 text-muted-foreground" />}
+          fileName={fileName}
+          sourceUrl={timestampedMediaUrl(sourceUrl, startSecs, endSecs)}
+          timestamp={
+            startSecs !== undefined
+              ? `${formatTimestamp(startSecs)}${
+                  endSecs !== undefined ? ` – ${formatTimestamp(endSecs)}` : ''
+                }`
+              : undefined
+          }
+        />
+      </div>
+    );
+  }
+
+  if (mediaType === 'audio') {
+    return (
+      <div className="my-2 w-full max-w-xl overflow-hidden rounded-lg border border-border bg-card">
+        <div className="flex items-center gap-2.5 border-b border-border px-3 py-2">
+          <div className="flex size-7 shrink-0 items-center justify-center rounded-md bg-primary/10 text-primary">
+            <AudioLines className="size-3.5" />
+          </div>
+          <div className="min-w-0 flex-1">
+            <p className="truncate text-[11px] font-medium text-foreground">
+              {fileName ?? 'Audio recording'}
+            </p>
+            {startSecs !== undefined ? (
+              <p className="mt-0.5 text-[10px] tabular-nums text-muted-foreground">
+                {formatTimestamp(startSecs)}
+                {endSecs !== undefined ? ` – ${formatTimestamp(endSecs)}` : ''}
+              </p>
+            ) : (
+              <p className="mt-0.5 text-[10px] text-muted-foreground">Audio reference</p>
+            )}
+          </div>
+        </div>
+        <div className="px-3 py-2">
+          <audio src={playbackUrl} controls preload="metadata" className="h-8 w-full" />
+        </div>
+        {sourceUrl ? (
+          <div className="flex justify-end border-t border-border px-3 py-1.5">
+            <a
+              href={timestampedMediaUrl(sourceUrl, startSecs, endSecs)}
+              target="_blank"
+              rel="noreferrer"
+              className="text-[11px] font-medium text-primary hover:text-primary/80"
+            >
+              Open source
+            </a>
+          </div>
+        ) : null}
+      </div>
+    );
+  }
+
+  if (mediaType === 'frame-preview') {
+    if (imageUnavailable) {
+      return (
+        <div className="my-2 w-full max-w-xl rounded-lg border border-border bg-card p-3">
+          <p className="text-xs text-muted-foreground">This frame could not be extracted.</p>
+          <MediaCitationFooter
+            icon={<Film className="size-3 text-muted-foreground" />}
+            fileName={fileName}
+            sourceUrl={timestampedMediaUrl(sourceUrl, startSecs)}
+            timestamp={startSecs !== undefined ? formatTimestamp(startSecs) : undefined}
+          />
+        </div>
+      );
+    }
+    return (
+      <>
+        <div className="my-2 w-full max-w-xl overflow-hidden rounded-lg border border-border bg-card">
+          <button
+            type="button"
+            onClick={() => setExpanded(true)}
+            className="group relative block w-full overflow-hidden"
+          >
+            <img
+              src={assetUrl}
+              alt={`Frame at ${startSecs !== undefined ? formatTimestamp(startSecs) : 'unknown'}`}
+              className="max-h-105 w-full object-contain"
+              loading="lazy"
+              onError={() => setImageUnavailable(true)}
+            />
+            {startSecs !== undefined && (
+              <div className="absolute bottom-2 left-2 rounded-md bg-black/70 px-2 py-0.5">
+                <span className="text-[11px] font-medium tabular-nums text-white">
+                  {formatTimestamp(startSecs)}
+                </span>
+              </div>
+            )}
+            <div className="absolute inset-0 flex items-center justify-center bg-black/0 opacity-0 transition-all group-hover:bg-black/20 group-hover:opacity-100">
+              <Maximize2 className="size-5 text-white" />
+            </div>
+          </button>
+          <MediaCitationFooter
+            icon={<Film className="size-3 text-muted-foreground" />}
+            fileName={fileName ? `${fileName} — Frame` : 'Video Frame'}
+            sourceUrl={timestampedMediaUrl(sourceUrl, startSecs)}
+            timestamp={startSecs !== undefined ? formatTimestamp(startSecs) : undefined}
+          />
+        </div>
+
+        {expanded && (
+          <div
+            className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-8"
+            onClick={() => setExpanded(false)}
+          >
+            <button
+              type="button"
+              aria-label="Close frame preview"
+              className="absolute right-4 top-4 rounded-full bg-black/50 p-2 text-white hover:bg-black/70"
+              onClick={() => setExpanded(false)}
+            >
+              <X className="size-5" />
+            </button>
+            <img
+              src={assetUrl}
+              alt={`Frame at ${startSecs !== undefined ? formatTimestamp(startSecs) : 'unknown'}`}
+              className="max-h-[80vh] max-w-[90vw] rounded-lg object-contain"
+              onClick={(event) => event.stopPropagation()}
+              onError={() => setImageUnavailable(true)}
+            />
+          </div>
+        )}
+      </>
+    );
+  }
+
+  return null;
+}
+
+/**
+ * Use a first-party embed only for providers with a known safe player. All
+ * other URLs fall back to the locally indexed asset, which also keeps private
+ * uploads and direct video links reliable in chat.
+ */
+export function getProviderEmbedUrl(
+  sourceUrl?: string,
+  startSecs?: number,
+  endSecs?: number,
+): string | undefined {
+  if (!sourceUrl) return undefined;
+
+  try {
+    const url = new URL(sourceUrl);
+    const host = url.hostname.toLowerCase().replace(/^www\./, '');
+    const start = Math.max(0, Math.floor(startSecs ?? 0));
+    const end =
+      Number.isFinite(endSecs) && (endSecs as number) > start ? Math.floor(endSecs!) : undefined;
+
+    const youtubeEmbedUrl = (videoId: string) => {
+      const embed = new URL(`https://www.youtube-nocookie.com/embed/${videoId}`);
+      embed.searchParams.set('start', String(start));
+      embed.searchParams.set('rel', '0');
+      if (end !== undefined) embed.searchParams.set('end', String(end));
+      return embed.toString();
+    };
+
+    if (host === 'youtu.be' || host.endsWith('.youtu.be')) {
+      const videoId = url.pathname.split('/').filter(Boolean)[0];
+      return videoId ? youtubeEmbedUrl(videoId) : undefined;
+    }
+
+    if (host === 'youtube.com' || host.endsWith('.youtube.com')) {
+      const pathParts = url.pathname.split('/').filter(Boolean);
+      const videoId =
+        url.searchParams.get('v') ||
+        (['embed', 'shorts', 'live'].includes(pathParts[0]) ? pathParts[1] : undefined);
+      return videoId ? youtubeEmbedUrl(videoId) : undefined;
+    }
+
+    if (host === 'vimeo.com' || host.endsWith('.vimeo.com')) {
+      const videoId = url.pathname.split('/').find((part) => /^\d+$/.test(part));
+      return videoId ? `https://player.vimeo.com/video/${videoId}#t=${start}s` : undefined;
+    }
+  } catch {
+    // The local media URL remains the reliable fallback for malformed or
+    // unsupported provider URLs.
+  }
+
+  return undefined;
+}
+
+/** Build a provider-aware source link that opens at the cited evidence. */
+export function timestampedMediaUrl(
+  sourceUrl?: string,
+  startSecs?: number,
+  endSecs?: number,
+): string | undefined {
+  if (!sourceUrl || !Number.isFinite(startSecs)) return sourceUrl;
+  const start = Math.max(0, Math.floor(startSecs as number));
+  const end =
+    Number.isFinite(endSecs) && (endSecs as number) > start ? Math.floor(endSecs as number) : null;
+  try {
+    const url = new URL(sourceUrl, 'http://larkup.local');
+    const host = url.hostname.toLowerCase().replace(/^www\./, '');
+    if (host === 'youtube.com' || host.endsWith('.youtube.com') || host === 'youtu.be') {
+      url.searchParams.set('t', `${start}s`);
+      return url.origin === 'http://larkup.local'
+        ? `${url.pathname}${url.search}${url.hash}`
+        : url.toString();
+    }
+    const fragment = `t=${start}${end !== null ? `,${end}` : ''}`;
+    url.hash = fragment;
+    return url.origin === 'http://larkup.local'
+      ? `${url.pathname}${url.search}${url.hash}`
+      : url.toString();
+  } catch {
+    return `${sourceUrl.split('#')[0]}#t=${start}${end !== null ? `,${end}` : ''}`;
+  }
+}
+
+function MediaCitationFooter({
+  icon,
+  fileName,
+  sourceUrl,
+  timestamp,
+}: {
+  icon: React.ReactNode;
+  fileName?: string;
+  sourceUrl?: string;
+  timestamp?: string;
+}) {
+  if (!fileName && !sourceUrl && !timestamp) return null;
+
+  return (
+    <div className="flex items-center justify-between gap-3 border-t border-border px-3 py-1.5">
+      <div className="flex min-w-0 items-center gap-1.5">
+        {icon}
+        <span className="truncate text-[11px] text-muted-foreground">
+          {fileName ?? 'Media citation'}
+          {timestamp ? ` · ${timestamp}` : ''}
+        </span>
+      </div>
+      {sourceUrl ? (
+        <a
+          href={sourceUrl}
+          target="_blank"
+          rel="noreferrer"
+          className="shrink-0 text-[11px] font-medium text-primary hover:text-primary/80"
+        >
+          Open source
+        </a>
+      ) : null}
+    </div>
+  );
+}
+
+/* Media preview helpers. */
+/* Markdown media reference parser                                     */
+/* End media preview helpers. */
+
+/**
+ * Parse media references from assistant text.
+ *
+ * Supported formats:
+ * - [IMAGE_REF:assetId]
+ * - [IMAGE_REF:assetId:fileName]
+ * - [VIDEO_REF:assetId:startSecs-endSecs]
+ * - [AUDIO_REF:assetId]
+ */
+const MEDIA_REF_REGEX = /\[(IMAGE_REF|VIDEO_REF|AUDIO_REF):([a-f0-9-]+)(?::([^\]]*))?\]/g;
+
+export interface ParsedMediaRef {
+  fullMatch: string;
+  type: 'image' | 'video' | 'audio';
+  assetId: string;
+  extra?: string;
+  startSecs?: number;
+  endSecs?: number;
+}
+
+export function parseMediaRefs(text: string): ParsedMediaRef[] {
+  const refs: ParsedMediaRef[] = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = MEDIA_REF_REGEX.exec(text)) !== null) {
+    const [fullMatch, refType, assetId, extra] = match;
+    const type = refType === 'IMAGE_REF' ? 'image' : refType === 'VIDEO_REF' ? 'video' : 'audio';
+
+    let startSecs: number | undefined;
+    let endSecs: number | undefined;
+
+    if ((type === 'video' || type === 'audio') && extra?.includes('-')) {
+      const [s, e] = extra.split('-').map(Number);
+      if (!isNaN(s)) startSecs = s;
+      if (!isNaN(e)) endSecs = e;
+    }
+
+    refs.push({ fullMatch, type, assetId, extra, startSecs, endSecs });
+  }
+
+  return refs;
+}
+
+function formatTimestamp(secs: number): string {
+  const m = Math.floor(secs / 60);
+  const s = Math.floor(secs % 60);
+  return `${m}:${String(s).padStart(2, '0')}`;
+}

package/src/components/chat/tools/chat-sandbox-result.tsx

@@ -0,0 +1,130 @@
+'use client';
+
+import { useMemo } from 'react';
+import { ChevronDown, Play, Clock, Image as ImageIcon, Table2 } from 'lucide-react';
+import { ChatDataTable, type DataTableConfig } from './chat-data-table';
+import { Card, CardContent } from '../../ui/card';
+
+interface SandboxArtifact {
+  name: string;
+  mimeType: string;
+  data: string; // base64
+}
+
+export interface SandboxResultConfig {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  artifacts: SandboxArtifact[];
+  executionTimeMs: number;
+}
+
+function tryParseStdoutAsTable(stdout: string): DataTableConfig | null {
+  return null;
+}
+
+export function ChatSandboxResult({ config }: { config: SandboxResultConfig }) {
+  const { stdout, stderr, exitCode, artifacts, executionTimeMs } = config;
+
+  const images = useMemo(
+    () => artifacts.filter((a) => a.mimeType.startsWith('image/')),
+    [artifacts],
+  );
+
+  // Try to parse stdout as structured table data
+  const stdoutTable = useMemo(() => tryParseStdoutAsTable(stdout), [stdout]);
+
+  const isSuccess = exitCode === 0;
+  const isRuntimeUnavailable =
+    /docker\.sock|docker (?:isn't|is not) running|code execution is unavailable|needs python 3/i.test(
+      stderr,
+    );
+
+  return (
+    <Card className="overflow-hidden  animate-in fade-in slide-in-from-bottom-2 duration-500">
+      {/* A successful result speaks for itself; retain a clear header only for failures. */}
+      {!isSuccess && (
+        <div className="flex items-center justify-between border-b border-border/40 px-4 py-2.5">
+          <div className="flex items-center gap-2">
+            <div className="flex h-6 w-6 items-center justify-center rounded-md bg-red-100 text-red-600 dark:bg-red-900/30 dark:text-red-400">
+              <Play className="size-3" />
+            </div>
+            <span className="text-xs font-medium text-foreground">Analysis failed</span>
+          </div>
+          <span className="flex items-center gap-1 text-[10px] tabular-nums text-muted-foreground">
+            <Clock className="size-3" />
+            {executionTimeMs < 1000
+              ? `${executionTimeMs}ms`
+              : `${(executionTimeMs / 1000).toFixed(1)}s`}
+          </span>
+        </div>
+      )}
+
+      {/* Generated images (matplotlib charts, etc.) */}
+      {images.length > 0 && (
+        <div className="space-y-3 p-4">
+          {images.map((img, i) => (
+            <div key={i} className="overflow-hidden rounded-lg border border-border/40">
+              <img
+                src={`data:${img.mimeType};base64,${img.data}`}
+                alt={img.name}
+                className="w-full"
+              />
+              <div className="flex items-center gap-2 border-t border-border/40 px-3 py-1.5">
+                <ImageIcon className="size-3 text-muted-foreground" />
+                <span className="text-[10px] text-muted-foreground">{img.name}</span>
+              </div>
+            </div>
+          ))}
+        </div>
+      )}
+
+      {/* Structured stdout (rendered as table) */}
+      {stdoutTable && (
+        <CardContent className="p-3 pt-3">
+          <div className="flex items-center gap-1.5 mb-2 text-xs text-muted-foreground">
+            <Table2 className="size-3" />
+            <span className="font-medium">Analysis Results</span>
+          </div>
+          <ChatDataTable config={stdoutTable} />
+        </CardContent>
+      )}
+
+      {/* Plain stdout (only if not parsed as table) */}
+      {stdout && !stdoutTable && (
+        <div className="border-b border-border/40 px-4 py-3">
+          <pre className="max-h-48 overflow-auto whitespace-pre-wrap text-xs leading-relaxed text-foreground [&::-webkit-scrollbar]:hidden">
+            {stdout}
+          </pre>
+        </div>
+      )}
+
+      {/* Error display */}
+      {!isSuccess && (
+        <div className="border-t border-border/40 bg-red-50/50 p-4 dark:bg-red-900/10">
+          <div className="flex flex-col gap-2">
+            <span className="text-sm font-medium text-red-800 dark:text-red-400">
+              {isRuntimeUnavailable ? 'Analysis is unavailable' : 'Analysis encountered an issue'}
+            </span>
+            <span className="text-xs text-red-600/80 dark:text-red-400/80">
+              {isRuntimeUnavailable
+                ? 'This workspace does not have a code-analysis runtime configured.'
+                : 'This calculation could not be completed.'}
+            </span>
+            {stderr && !isRuntimeUnavailable && (
+              <details className="mt-2 group">
+                <summary className="cursor-pointer text-xs font-medium text-red-600/70 hover:text-red-600 dark:text-red-400/70 dark:hover:text-red-400 transition-colors list-none flex items-center gap-1">
+                  <ChevronDown className="size-3 transition-transform group-open:rotate-180" />
+                  Technical Details
+                </summary>
+                <pre className="mt-2 max-h-40 overflow-auto whitespace-pre-wrap rounded bg-red-100/50 p-3 text-[11px] leading-relaxed text-red-900/70 dark:bg-red-900/20 dark:text-red-200/70 [&::-webkit-scrollbar]:hidden">
+                  {stderr}
+                </pre>
+              </details>
+            )}
+          </div>
+        </div>
+      )}
+    </Card>
+  );
+}