@puredesktop/puredesktop-ui-bridge 1.0.0-beta.1

package/src/components/agent-feedback/ScreenAnnotationAgentDialog.tsx

@@ -0,0 +1,509 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { styled } from 'styled-components'
+import {
+  callAgentRunModel,
+  createAgentSession,
+  startAgentRun,
+} from '../../bridge/agents.js'
+import { Button } from '../common/buttons/Button.js'
+import { InlineBanner } from '../common/feedback/InlineBanner.js'
+import { TextAreaField } from '../common/inputs/TextAreaField.js'
+import { Modal } from '../common/overlays/Modal.js'
+import { Text } from '../common/typography/Text.js'
+
+export interface ScreenAnnotationAgentPayload {
+  note: string
+  imageDataUrl: string
+}
+
+export interface ScreenAnnotationAgentDialogProps {
+  open: boolean
+  appSlug: string
+  contextLabel?: string
+  context?: string
+  title?: string
+  onClose: () => void
+  onSent?: (sessionId: string) => void
+  onCapture?: (payload: ScreenAnnotationAgentPayload) => void | Promise<void>
+  submitLabel?: string
+  buildMessage?: (payload: ScreenAnnotationAgentPayload) => string
+}
+
+const MAX_MODEL_STEPS = 3
+type AnnotationStage = 'empty' | 'select' | 'draw'
+type Rect = { x: number; y: number; width: number; height: number }
+
+const StyledBody = styled.div`
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) 220px;
+  gap: var(--platform-spacing-sm);
+  min-height: 0;
+
+  @media (max-width: 860px) {
+    grid-template-columns: minmax(0, 1fr);
+  }
+`
+
+const StyledCapturePane = styled.div`
+  display: flex;
+  min-width: 0;
+  min-height: 260px;
+  align-items: center;
+  justify-content: center;
+  overflow: auto;
+  border: 1px solid var(--platform-colors-border);
+  border-radius: var(--platform-radius-sm);
+  background: var(--platform-colors-surface);
+`
+
+const StyledCanvas = styled.canvas`
+  display: block;
+  width: auto;
+  height: auto;
+  max-width: 100%;
+  max-height: min(54vh, 560px);
+  cursor: crosshair;
+
+  &[hidden] {
+    display: none;
+  }
+`
+
+const StyledToolTitle = styled(Text)`
+  margin: 0;
+  font-size: var(--platform-typography-font-size-xs);
+  font-weight: 700;
+  letter-spacing: 0.06em;
+  text-transform: uppercase;
+`
+
+const StyledEmpty = styled.div`
+  display: flex;
+  max-width: 24ch;
+  flex-direction: column;
+  gap: var(--platform-spacing-sm);
+  padding: var(--platform-spacing-md);
+  text-align: center;
+`
+
+const StyledSide = styled.div`
+  display: flex;
+  flex-direction: column;
+  gap: var(--platform-spacing-sm);
+  min-width: 0;
+`
+
+const StyledActions = styled.div`
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: flex-end;
+  gap: var(--platform-spacing-sm);
+`
+
+const StyledNote = styled(TextAreaField)`
+  min-height: 96px;
+`
+
+const StyledStatus = styled(Text)`
+  margin: 0;
+`
+
+function canvasPoint(
+  canvas: HTMLCanvasElement,
+  event: React.PointerEvent<HTMLCanvasElement>,
+): { x: number; y: number } {
+  const rect = canvas.getBoundingClientRect()
+  return {
+    x: ((event.clientX - rect.left) / rect.width) * canvas.width,
+    y: ((event.clientY - rect.top) / rect.height) * canvas.height,
+  }
+}
+
+function drawLine(
+  canvas: HTMLCanvasElement,
+  from: { x: number; y: number },
+  to: { x: number; y: number },
+): void {
+  const context = canvas.getContext('2d')
+  if (!context) return
+  context.save()
+  context.lineWidth = Math.max(4, Math.min(canvas.width, canvas.height) * 0.005)
+  context.lineCap = 'round'
+  context.lineJoin = 'round'
+  context.strokeStyle = '#24272b'
+  context.beginPath()
+  context.moveTo(from.x, from.y)
+  context.lineTo(to.x, to.y)
+  context.stroke()
+  context.restore()
+}
+
+function normalizeRect(start: { x: number; y: number }, end: { x: number; y: number }): Rect {
+  return {
+    x: Math.min(start.x, end.x),
+    y: Math.min(start.y, end.y),
+    width: Math.abs(end.x - start.x),
+    height: Math.abs(end.y - start.y),
+  }
+}
+
+function defaultSelection(canvas: HTMLCanvasElement): Rect {
+  const width = canvas.width * 0.62
+  const height = canvas.height * 0.62
+  return {
+    x: (canvas.width - width) / 2,
+    y: (canvas.height - height) / 2,
+    width,
+    height,
+  }
+}
+
+function drawSelectionOverlay(canvas: HTMLCanvasElement, source: HTMLCanvasElement, rect: Rect): void {
+  const context = canvas.getContext('2d')
+  if (!context) return
+  canvas.width = source.width
+  canvas.height = source.height
+  context.drawImage(source, 0, 0)
+  context.save()
+  context.fillStyle = 'rgba(255, 255, 255, 0.58)'
+  context.fillRect(0, 0, canvas.width, canvas.height)
+  context.clearRect(rect.x, rect.y, rect.width, rect.height)
+  context.drawImage(
+    source,
+    rect.x,
+    rect.y,
+    rect.width,
+    rect.height,
+    rect.x,
+    rect.y,
+    rect.width,
+    rect.height,
+  )
+  context.strokeStyle = '#24272b'
+  context.lineWidth = Math.max(2, Math.min(canvas.width, canvas.height) * 0.002)
+  context.setLineDash([8, 6])
+  context.strokeRect(rect.x, rect.y, rect.width, rect.height)
+  context.setLineDash([])
+  context.fillStyle = '#24272b'
+  context.fillRect(rect.x + rect.width - 8, rect.y + rect.height - 8, 8, 8)
+  context.restore()
+}
+
+function cropCanvas(source: HTMLCanvasElement, rect: Rect): HTMLCanvasElement {
+  const canvas = document.createElement('canvas')
+  canvas.width = Math.max(1, Math.round(rect.width))
+  canvas.height = Math.max(1, Math.round(rect.height))
+  const context = canvas.getContext('2d')
+  if (!context) throw new Error('Could not prepare the annotation crop.')
+  context.drawImage(
+    source,
+    rect.x,
+    rect.y,
+    rect.width,
+    rect.height,
+    0,
+    0,
+    canvas.width,
+    canvas.height,
+  )
+  return canvas
+}
+
+async function captureDisplayFrame(): Promise<HTMLCanvasElement> {
+  if (!navigator.mediaDevices?.getDisplayMedia) {
+    throw new Error('Screen capture is not available in this environment.')
+  }
+
+  const stream = await navigator.mediaDevices.getDisplayMedia({
+    video: true,
+    audio: false,
+  })
+  try {
+    const video = document.createElement('video')
+    video.muted = true
+    video.srcObject = stream
+    await video.play()
+    await new Promise<void>(resolve => {
+      if (video.videoWidth > 0 && video.videoHeight > 0) {
+        resolve()
+        return
+      }
+      video.onloadedmetadata = () => resolve()
+    })
+
+    const canvas = document.createElement('canvas')
+    canvas.width = video.videoWidth
+    canvas.height = video.videoHeight
+    const context = canvas.getContext('2d')
+    if (!context) throw new Error('Could not prepare the screenshot.')
+    context.drawImage(video, 0, 0, canvas.width, canvas.height)
+    return canvas
+  } finally {
+    stream.getTracks().forEach(track => track.stop())
+  }
+}
+
+function defaultMessage({
+  note,
+  imageDataUrl,
+}: ScreenAnnotationAgentPayload): string {
+  return [
+    'Please use this annotated screenshot to make the requested UI change.',
+    '',
+    note.trim() ? `Note from the user:\n${note.trim()}` : 'No extra note.',
+    '',
+    'Annotated screenshot:',
+    '',
+    `![Annotated screenshot](${imageDataUrl})`,
+  ].join('\n')
+}
+
+export function ScreenAnnotationAgentDialog({
+  open,
+  appSlug,
+  contextLabel = 'current screen',
+  context,
+  title = 'Annotation',
+  onClose,
+  onSent,
+  onCapture,
+  submitLabel,
+  buildMessage = defaultMessage,
+}: ScreenAnnotationAgentDialogProps): React.ReactElement {
+  const canvasRef = useRef<HTMLCanvasElement | null>(null)
+  const baseCanvasRef = useRef<HTMLCanvasElement | null>(null)
+  const lastPointRef = useRef<{ x: number; y: number } | null>(null)
+  const [captured, setCaptured] = useState(false)
+  const [note, setNote] = useState('')
+  const [error, setError] = useState<string | null>(null)
+  const [status, setStatus] = useState('')
+  const [busy, setBusy] = useState(false)
+
+  const paintBase = useCallback((source: HTMLCanvasElement) => {
+    const canvas = canvasRef.current
+    const context2d = canvas?.getContext('2d')
+    if (!canvas || !context2d) return
+    canvas.width = source.width
+    canvas.height = source.height
+    context2d.drawImage(source, 0, 0)
+    setCaptured(true)
+  }, [])
+
+  useEffect(() => {
+    if (!open) {
+      setNote('')
+      setError(null)
+      setStatus('')
+      setCaptured(false)
+      baseCanvasRef.current = null
+      lastPointRef.current = null
+    }
+  }, [open])
+
+  const handleCapture = useCallback(async () => {
+    setBusy(true)
+    setError(null)
+    setStatus('Choose the screen or window to capture.')
+    try {
+      const canvas = await captureDisplayFrame()
+      baseCanvasRef.current = canvas
+      paintBase(canvas)
+      setStatus('Draw on the screenshot, then send.')
+    } catch (cause) {
+      setError(cause instanceof Error ? cause.message : String(cause))
+      setStatus('')
+    } finally {
+      setBusy(false)
+    }
+  }, [paintBase])
+
+  const handleClearMarks = useCallback(() => {
+    const base = baseCanvasRef.current
+    if (!base) return
+    paintBase(base)
+    setStatus('Marks cleared.')
+  }, [paintBase])
+
+  const handlePointerDown = useCallback(
+    (event: React.PointerEvent<HTMLCanvasElement>) => {
+      if (!captured) return
+      event.currentTarget.setPointerCapture(event.pointerId)
+      lastPointRef.current = canvasPoint(event.currentTarget, event)
+    },
+    [captured],
+  )
+
+  const handlePointerMove = useCallback(
+    (event: React.PointerEvent<HTMLCanvasElement>) => {
+      if (!captured || !lastPointRef.current) return
+      const next = canvasPoint(event.currentTarget, event)
+      drawLine(event.currentTarget, lastPointRef.current, next)
+      lastPointRef.current = next
+    },
+    [captured],
+  )
+
+  const handlePointerUp = useCallback(
+    (event: React.PointerEvent<HTMLCanvasElement>) => {
+      if (event.currentTarget.hasPointerCapture(event.pointerId)) {
+        event.currentTarget.releasePointerCapture(event.pointerId)
+      }
+      lastPointRef.current = null
+    },
+    [],
+  )
+
+  const handleSend = useCallback(async () => {
+    const canvas = canvasRef.current
+    if (!canvas || !captured) return
+    setBusy(true)
+    setError(null)
+    setStatus(onCapture ? 'Adding annotation…' : 'Sending annotation…')
+    try {
+      const imageDataUrl = canvas.toDataURL('image/png')
+      if (onCapture) {
+        await onCapture({ note, imageDataUrl })
+        setStatus('Annotation added.')
+        onClose()
+        return
+      }
+      const message = [
+        buildMessage({ note, imageDataUrl }),
+        context
+          ? `\n\nContext (${contextLabel}):\n${context}`
+          : `\n\nContext: ${contextLabel}.`,
+      ].join('')
+      const record = await createAgentSession({
+        scope: 'app',
+        appSlug,
+        title: `Screenshot note · ${contextLabel}`,
+      })
+      const session = {
+        scope: 'app' as const,
+        appSlug,
+        sessionId: record.sessionId,
+      }
+      let snapshot = await startAgentRun({ session, message })
+      for (let step = 0; step < MAX_MODEL_STEPS; step += 1) {
+        if (snapshot.state !== 'ready') break
+        snapshot = await callAgentRunModel({
+          runId: snapshot.runId,
+          sessionId: record.sessionId,
+        })
+      }
+      setStatus('Annotation sent.')
+      onSent?.(record.sessionId)
+      onClose()
+    } catch (cause) {
+      setError(cause instanceof Error ? cause.message : String(cause))
+      setStatus('')
+    } finally {
+      setBusy(false)
+    }
+  }, [
+    appSlug,
+    buildMessage,
+    captured,
+    context,
+    contextLabel,
+    note,
+    onCapture,
+    onClose,
+    onSent,
+  ])
+
+  return (
+    <Modal open={open} onClose={onClose} title={title} size="lg" chrome>
+      <StyledBody>
+        <StyledCapturePane>
+          <StyledCanvas
+            ref={canvasRef}
+            hidden={!captured}
+            aria-label="Annotated screenshot"
+            onPointerDown={handlePointerDown}
+            onPointerMove={handlePointerMove}
+            onPointerCancel={handlePointerUp}
+            onPointerUp={handlePointerUp}
+          />
+          {!captured ? (
+            <StyledEmpty>
+              <Text as="p" size="base">
+                Capture, mark, send.
+              </Text>
+              <Button
+                type="button"
+                variant="subtle"
+                size="sm"
+                loading={busy}
+                onClick={() => void handleCapture()}
+              >
+                Capture screen
+              </Button>
+            </StyledEmpty>
+          ) : null}
+        </StyledCapturePane>
+        <StyledSide>
+          <Text as="p" size="sm" tone="secondary">
+            Draw with the pen. Add a note if the mark needs context.
+          </Text>
+          <StyledNote
+            value={note}
+            rows={4}
+            disabled={busy}
+            placeholder="Annotation"
+            onChange={event => setNote(event.target.value)}
+          />
+          {status ? (
+            <StyledStatus as="p" size="sm" tone="secondary">
+              {status}
+            </StyledStatus>
+          ) : null}
+          {error ? <InlineBanner variant="danger">{error}</InlineBanner> : null}
+          <StyledActions>
+            <Button
+              type="button"
+              variant="text"
+              size="sm"
+              disabled={busy}
+              onClick={onClose}
+            >
+              Cancel
+            </Button>
+            {captured ? (
+              <>
+                <Button
+                  type="button"
+                  variant="text"
+                  size="sm"
+                  disabled={busy}
+                  onClick={() => void handleCapture()}
+                >
+                  Recapture
+                </Button>
+                <Button
+                  type="button"
+                  variant="text"
+                  size="sm"
+                  disabled={busy}
+                  onClick={handleClearMarks}
+                >
+                  Clear marks
+                </Button>
+              </>
+            ) : null}
+            <Button
+              type="button"
+              variant="primary"
+              size="md"
+              loading={busy}
+              disabled={!captured || busy}
+              onClick={() => void handleSend()}
+            >
+              {submitLabel ?? 'Send'}
+            </Button>
+          </StyledActions>
+        </StyledSide>
+      </StyledBody>
+    </Modal>
+  )
+}

package/src/components/agent-feedback/index.ts

@@ -0,0 +1,5 @@
+export {
+  ScreenAnnotationAgentDialog,
+  type ScreenAnnotationAgentDialogProps,
+  type ScreenAnnotationAgentPayload,
+} from './ScreenAnnotationAgentDialog.js'

package/src/components/agents/AgentComposer.tsx

@@ -0,0 +1,34 @@
+import { ChatBox } from '../common/chat/ChatBox.js'
+
+export interface AgentComposerProps {
+  value: string
+  disabled?: boolean
+  busy?: boolean
+  leading?: React.ReactNode
+  onChange: (value: string) => void
+  onSend: () => void
+  onStop?: () => void
+}
+
+export function AgentComposer({
+  value,
+  disabled = false,
+  busy = false,
+  leading,
+  onChange,
+  onSend,
+  onStop,
+}: AgentComposerProps): React.ReactElement {
+  return (
+    <ChatBox
+      value={value}
+      disabled={disabled}
+      sending={busy}
+      placeholder="Message the agent…"
+      leading={leading}
+      onChange={onChange}
+      onSend={onSend}
+      onStop={onStop}
+    />
+  )
+}

package/src/components/agents/AgentContextUsageBar.tsx

@@ -0,0 +1,127 @@
+import { styled } from 'styled-components'
+import { contextUsagePercent } from '../../agents/runtime/contextWindowUsage.js'
+import type { AgentLiveTurn, AgentUiRunUsage } from './agentTypes.js'
+
+export function formatAgentTokenCount(value: number): string {
+  if (value >= 1_000_000) {
+    const millions = value / 1_000_000
+    const rounded =
+      millions >= 10
+        ? Math.round(millions)
+        : Number(millions.toFixed(1).replace(/\.0$/, ''))
+    return `${rounded}M`
+  }
+  if (value >= 1000) {
+    return `${Math.round(value / 1000)}k`
+  }
+  return String(value)
+}
+
+export interface AgentContextUsageBarProps {
+  usage?: AgentUiRunUsage | null
+  liveTurn?: AgentLiveTurn | null
+}
+
+const DONUT_SIZE = 12
+const DONUT_STROKE = 2
+const DONUT_RADIUS = (DONUT_SIZE - DONUT_STROKE) / 2
+const DONUT_CIRCUMFERENCE = 2 * Math.PI * DONUT_RADIUS
+
+export function AgentContextUsageBar({
+  usage,
+  liveTurn,
+}: AgentContextUsageBarProps): React.ReactElement | null {
+  if (liveTurn?.contextCompaction === 'summarizing') {
+    return (
+      <Root aria-label="Summarizing context" $active>
+        Summarizing context…
+      </Root>
+    )
+  }
+
+  const hasUsage = usage && (usage.totalTokens > 0 || usage.contextFillTokens)
+  if (!hasUsage) return null
+
+  const prefix = usage?.estimated ? '~' : ''
+  const fill = usage?.contextFillTokens ?? usage?.totalTokens ?? 0
+  const total = fill > 0 ? formatAgentTokenCount(fill) : null
+  const percent = usage ? contextUsagePercent(usage) : undefined
+  const label =
+    total && usage?.contextWindowTokens
+      ? `${prefix}${total} / ${formatAgentTokenCount(usage.contextWindowTokens)}`
+      : total
+        ? `${prefix}${total} tokens`
+        : null
+
+  if (!label) return null
+
+  return (
+    <Root
+      aria-label={
+        percent != null ? `Context usage ${percent}%` : 'Context usage'
+      }
+    >
+      {percent != null ? (
+        <Donut
+          width={DONUT_SIZE}
+          height={DONUT_SIZE}
+          viewBox={`0 0 ${DONUT_SIZE} ${DONUT_SIZE}`}
+          aria-hidden
+        >
+          <DonutTrack
+            cx={DONUT_SIZE / 2}
+            cy={DONUT_SIZE / 2}
+            r={DONUT_RADIUS}
+          />
+          <DonutFill
+            cx={DONUT_SIZE / 2}
+            cy={DONUT_SIZE / 2}
+            r={DONUT_RADIUS}
+            $percent={percent}
+          />
+        </Donut>
+      ) : null}
+      <Label>{label}</Label>
+    </Root>
+  )
+}
+
+const Root = styled.span<{ $active?: boolean }>`
+  display: inline-flex;
+  align-items: center;
+  gap: 5px;
+  color: var(--platform-colors-text-disabled);
+  font-size: 11px;
+  line-height: 1.4;
+  white-space: nowrap;
+  font-style: ${({ $active }) => ($active ? 'italic' : 'normal')};
+`
+
+const Donut = styled.svg`
+  flex: none;
+`
+
+const donutRing = `
+  fill: none;
+  stroke-width: ${DONUT_STROKE};
+`
+
+const DonutTrack = styled.circle`
+  ${donutRing}
+  stroke: var(--platform-colors-border);
+`
+
+const DonutFill = styled.circle<{ $percent: number }>`
+  ${donutRing}
+  stroke: var(--platform-colors-accent);
+  stroke-dasharray: ${DONUT_CIRCUMFERENCE};
+  stroke-dashoffset: ${({ $percent }) =>
+    DONUT_CIRCUMFERENCE * (1 - $percent / 100)};
+  stroke-linecap: round;
+  transform: rotate(-90deg);
+  transform-origin: center;
+`
+
+const Label = styled.span`
+  line-height: 1.4;
+`