@puredesktop/puredesktop-ui-bridge 1.0.0-beta.1 → 2.1.0

package/src/bridge/screenAnnotation.ts

@@ -1,269 +1,269 @@
-import {
-  callAgentRunModel,
-  createAgentSession,
-  resolveAgentRunToolCalls,
-  startAgentRun,
-} from './agents.js'
-import { startScreenCapture } from './window.mjs'
-import type {
-  PlatformScreenCaptureOption,
-  PlatformScreenCaptureRect,
-  PlatformScreenCaptureResult,
-} from './window.mjs'
-
-/**
- * Shared screen-annotation capture → agent flow. Any app can call
- * {@link captureAndSendToAgent}: it opens the shell's chromeless capture
- * overlay, resolves the text under the selection (content mode), and starts an
- * agent run with the annotated image attached. App-specific wording and tools
- * are supplied via `buildMessage`.
- *
- * Requires the `agents` permission in plugin.json.
- */
-
-const DEFAULT_MAX_STEPS = 8
-
-export interface TextCandidate {
-  text: string
-  rect: PlatformScreenCaptureRect
-}
-
-function rectsIntersect(
-  a: PlatformScreenCaptureRect,
-  b: PlatformScreenCaptureRect,
-): boolean {
-  return (
-    a.x < b.x + b.width &&
-    a.x + a.width > b.x &&
-    a.y < b.y + b.height &&
-    a.y + a.height > b.y
-  )
-}
-
-/**
- * Pure core: pick the text of candidates whose rect overlaps the selection, in
- * order, deduped + whitespace-normalized. Separated from the DOM so it is
- * unit-testable (jsdom has no layout).
- */
-export function selectTextFromCandidates(
-  candidates: TextCandidate[],
-  selection: PlatformScreenCaptureRect,
-): string {
-  const seen = new Set<string>()
-  const parts: string[] = []
-  for (const candidate of candidates) {
-    if (!rectsIntersect(candidate.rect, selection)) continue
-    const text = candidate.text.replace(/\s+/g, ' ').trim()
-    if (!text || seen.has(text)) continue
-    seen.add(text)
-    parts.push(text)
-  }
-  return parts.join('\n\n')
-}
-
-const DEFAULT_BLOCK_SELECTOR =
-  'p, li, h1, h2, h3, h4, h5, h6, blockquote, figcaption, td, th, pre, dt, dd'
-
-export interface ResolveSelectionTextOptions {
-  /** Root to search within. Defaults to the active editor / contenteditable. */
-  root?: ParentNode | null
-  /** Block-level selector to collect. */
-  selector?: string
-}
-
-/**
- * Resolve the text that intersects an app-local selection rect (CSS px relative
- * to the app's own document viewport). Reads block elements inside the editor
- * surface; returns '' when nothing meaningful overlaps.
- */
-export function resolveSelectionText(
-  selection: PlatformScreenCaptureRect,
-  options: ResolveSelectionTextOptions = {},
-): string {
-  const root =
-    options.root ??
-    document.querySelector('.ProseMirror, [contenteditable="true"]')
-  if (!root) return ''
-  const selector = options.selector ?? DEFAULT_BLOCK_SELECTOR
-  const blocks = Array.from(root.querySelectorAll<HTMLElement>(selector))
-  const candidates: TextCandidate[] = blocks.map(element => {
-    const r = element.getBoundingClientRect()
-    return {
-      text: element.textContent ?? '',
-      rect: { x: r.left, y: r.top, width: r.width, height: r.height },
-    }
-  })
-  return selectTextFromCandidates(candidates, selection)
-}
-
-export interface ScreenAnnotationResult {
-  capture: PlatformScreenCaptureResult
-  /** Text resolved under the selection (content mode); '' in vibe mode. */
-  resolvedText: string
-}
-
-/** In-progress + terminal states a caller can reflect with quiet UI. */
-export type CaptureAgentStatus =
-  | 'working'
-  | 'applied'
-  | 'no-changes'
-  | 'needs-approval'
-  | 'error'
-
-export interface CaptureAgentOutcome {
-  sessionId: string | null
-  status: 'applied' | 'no-changes' | 'needs-approval' | 'error' | 'cancelled'
-  error?: string
-}
-
-export interface CaptureAndSendOptions {
-  appSlug: string
-  /** What this capture transforms, shown in the overlay (e.g. "book content", "page design"). */
-  label?: string
-  /** Contextual option controls rendered in the overlay (e.g. design scope). */
-  options?: PlatformScreenCaptureOption[]
-  /** Build the agent user message from the capture + resolved text. */
-  buildMessage?: (result: ScreenAnnotationResult) => string
-  /** Session title (string or builder). */
-  title?: string | ((result: ScreenAnnotationResult) => string)
-  /** Resolve text under the selection in content mode. Defaults to {@link resolveSelectionText}. */
-  resolveText?: (rect: PlatformScreenCaptureRect) => string
-  /** Total model + tool-resolution turns to drive. Default 8. */
-  maxSteps?: number
-  /** Progress callback for quiet UI: 'working' first, then a terminal status. */
-  onStatus?: (status: CaptureAgentStatus) => void
-}
-
-/**
- * Guidance for interpreting pen marks. The marks (circles, underlines, arrows)
- * indicate the focus; their scope is a judgment call the agent must make.
- */
-export function penFocusGuidance(): string {
-  return [
-    'The user drew marks on the screenshot — focus on what they marked, not the whole captured region.',
-    'Marks may circle, underline, or point at content. Use judgment about scope: the user may mean exactly the marked words, the sentence that contains them, or the whole paragraph/section.',
-    'Infer the most sensible scope from the marks and the note together. If it is genuinely ambiguous, prefer the smallest coherent unit (usually the sentence) unless the note implies a larger change.',
-  ].join('\n')
-}
-
-/** Generic fallback message when an app doesn't supply `buildMessage`. */
-export function defaultAnnotationMessage({
-  capture,
-  resolvedText,
-}: ScreenAnnotationResult): string {
-  const note = capture.note.trim()
-  return [
-    capture.mode === 'content'
-      ? 'The user captured and annotated a region of this app and wants a change applied there.'
-      : 'The user captured and annotated a region of the app UI (vibe mode).',
-    '',
-    note ? `Note from the user:\n${note}` : 'No extra note.',
-    capture.hasMarks ? `\n${penFocusGuidance()}` : '',
-    resolvedText
-      ? `\nText in the captured region (for reference — the marks show the focus):\n"""\n${resolvedText}\n"""`
-      : '',
-    '',
-    'Use the attached annotated screenshot (with the user’s drawing) to make the change.',
-  ]
-    .filter(line => line !== '')
-    .join('\n')
-}
-
-interface DataUrlImage {
-  mediaType: string
-  dataBase64: string
-}
-
-function splitDataUrl(dataUrl: string): DataUrlImage | null {
-  const match = /^data:([^;]+);base64,(.+)$/s.exec(dataUrl)
-  if (!match) return null
-  return { mediaType: match[1], dataBase64: match[2] }
-}
-
-/**
- * Open the capture overlay and, on send, run an agent end to end: call the
- * model, then route + resolve its tool calls so the edit is actually applied
- * (not left pending). Content mode targets the app's own agent (scope 'app');
- * vibe mode targets a global agent. `onStatus` reports progress so the caller
- * can show quiet feedback. Returns an outcome describing what happened.
- */
-export async function captureAndSendToAgent(
-  options: CaptureAndSendOptions,
-): Promise<CaptureAgentOutcome> {
-  const capture = await startScreenCapture({
-    label: options.label,
-    options: options.options,
-  })
-  if (!capture) return { sessionId: null, status: 'cancelled' }
-
-  const resolveText = options.resolveText ?? resolveSelectionText
-  const resolvedText =
-    capture.mode === 'content' ? resolveText(capture.rect) : ''
-  const result: ScreenAnnotationResult = { capture, resolvedText }
-
-  const buildMessage = options.buildMessage ?? defaultAnnotationMessage
-  const message = buildMessage(result)
-  const image = splitDataUrl(capture.imageDataUrl)
-  const title =
-    typeof options.title === 'function'
-      ? options.title(result)
-      : options.title ?? 'Screen annotation'
-
-  const scope = capture.mode === 'content' ? ('app' as const) : ('global' as const)
-
-  options.onStatus?.('working')
-  try {
-    const record = await createAgentSession({
-      scope,
-      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
-      title,
-    })
-    const session = {
-      scope,
-      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
-      sessionId: record.sessionId,
-    }
-
-    let snapshot = await startAgentRun({
-      session,
-      message,
-      ...(image ? { images: [image] } : {}),
-    })
-    const runId = snapshot.runId
-    const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS
-    let appliedTools = false
-
-    for (let step = 0; step < maxSteps; step += 1) {
-      if (snapshot.state === 'ready') {
-        snapshot = await callAgentRunModel({ runId, sessionId: record.sessionId })
-        continue
-      }
-      if (snapshot.state === 'pending_tools') {
-        const resolved = await resolveAgentRunToolCalls({ runId, session })
-        if (resolved.state === 'pending_tools') {
-          // A tool still needs explicit approval — hand off to the drawer.
-          options.onStatus?.('needs-approval')
-          return { sessionId: record.sessionId, status: 'needs-approval' }
-        }
-        appliedTools = true
-        snapshot = resolved
-        continue
-      }
-      break
-    }
-
-    if (snapshot.state === 'failed') {
-      const error = snapshot.error ?? 'The agent run failed.'
-      options.onStatus?.('error')
-      return { sessionId: record.sessionId, status: 'error', error }
-    }
-
-    const status = appliedTools ? 'applied' : 'no-changes'
-    options.onStatus?.(status)
-    return { sessionId: record.sessionId, status }
-  } catch (cause) {
-    const error = cause instanceof Error ? cause.message : String(cause)
-    options.onStatus?.('error')
-    return { sessionId: null, status: 'error', error }
-  }
-}
+import {
+  callAgentRunModel,
+  createAgentSession,
+  resolveAgentRunToolCalls,
+  startAgentRun,
+} from './agents.js'
+import { startScreenCapture } from './window.mjs'
+import type {
+  PlatformScreenCaptureOption,
+  PlatformScreenCaptureRect,
+  PlatformScreenCaptureResult,
+} from './window.mjs'
+
+/**
+ * Shared screen-annotation capture → agent flow. Any app can call
+ * {@link captureAndSendToAgent}: it opens the shell's chromeless capture
+ * overlay, resolves the text under the selection (content mode), and starts an
+ * agent run with the annotated image attached. App-specific wording and tools
+ * are supplied via `buildMessage`.
+ *
+ * Requires the `agents` permission in plugin.json.
+ */
+
+const DEFAULT_MAX_STEPS = 8
+
+export interface TextCandidate {
+  text: string
+  rect: PlatformScreenCaptureRect
+}
+
+function rectsIntersect(
+  a: PlatformScreenCaptureRect,
+  b: PlatformScreenCaptureRect,
+): boolean {
+  return (
+    a.x < b.x + b.width &&
+    a.x + a.width > b.x &&
+    a.y < b.y + b.height &&
+    a.y + a.height > b.y
+  )
+}
+
+/**
+ * Pure core: pick the text of candidates whose rect overlaps the selection, in
+ * order, deduped + whitespace-normalized. Separated from the DOM so it is
+ * unit-testable (jsdom has no layout).
+ */
+export function selectTextFromCandidates(
+  candidates: TextCandidate[],
+  selection: PlatformScreenCaptureRect,
+): string {
+  const seen = new Set<string>()
+  const parts: string[] = []
+  for (const candidate of candidates) {
+    if (!rectsIntersect(candidate.rect, selection)) continue
+    const text = candidate.text.replace(/\s+/g, ' ').trim()
+    if (!text || seen.has(text)) continue
+    seen.add(text)
+    parts.push(text)
+  }
+  return parts.join('\n\n')
+}
+
+const DEFAULT_BLOCK_SELECTOR =
+  'p, li, h1, h2, h3, h4, h5, h6, blockquote, figcaption, td, th, pre, dt, dd'
+
+export interface ResolveSelectionTextOptions {
+  /** Root to search within. Defaults to the active editor / contenteditable. */
+  root?: ParentNode | null
+  /** Block-level selector to collect. */
+  selector?: string
+}
+
+/**
+ * Resolve the text that intersects an app-local selection rect (CSS px relative
+ * to the app's own document viewport). Reads block elements inside the editor
+ * surface; returns '' when nothing meaningful overlaps.
+ */
+export function resolveSelectionText(
+  selection: PlatformScreenCaptureRect,
+  options: ResolveSelectionTextOptions = {},
+): string {
+  const root =
+    options.root ??
+    document.querySelector('.ProseMirror, [contenteditable="true"]')
+  if (!root) return ''
+  const selector = options.selector ?? DEFAULT_BLOCK_SELECTOR
+  const blocks = Array.from(root.querySelectorAll<HTMLElement>(selector))
+  const candidates: TextCandidate[] = blocks.map(element => {
+    const r = element.getBoundingClientRect()
+    return {
+      text: element.textContent ?? '',
+      rect: { x: r.left, y: r.top, width: r.width, height: r.height },
+    }
+  })
+  return selectTextFromCandidates(candidates, selection)
+}
+
+export interface ScreenAnnotationResult {
+  capture: PlatformScreenCaptureResult
+  /** Text resolved under the selection (content mode); '' in vibe mode. */
+  resolvedText: string
+}
+
+/** In-progress + terminal states a caller can reflect with quiet UI. */
+export type CaptureAgentStatus =
+  | 'working'
+  | 'applied'
+  | 'no-changes'
+  | 'needs-approval'
+  | 'error'
+
+export interface CaptureAgentOutcome {
+  sessionId: string | null
+  status: 'applied' | 'no-changes' | 'needs-approval' | 'error' | 'cancelled'
+  error?: string
+}
+
+export interface CaptureAndSendOptions {
+  appSlug: string
+  /** What this capture transforms, shown in the overlay (e.g. "book content", "page design"). */
+  label?: string
+  /** Contextual option controls rendered in the overlay (e.g. design scope). */
+  options?: PlatformScreenCaptureOption[]
+  /** Build the agent user message from the capture + resolved text. */
+  buildMessage?: (result: ScreenAnnotationResult) => string
+  /** Session title (string or builder). */
+  title?: string | ((result: ScreenAnnotationResult) => string)
+  /** Resolve text under the selection in content mode. Defaults to {@link resolveSelectionText}. */
+  resolveText?: (rect: PlatformScreenCaptureRect) => string
+  /** Total model + tool-resolution turns to drive. Default 8. */
+  maxSteps?: number
+  /** Progress callback for quiet UI: 'working' first, then a terminal status. */
+  onStatus?: (status: CaptureAgentStatus) => void
+}
+
+/**
+ * Guidance for interpreting pen marks. The marks (circles, underlines, arrows)
+ * indicate the focus; their scope is a judgment call the agent must make.
+ */
+export function penFocusGuidance(): string {
+  return [
+    'The user drew marks on the screenshot — focus on what they marked, not the whole captured region.',
+    'Marks may circle, underline, or point at content. Use judgment about scope: the user may mean exactly the marked words, the sentence that contains them, or the whole paragraph/section.',
+    'Infer the most sensible scope from the marks and the note together. If it is genuinely ambiguous, prefer the smallest coherent unit (usually the sentence) unless the note implies a larger change.',
+  ].join('\n')
+}
+
+/** Generic fallback message when an app doesn't supply `buildMessage`. */
+export function defaultAnnotationMessage({
+  capture,
+  resolvedText,
+}: ScreenAnnotationResult): string {
+  const note = capture.note.trim()
+  return [
+    capture.mode === 'content'
+      ? 'The user captured and annotated a region of this app and wants a change applied there.'
+      : 'The user captured and annotated a region of the app UI (vibe mode).',
+    '',
+    note ? `Note from the user:\n${note}` : 'No extra note.',
+    capture.hasMarks ? `\n${penFocusGuidance()}` : '',
+    resolvedText
+      ? `\nText in the captured region (for reference — the marks show the focus):\n"""\n${resolvedText}\n"""`
+      : '',
+    '',
+    'Use the attached annotated screenshot (with the user’s drawing) to make the change.',
+  ]
+    .filter(line => line !== '')
+    .join('\n')
+}
+
+interface DataUrlImage {
+  mediaType: string
+  dataBase64: string
+}
+
+function splitDataUrl(dataUrl: string): DataUrlImage | null {
+  const match = /^data:([^;]+);base64,(.+)$/s.exec(dataUrl)
+  if (!match) return null
+  return { mediaType: match[1], dataBase64: match[2] }
+}
+
+/**
+ * Open the capture overlay and, on send, run an agent end to end: call the
+ * model, then route + resolve its tool calls so the edit is actually applied
+ * (not left pending). Content mode targets the app's own agent (scope 'app');
+ * vibe mode targets a global agent. `onStatus` reports progress so the caller
+ * can show quiet feedback. Returns an outcome describing what happened.
+ */
+export async function captureAndSendToAgent(
+  options: CaptureAndSendOptions,
+): Promise<CaptureAgentOutcome> {
+  const capture = await startScreenCapture({
+    label: options.label,
+    options: options.options,
+  })
+  if (!capture) return { sessionId: null, status: 'cancelled' }
+
+  const resolveText = options.resolveText ?? resolveSelectionText
+  const resolvedText =
+    capture.mode === 'content' ? resolveText(capture.rect) : ''
+  const result: ScreenAnnotationResult = { capture, resolvedText }
+
+  const buildMessage = options.buildMessage ?? defaultAnnotationMessage
+  const message = buildMessage(result)
+  const image = splitDataUrl(capture.imageDataUrl)
+  const title =
+    typeof options.title === 'function'
+      ? options.title(result)
+      : options.title ?? 'Screen annotation'
+
+  const scope = capture.mode === 'content' ? ('app' as const) : ('global' as const)
+
+  options.onStatus?.('working')
+  try {
+    const record = await createAgentSession({
+      scope,
+      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
+      title,
+    })
+    const session = {
+      scope,
+      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
+      sessionId: record.sessionId,
+    }
+
+    let snapshot = await startAgentRun({
+      session,
+      message,
+      ...(image ? { images: [image] } : {}),
+    })
+    const runId = snapshot.runId
+    const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS
+    let appliedTools = false
+
+    for (let step = 0; step < maxSteps; step += 1) {
+      if (snapshot.state === 'ready') {
+        snapshot = await callAgentRunModel({ runId, sessionId: record.sessionId })
+        continue
+      }
+      if (snapshot.state === 'pending_tools') {
+        const resolved = await resolveAgentRunToolCalls({ runId, session })
+        if (resolved.state === 'pending_tools') {
+          // A tool still needs explicit approval — hand off to the drawer.
+          options.onStatus?.('needs-approval')
+          return { sessionId: record.sessionId, status: 'needs-approval' }
+        }
+        appliedTools = true
+        snapshot = resolved
+        continue
+      }
+      break
+    }
+
+    if (snapshot.state === 'failed') {
+      const error = snapshot.error ?? 'The agent run failed.'
+      options.onStatus?.('error')
+      return { sessionId: record.sessionId, status: 'error', error }
+    }
+
+    const status = appliedTools ? 'applied' : 'no-changes'
+    options.onStatus?.(status)
+    return { sessionId: record.sessionId, status }
+  } catch (cause) {
+    const error = cause instanceof Error ? cause.message : String(cause)
+    options.onStatus?.('error')
+    return { sessionId: null, status: 'error', error }
+  }
+}

package/src/bridge/theme.d.mts

@@ -1,3 +1,3 @@
-export function setSurfacePalette(
-  paletteId: string,
-): Promise<import('./types.ts').PlatformShellPreferences>
+export function setSurfacePalette(
+  paletteId: string,
+): Promise<import('./types.ts').PlatformShellPreferences>

package/src/bridge/theme.mjs

@@ -1,9 +1,9 @@
-import { bridge } from './client.mjs'
-import { PLATFORM_BRIDGE_METHODS } from './methods.mjs'
-
-/** Ask the shell to store a surface palette for the calling app (whole desktop theme). */
-export async function setSurfacePalette(paletteId) {
-  return bridge.call(PLATFORM_BRIDGE_METHODS.THEME_SET_SURFACE_PALETTE, [
-    { paletteId },
-  ])
-}
+import { bridge } from './client.mjs'
+import { PLATFORM_BRIDGE_METHODS } from './methods.mjs'
+
+/** Ask the shell to store a surface palette for the calling app (whole desktop theme). */
+export async function setSurfacePalette(paletteId) {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.THEME_SET_SURFACE_PALETTE, [
+    { paletteId },
+  ])
+}