@puredesktop/puredesktop-ui-bridge 1.0.0-beta.1

package/src/bridge/react/usePlatformViewportResource.tsx

@@ -0,0 +1,72 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { bridge } from '../client.mjs'
+import { PLATFORM_BRIDGE_EVENTS } from '../events.mjs'
+import { PLATFORM_BRIDGE_METHODS } from '../methods.mjs'
+import type { PlatformBridgeReadyMeta, ResourceOpenEvent } from '../types.js'
+
+function isResourceOpenEvent(value: unknown): value is ResourceOpenEvent {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    typeof (value as ResourceOpenEvent).path === 'string' &&
+    (value as ResourceOpenEvent).path.trim().length > 0
+  )
+}
+
+export interface UsePlatformViewportResourceResult {
+  /** Latest resource for this viewport (boot binding or warm `resource.open`). */
+  resource: ResourceOpenEvent | null
+  /** Clears `resource` after the app has applied it (avoids re-open loops). */
+  clearResource: () => void
+}
+
+/**
+ * Per-tab resource binding from the shell workspace.
+ * Prefer `meta.viewport.resource` on bridge ready; also listens for `resource.open`.
+ */
+export function usePlatformViewportResource(
+  ready: boolean,
+  meta: PlatformBridgeReadyMeta | null,
+): UsePlatformViewportResourceResult {
+  const [resource, setResource] = useState<ResourceOpenEvent | null>(null)
+  const bootTabIdRef = useRef<string | null>(null)
+
+  useEffect(() => {
+    return bridge.onEvent(PLATFORM_BRIDGE_EVENTS.RESOURCE_OPEN, payload => {
+      if (!isResourceOpenEvent(payload)) return
+      setResource(payload)
+    })
+  }, [])
+
+  useEffect(() => {
+    if (!ready || !meta?.viewport) return
+    if (bootTabIdRef.current === meta.viewport.tabId) return
+    bootTabIdRef.current = meta.viewport.tabId
+    if (meta.viewport.resource) {
+      setResource(meta.viewport.resource)
+      return
+    }
+
+    let cancelled = false
+    bridge
+      .call<{ resource?: ResourceOpenEvent | null }>(
+        PLATFORM_BRIDGE_METHODS.CATALOG_CONSUME_PENDING_OPEN,
+        [],
+      )
+      .then(result => {
+        if (cancelled || !isResourceOpenEvent(result?.resource)) return
+        setResource(result.resource)
+      })
+      .catch(() => undefined)
+
+    return () => {
+      cancelled = true
+    }
+  }, [meta, ready])
+
+  const clearResource = useCallback(() => {
+    setResource(null)
+  }, [])
+
+  return { resource, clearResource }
+}

package/src/bridge/react/useScheduledTask.tsx

@@ -0,0 +1,59 @@
+import { useEffect, useRef } from 'react'
+import { bridge } from '../client.mjs'
+import { PLATFORM_BRIDGE_EVENTS } from '../events.mjs'
+import { PLATFORM_BRIDGE_METHODS } from '../methods.mjs'
+
+/**
+ * Run `task` every `everyMinutes`, scheduled by the shell
+ * (`scheduler.register` + `schedule.fire` events) so all recurring work
+ * across the suite is declared in one place. Outside the shell
+ * (standalone dev) it falls back to a plain interval. `enabled: false`
+ * unregisters.
+ */
+export function useScheduledTask(
+  taskName: string,
+  everyMinutes: number,
+  task: () => void,
+  options?: { enabled?: boolean },
+): void {
+  const taskRef = useRef(task)
+  taskRef.current = task
+  const enabled = options?.enabled ?? true
+
+  useEffect(() => {
+    if (!enabled) return undefined
+
+    const standalone = typeof window !== 'undefined' && window.parent === window
+    if (standalone) {
+      const timer = setInterval(
+        () => taskRef.current(),
+        Math.max(1, everyMinutes) * 60_000,
+      )
+      return () => clearInterval(timer)
+    }
+
+    const unsubscribe = bridge.onEvent(
+      PLATFORM_BRIDGE_EVENTS.SCHEDULE_FIRE,
+      payload => {
+        if ((payload as { taskName?: string } | null)?.taskName === taskName) {
+          taskRef.current()
+        }
+      },
+    )
+    void bridge
+      .waitForReady()
+      .then(() =>
+        bridge.call(PLATFORM_BRIDGE_METHODS.SCHEDULER_REGISTER, [
+          { taskName, everyMinutes },
+        ]),
+      )
+      .catch(() => undefined)
+
+    return () => {
+      unsubscribe()
+      void bridge
+        .call(PLATFORM_BRIDGE_METHODS.SCHEDULER_UNREGISTER, [{ taskName }])
+        .catch(() => undefined)
+    }
+  }, [enabled, everyMinutes, taskName])
+}

package/src/bridge/render.d.mts

@@ -0,0 +1,31 @@
+import type {
+  PrintPageSize,
+  RenderPagedPreviewRequest,
+  RenderPagedPreviewResult,
+  RenderPagedSnapshotRequest,
+  RenderPagedSnapshotResult,
+  RenderPrintHtmlRequest,
+} from './types.js'
+
+export declare function renderPrintHtml(
+  request: RenderPrintHtmlRequest,
+): Promise<string>
+
+export declare function preparePagedPreview(
+  request: RenderPagedPreviewRequest,
+): Promise<RenderPagedPreviewResult>
+
+export declare function capturePagedSnapshot(
+  request: RenderPagedSnapshotRequest,
+): Promise<RenderPagedSnapshotResult>
+
+export declare function getDefaultPrintStylesheet(
+  pageSize?: PrintPageSize,
+): Promise<string>
+
+/** @returns unsubscribe */
+export declare function onRenderProgress(
+  listener: (event: import('./types.js').RenderProgress) => void,
+): () => void
+
+export declare function renderCancel(): Promise<void>

package/src/bridge/render.mjs

@@ -0,0 +1,62 @@
+import { bridge } from './client.mjs'
+import { PLATFORM_BRIDGE_EVENTS } from './events.mjs'
+import { PLATFORM_BRIDGE_METHODS } from './methods.mjs'
+
+/**
+ * Print an app-prepared HTML file to PDF (shell: Paged.js + Electron printToPDF).
+ * Requires `render` permission in plugin.json.
+ *
+ * @param {import('./types.js').RenderPrintHtmlRequest} request
+ * @returns {Promise<string>} Absolute path to the output PDF
+ */
+export async function renderPrintHtml(request) {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.RENDER_PRINT_HTML, [request])
+}
+
+/**
+ * Prepare an app print HTML file for preview (shell: Paged.js pagination).
+ * Requires `render` permission in plugin.json.
+ *
+ * @param {import('./types.js').RenderPagedPreviewRequest} request
+ * @returns {Promise<import('./types.js').RenderPagedPreviewResult>}
+ */
+export async function preparePagedPreview(request) {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.RENDER_PREPARE_PAGED_PREVIEW, [
+    request,
+  ])
+}
+
+/**
+ * Capture a paginated page image from an app print HTML file.
+ * Requires `render` permission.
+ *
+ * @param {import('./types.js').RenderPagedSnapshotRequest} request
+ * @returns {Promise<import('./types.js').RenderPagedSnapshotResult>}
+ */
+export async function capturePagedSnapshot(request) {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.RENDER_CAPTURE_PAGED_SNAPSHOT, [
+    request,
+  ])
+}
+
+/**
+ * Default Paged.js print CSS for a built-in page size (shell-owned).
+ * Requires `render` permission.
+ *
+ * @param {import('./types.js').PrintPageSize} [pageSize='A5']
+ * @returns {Promise<string>}
+ */
+export async function getDefaultPrintStylesheet(pageSize = 'A5') {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.RENDER_GET_PRINT_STYLESHEET, [
+    pageSize,
+  ])
+}
+
+/** @returns {() => void} unsubscribe */
+export function onRenderProgress(listener) {
+  return bridge.onEvent(PLATFORM_BRIDGE_EVENTS.RENDER_PROGRESS, listener)
+}
+
+export async function renderCancel() {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.RENDER_CANCEL, [])
+}

package/src/bridge/render.ts

@@ -0,0 +1,8 @@
+export {
+  getDefaultPrintStylesheet,
+  onRenderProgress,
+  capturePagedSnapshot,
+  preparePagedPreview,
+  renderCancel,
+  renderPrintHtml,
+} from './render.mjs'

package/src/bridge/screenAnnotation.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from 'vitest'
+import {
+  defaultAnnotationMessage,
+  selectTextFromCandidates,
+  type TextCandidate,
+} from './screenAnnotation.js'
+
+const candidates: TextCandidate[] = [
+  { text: 'First paragraph', rect: { x: 0, y: 0, width: 100, height: 20 } },
+  { text: 'Second paragraph', rect: { x: 0, y: 30, width: 100, height: 20 } },
+  { text: 'Far away', rect: { x: 0, y: 500, width: 100, height: 20 } },
+]
+
+describe('selectTextFromCandidates', () => {
+  it('returns text of blocks overlapping the selection, in order', () => {
+    expect(
+      selectTextFromCandidates(candidates, {
+        x: 10,
+        y: 10,
+        width: 50,
+        height: 30,
+      }),
+    ).toBe('First paragraph\n\nSecond paragraph')
+  })
+
+  it('excludes blocks outside the selection', () => {
+    expect(
+      selectTextFromCandidates(candidates, {
+        x: 10,
+        y: 0,
+        width: 50,
+        height: 15,
+      }),
+    ).toBe('First paragraph')
+  })
+
+  it('dedupes and normalizes whitespace', () => {
+    expect(
+      selectTextFromCandidates(
+        [
+          { text: '  Hello   world  ', rect: { x: 0, y: 0, width: 10, height: 10 } },
+          { text: 'Hello world', rect: { x: 0, y: 0, width: 10, height: 10 } },
+        ],
+        { x: 0, y: 0, width: 10, height: 10 },
+      ),
+    ).toBe('Hello world')
+  })
+
+  it('returns empty string when nothing overlaps', () => {
+    expect(
+      selectTextFromCandidates(candidates, {
+        x: 1000,
+        y: 1000,
+        width: 10,
+        height: 10,
+      }),
+    ).toBe('')
+  })
+})
+
+describe('defaultAnnotationMessage', () => {
+  const baseCapture = {
+    imageDataUrl: 'data:image/png;base64,AAAA',
+    rect: { x: 0, y: 0, width: 10, height: 10 },
+    mode: 'content' as const,
+    note: 'tighten this',
+    hasMarks: false,
+    optionValues: {},
+  }
+
+  it('includes the note and resolved text in content mode', () => {
+    const message = defaultAnnotationMessage({
+      capture: baseCapture,
+      resolvedText: 'The paragraph text',
+    })
+    expect(message).toContain('tighten this')
+    expect(message).toContain('The paragraph text')
+    expect(message).toContain('annotated screenshot')
+  })
+
+  it('omits the resolved-text block in vibe mode', () => {
+    const message = defaultAnnotationMessage({
+      capture: { ...baseCapture, mode: 'ui', note: '' },
+      resolvedText: '',
+    })
+    expect(message).toContain('vibe mode')
+    expect(message).not.toContain('captured region')
+  })
+
+  it('adds pen-focus guidance only when marks were drawn', () => {
+    const withoutMarks = defaultAnnotationMessage({
+      capture: baseCapture,
+      resolvedText: 'text',
+    })
+    expect(withoutMarks).not.toContain('focus on what they marked')
+
+    const withMarks = defaultAnnotationMessage({
+      capture: { ...baseCapture, hasMarks: true },
+      resolvedText: 'text',
+    })
+    expect(withMarks).toContain('focus on what they marked')
+  })
+})

package/src/bridge/screenAnnotation.ts

@@ -0,0 +1,269 @@
+import {
+  callAgentRunModel,
+  createAgentSession,
+  resolveAgentRunToolCalls,
+  startAgentRun,
+} from './agents.js'
+import { startScreenCapture } from './window.mjs'
+import type {
+  PlatformScreenCaptureOption,
+  PlatformScreenCaptureRect,
+  PlatformScreenCaptureResult,
+} from './window.mjs'
+
+/**
+ * Shared screen-annotation capture → agent flow. Any app can call
+ * {@link captureAndSendToAgent}: it opens the shell's chromeless capture
+ * overlay, resolves the text under the selection (content mode), and starts an
+ * agent run with the annotated image attached. App-specific wording and tools
+ * are supplied via `buildMessage`.
+ *
+ * Requires the `agents` permission in plugin.json.
+ */
+
+const DEFAULT_MAX_STEPS = 8
+
+export interface TextCandidate {
+  text: string
+  rect: PlatformScreenCaptureRect
+}
+
+function rectsIntersect(
+  a: PlatformScreenCaptureRect,
+  b: PlatformScreenCaptureRect,
+): boolean {
+  return (
+    a.x < b.x + b.width &&
+    a.x + a.width > b.x &&
+    a.y < b.y + b.height &&
+    a.y + a.height > b.y
+  )
+}
+
+/**
+ * Pure core: pick the text of candidates whose rect overlaps the selection, in
+ * order, deduped + whitespace-normalized. Separated from the DOM so it is
+ * unit-testable (jsdom has no layout).
+ */
+export function selectTextFromCandidates(
+  candidates: TextCandidate[],
+  selection: PlatformScreenCaptureRect,
+): string {
+  const seen = new Set<string>()
+  const parts: string[] = []
+  for (const candidate of candidates) {
+    if (!rectsIntersect(candidate.rect, selection)) continue
+    const text = candidate.text.replace(/\s+/g, ' ').trim()
+    if (!text || seen.has(text)) continue
+    seen.add(text)
+    parts.push(text)
+  }
+  return parts.join('\n\n')
+}
+
+const DEFAULT_BLOCK_SELECTOR =
+  'p, li, h1, h2, h3, h4, h5, h6, blockquote, figcaption, td, th, pre, dt, dd'
+
+export interface ResolveSelectionTextOptions {
+  /** Root to search within. Defaults to the active editor / contenteditable. */
+  root?: ParentNode | null
+  /** Block-level selector to collect. */
+  selector?: string
+}
+
+/**
+ * Resolve the text that intersects an app-local selection rect (CSS px relative
+ * to the app's own document viewport). Reads block elements inside the editor
+ * surface; returns '' when nothing meaningful overlaps.
+ */
+export function resolveSelectionText(
+  selection: PlatformScreenCaptureRect,
+  options: ResolveSelectionTextOptions = {},
+): string {
+  const root =
+    options.root ??
+    document.querySelector('.ProseMirror, [contenteditable="true"]')
+  if (!root) return ''
+  const selector = options.selector ?? DEFAULT_BLOCK_SELECTOR
+  const blocks = Array.from(root.querySelectorAll<HTMLElement>(selector))
+  const candidates: TextCandidate[] = blocks.map(element => {
+    const r = element.getBoundingClientRect()
+    return {
+      text: element.textContent ?? '',
+      rect: { x: r.left, y: r.top, width: r.width, height: r.height },
+    }
+  })
+  return selectTextFromCandidates(candidates, selection)
+}
+
+export interface ScreenAnnotationResult {
+  capture: PlatformScreenCaptureResult
+  /** Text resolved under the selection (content mode); '' in vibe mode. */
+  resolvedText: string
+}
+
+/** In-progress + terminal states a caller can reflect with quiet UI. */
+export type CaptureAgentStatus =
+  | 'working'
+  | 'applied'
+  | 'no-changes'
+  | 'needs-approval'
+  | 'error'
+
+export interface CaptureAgentOutcome {
+  sessionId: string | null
+  status: 'applied' | 'no-changes' | 'needs-approval' | 'error' | 'cancelled'
+  error?: string
+}
+
+export interface CaptureAndSendOptions {
+  appSlug: string
+  /** What this capture transforms, shown in the overlay (e.g. "book content", "page design"). */
+  label?: string
+  /** Contextual option controls rendered in the overlay (e.g. design scope). */
+  options?: PlatformScreenCaptureOption[]
+  /** Build the agent user message from the capture + resolved text. */
+  buildMessage?: (result: ScreenAnnotationResult) => string
+  /** Session title (string or builder). */
+  title?: string | ((result: ScreenAnnotationResult) => string)
+  /** Resolve text under the selection in content mode. Defaults to {@link resolveSelectionText}. */
+  resolveText?: (rect: PlatformScreenCaptureRect) => string
+  /** Total model + tool-resolution turns to drive. Default 8. */
+  maxSteps?: number
+  /** Progress callback for quiet UI: 'working' first, then a terminal status. */
+  onStatus?: (status: CaptureAgentStatus) => void
+}
+
+/**
+ * Guidance for interpreting pen marks. The marks (circles, underlines, arrows)
+ * indicate the focus; their scope is a judgment call the agent must make.
+ */
+export function penFocusGuidance(): string {
+  return [
+    'The user drew marks on the screenshot — focus on what they marked, not the whole captured region.',
+    'Marks may circle, underline, or point at content. Use judgment about scope: the user may mean exactly the marked words, the sentence that contains them, or the whole paragraph/section.',
+    'Infer the most sensible scope from the marks and the note together. If it is genuinely ambiguous, prefer the smallest coherent unit (usually the sentence) unless the note implies a larger change.',
+  ].join('\n')
+}
+
+/** Generic fallback message when an app doesn't supply `buildMessage`. */
+export function defaultAnnotationMessage({
+  capture,
+  resolvedText,
+}: ScreenAnnotationResult): string {
+  const note = capture.note.trim()
+  return [
+    capture.mode === 'content'
+      ? 'The user captured and annotated a region of this app and wants a change applied there.'
+      : 'The user captured and annotated a region of the app UI (vibe mode).',
+    '',
+    note ? `Note from the user:\n${note}` : 'No extra note.',
+    capture.hasMarks ? `\n${penFocusGuidance()}` : '',
+    resolvedText
+      ? `\nText in the captured region (for reference — the marks show the focus):\n"""\n${resolvedText}\n"""`
+      : '',
+    '',
+    'Use the attached annotated screenshot (with the user’s drawing) to make the change.',
+  ]
+    .filter(line => line !== '')
+    .join('\n')
+}
+
+interface DataUrlImage {
+  mediaType: string
+  dataBase64: string
+}
+
+function splitDataUrl(dataUrl: string): DataUrlImage | null {
+  const match = /^data:([^;]+);base64,(.+)$/s.exec(dataUrl)
+  if (!match) return null
+  return { mediaType: match[1], dataBase64: match[2] }
+}
+
+/**
+ * Open the capture overlay and, on send, run an agent end to end: call the
+ * model, then route + resolve its tool calls so the edit is actually applied
+ * (not left pending). Content mode targets the app's own agent (scope 'app');
+ * vibe mode targets a global agent. `onStatus` reports progress so the caller
+ * can show quiet feedback. Returns an outcome describing what happened.
+ */
+export async function captureAndSendToAgent(
+  options: CaptureAndSendOptions,
+): Promise<CaptureAgentOutcome> {
+  const capture = await startScreenCapture({
+    label: options.label,
+    options: options.options,
+  })
+  if (!capture) return { sessionId: null, status: 'cancelled' }
+
+  const resolveText = options.resolveText ?? resolveSelectionText
+  const resolvedText =
+    capture.mode === 'content' ? resolveText(capture.rect) : ''
+  const result: ScreenAnnotationResult = { capture, resolvedText }
+
+  const buildMessage = options.buildMessage ?? defaultAnnotationMessage
+  const message = buildMessage(result)
+  const image = splitDataUrl(capture.imageDataUrl)
+  const title =
+    typeof options.title === 'function'
+      ? options.title(result)
+      : options.title ?? 'Screen annotation'
+
+  const scope = capture.mode === 'content' ? ('app' as const) : ('global' as const)
+
+  options.onStatus?.('working')
+  try {
+    const record = await createAgentSession({
+      scope,
+      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
+      title,
+    })
+    const session = {
+      scope,
+      ...(scope === 'app' ? { appSlug: options.appSlug } : {}),
+      sessionId: record.sessionId,
+    }
+
+    let snapshot = await startAgentRun({
+      session,
+      message,
+      ...(image ? { images: [image] } : {}),
+    })
+    const runId = snapshot.runId
+    const maxSteps = options.maxSteps ?? DEFAULT_MAX_STEPS
+    let appliedTools = false
+
+    for (let step = 0; step < maxSteps; step += 1) {
+      if (snapshot.state === 'ready') {
+        snapshot = await callAgentRunModel({ runId, sessionId: record.sessionId })
+        continue
+      }
+      if (snapshot.state === 'pending_tools') {
+        const resolved = await resolveAgentRunToolCalls({ runId, session })
+        if (resolved.state === 'pending_tools') {
+          // A tool still needs explicit approval — hand off to the drawer.
+          options.onStatus?.('needs-approval')
+          return { sessionId: record.sessionId, status: 'needs-approval' }
+        }
+        appliedTools = true
+        snapshot = resolved
+        continue
+      }
+      break
+    }
+
+    if (snapshot.state === 'failed') {
+      const error = snapshot.error ?? 'The agent run failed.'
+      options.onStatus?.('error')
+      return { sessionId: record.sessionId, status: 'error', error }
+    }
+
+    const status = appliedTools ? 'applied' : 'no-changes'
+    options.onStatus?.(status)
+    return { sessionId: record.sessionId, status }
+  } catch (cause) {
+    const error = cause instanceof Error ? cause.message : String(cause)
+    options.onStatus?.('error')
+    return { sessionId: null, status: 'error', error }
+  }
+}

package/src/bridge/theme.d.mts

@@ -0,0 +1,3 @@
+export function setSurfacePalette(
+  paletteId: string,
+): Promise<import('./types.ts').PlatformShellPreferences>

package/src/bridge/theme.mjs

@@ -0,0 +1,9 @@
+import { bridge } from './client.mjs'
+import { PLATFORM_BRIDGE_METHODS } from './methods.mjs'
+
+/** Ask the shell to store a surface palette for the calling app (whole desktop theme). */
+export async function setSurfacePalette(paletteId) {
+  return bridge.call(PLATFORM_BRIDGE_METHODS.THEME_SET_SURFACE_PALETTE, [
+    { paletteId },
+  ])
+}