@yieldfabric/terminal 1.3.1 → 1.4.0

package/dist/index.cjs

@@ -8068,6 +8068,10 @@ var VoiceSession = class {
     this.finishTimer = null;
     this.muted = false;
     this.blobUrl = null;
+    /** What the person's page shows, as the host describes it, and what the
+     *  relay was last told (it keeps one note of it, replaced on change). */
+    this.screen = "";
+    this.screenSent = null;
   }
   /** Call from the tap itself: iOS only lets audio start inside a gesture,
    *  so the AudioContext is created before the first `await`. */
@@ -8168,6 +8172,23 @@ var VoiceSession = class {
   stop() {
     this.end("stopped");
   }
+  /**
+   * What the person's page shows, in a few plain lines: the voice keeps it
+   * as one note, replaced as it changes, so "this photo", "this document"
+   * or "this claim" means what they are looking at and it never asks which.
+   * Call it whenever the page changes; it is sent once the session is
+   * ready, and only when it differs. Empty clears it. An agents deployment
+   * older than the `screen` message answers with a warning and nothing else.
+   */
+  setScreen(text) {
+    this.screen = text.trim();
+    this.sendScreen();
+  }
+  sendScreen() {
+    if (!this.ready || this.ended || this.screen === (this.screenSent ?? "")) return;
+    this.send({ type: "screen", text: this.screen });
+    this.screenSent = this.screen;
+  }
   /**
    * Mute the microphone, or not. The track goes silent rather than the
    * stream stopping: the service keeps hearing (silence), so whatever the
@@ -8224,6 +8245,7 @@ var VoiceSession = class {
       case "session.ready":
         this.ready = true;
         this.setPhase("listening");
+        this.sendScreen();
         break;
       case "input_audio_buffer.speech_started":
         this.cancelFinish();
@@ -8410,6 +8432,7 @@ function useVoiceSession(host, connection) {
   hostRef.current = host;
   const connectionRef = React15.useRef(connection);
   connectionRef.current = connection;
+  const screen = React15.useRef("");
   const { baseUrl } = connection;
   React15.useEffect(() => {
     const controller = new AbortController();
@@ -8450,9 +8473,14 @@ function useVoiceSession(host, connection) {
       { ...connectionRef.current }
     );
     session.current = s;
+    s.setScreen(screen.current);
     void s.start();
   }, []);
   const stop = React15.useCallback(() => session.current?.stop(), []);
+  const setScreen = React15.useCallback((text) => {
+    screen.current = text;
+    session.current?.setScreen(text);
+  }, []);
   const toggle = React15.useCallback(() => session.current ? stop() : start(), [start, stop]);
   const dismiss = React15.useCallback(() => setEnded(null), []);
   const toggleMute = React15.useCallback(() => {
@@ -8484,7 +8512,8 @@ function useVoiceSession(host, connection) {
     start,
     stop,
     toggle,
-    dismiss
+    dismiss,
+    setScreen
   };
 }
 function voiceEndedMessage(ended, name = "the assistant") {

package/dist/index.d.cts

@@ -1365,7 +1365,9 @@ declare function encodePersona(persona: VoicePersona | undefined): string | null
  * Wire (agents `src/voice/protocol.rs`): binary frames are PCM16 mono
  * 24 kHz both ways; text frames are JSON events. The browser may send only
  * `tool_result`, `progress` (what the host has done and found while it
- * works, told as paced updates), `cancel_response` and `clear_input`.
+ * works, told as paced updates), `screen` (what the person's page shows,
+ * so "this photo" or "this document" means what they are looking at),
+ * `cancel_response` and `clear_input`.
  */
 
 /** How to reach agents, as whom, and who is talking. */
@@ -1488,11 +1490,25 @@ declare class VoiceSession {
     private finishTimer;
     private muted;
     private blobUrl;
+    /** What the person's page shows, as the host describes it, and what the
+     *  relay was last told (it keeps one note of it, replaced on change). */
+    private screen;
+    private screenSent;
     constructor(h: VoiceHandlers, connection: VoiceConnection);
     /** Call from the tap itself: iOS only lets audio start inside a gesture,
      *  so the AudioContext is created before the first `await`. */
     start(): Promise<void>;
     stop(): void;
+    /**
+     * What the person's page shows, in a few plain lines: the voice keeps it
+     * as one note, replaced as it changes, so "this photo", "this document"
+     * or "this claim" means what they are looking at and it never asks which.
+     * Call it whenever the page changes; it is sent once the session is
+     * ready, and only when it differs. Empty clears it. An agents deployment
+     * older than the `screen` message answers with a warning and nothing else.
+     */
+    setScreen(text: string): void;
+    private sendScreen;
     /**
      * Mute the microphone, or not. The track goes silent rather than the
      * stream stopping: the service keeps hearing (silence), so whatever the
@@ -1552,6 +1568,10 @@ interface VoiceSessionState {
     toggle: () => void;
     /** Clear `ended`. */
     dismiss: () => void;
+    /** What the person's page shows, in a few plain lines, so "this photo"
+     *  or "this document" means what they are looking at. Kept between
+     *  sessions: a session that starts later is told it once it is ready. */
+    setScreen: (text: string) => void;
 }
 /**
  * One live voice session for a conversation surface.

package/dist/index.d.ts

@@ -1365,7 +1365,9 @@ declare function encodePersona(persona: VoicePersona | undefined): string | null
  * Wire (agents `src/voice/protocol.rs`): binary frames are PCM16 mono
  * 24 kHz both ways; text frames are JSON events. The browser may send only
  * `tool_result`, `progress` (what the host has done and found while it
- * works, told as paced updates), `cancel_response` and `clear_input`.
+ * works, told as paced updates), `screen` (what the person's page shows,
+ * so "this photo" or "this document" means what they are looking at),
+ * `cancel_response` and `clear_input`.
  */
 
 /** How to reach agents, as whom, and who is talking. */
@@ -1488,11 +1490,25 @@ declare class VoiceSession {
     private finishTimer;
     private muted;
     private blobUrl;
+    /** What the person's page shows, as the host describes it, and what the
+     *  relay was last told (it keeps one note of it, replaced on change). */
+    private screen;
+    private screenSent;
     constructor(h: VoiceHandlers, connection: VoiceConnection);
     /** Call from the tap itself: iOS only lets audio start inside a gesture,
      *  so the AudioContext is created before the first `await`. */
     start(): Promise<void>;
     stop(): void;
+    /**
+     * What the person's page shows, in a few plain lines: the voice keeps it
+     * as one note, replaced as it changes, so "this photo", "this document"
+     * or "this claim" means what they are looking at and it never asks which.
+     * Call it whenever the page changes; it is sent once the session is
+     * ready, and only when it differs. Empty clears it. An agents deployment
+     * older than the `screen` message answers with a warning and nothing else.
+     */
+    setScreen(text: string): void;
+    private sendScreen;
     /**
      * Mute the microphone, or not. The track goes silent rather than the
      * stream stopping: the service keeps hearing (silence), so whatever the
@@ -1552,6 +1568,10 @@ interface VoiceSessionState {
     toggle: () => void;
     /** Clear `ended`. */
     dismiss: () => void;
+    /** What the person's page shows, in a few plain lines, so "this photo"
+     *  or "this document" means what they are looking at. Kept between
+     *  sessions: a session that starts later is told it once it is ready. */
+    setScreen: (text: string) => void;
 }
 /**
  * One live voice session for a conversation surface.

package/dist/index.js

@@ -8059,6 +8059,10 @@ var VoiceSession = class {
     this.finishTimer = null;
     this.muted = false;
     this.blobUrl = null;
+    /** What the person's page shows, as the host describes it, and what the
+     *  relay was last told (it keeps one note of it, replaced on change). */
+    this.screen = "";
+    this.screenSent = null;
   }
   /** Call from the tap itself: iOS only lets audio start inside a gesture,
    *  so the AudioContext is created before the first `await`. */
@@ -8159,6 +8163,23 @@ var VoiceSession = class {
   stop() {
     this.end("stopped");
   }
+  /**
+   * What the person's page shows, in a few plain lines: the voice keeps it
+   * as one note, replaced as it changes, so "this photo", "this document"
+   * or "this claim" means what they are looking at and it never asks which.
+   * Call it whenever the page changes; it is sent once the session is
+   * ready, and only when it differs. Empty clears it. An agents deployment
+   * older than the `screen` message answers with a warning and nothing else.
+   */
+  setScreen(text) {
+    this.screen = text.trim();
+    this.sendScreen();
+  }
+  sendScreen() {
+    if (!this.ready || this.ended || this.screen === (this.screenSent ?? "")) return;
+    this.send({ type: "screen", text: this.screen });
+    this.screenSent = this.screen;
+  }
   /**
    * Mute the microphone, or not. The track goes silent rather than the
    * stream stopping: the service keeps hearing (silence), so whatever the
@@ -8215,6 +8236,7 @@ var VoiceSession = class {
       case "session.ready":
         this.ready = true;
         this.setPhase("listening");
+        this.sendScreen();
         break;
       case "input_audio_buffer.speech_started":
         this.cancelFinish();
@@ -8401,6 +8423,7 @@ function useVoiceSession(host, connection) {
   hostRef.current = host;
   const connectionRef = useRef(connection);
   connectionRef.current = connection;
+  const screen = useRef("");
   const { baseUrl } = connection;
   useEffect(() => {
     const controller = new AbortController();
@@ -8441,9 +8464,14 @@ function useVoiceSession(host, connection) {
       { ...connectionRef.current }
     );
     session.current = s;
+    s.setScreen(screen.current);
     void s.start();
   }, []);
   const stop = useCallback(() => session.current?.stop(), []);
+  const setScreen = useCallback((text) => {
+    screen.current = text;
+    session.current?.setScreen(text);
+  }, []);
   const toggle = useCallback(() => session.current ? stop() : start(), [start, stop]);
   const dismiss = useCallback(() => setEnded(null), []);
   const toggleMute = useCallback(() => {
@@ -8475,7 +8503,8 @@ function useVoiceSession(host, connection) {
     start,
     stop,
     toggle,
-    dismiss
+    dismiss,
+    setScreen
   };
 }
 function voiceEndedMessage(ended, name = "the assistant") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yieldfabric/terminal",
-  "version": "1.3.1",
+  "version": "1.4.0",
   "description": "Reusable AI-powered terminal + knowledge-graph component for YieldFabric",
   "license": "SEE LICENSE IN LICENSE.md",
   "type": "module",