@clowder-ai/companion 0.1.0-alpha.21 → 0.1.0-alpha.25

package/renderer/surface.mjs

@@ -18,12 +18,14 @@ import { currentCompanionIdentity } from './companion-identity.mjs';
 import { FoldedChatUnread } from './chat-unread.mjs';
 import { COMPACT_BALL_SIZE, compactSizeAction } from './compact-size.mjs';
 import { WorkOverviewView } from './work-overview.mjs';
+import { MeetingStatus, renderMeetingStatus } from './meeting-state.mjs';
+import { VisibleTextPresentation } from './visible-text-receipt.mjs';
+import { ConversationSurface } from './conversation-surface.mjs';
 const $ = id => document.getElementById(id);
 const label = (id, text) => { $(id).querySelector('.label').textContent = text; };
 const status = text => {
   $('status').textContent = text;
   $('menu-status').textContent = text;
-  $('chat-status').textContent = text;
   $('announcement').textContent = text;
 };
 const transcript = new TranscriptView($('chat-transcript'));
@@ -35,6 +37,7 @@ const callTranscript = new CallTranscriptView($('call-transcript-log'), {
 });
 const bubble = new RecentBubble([$('bubble-first'), $('bubble-second')]);
 const callBubble = new RecentBubble([$('call-bubble-first'), $('call-bubble-second')]);
+const meetingStatus = new MeetingStatus();
 const foldedChatUnread = new FoldedChatUnread(count => {
   $('call-chat-unread').hidden = count === 0;
   $('call-chat-unread').textContent = count ? String(count) : '';
@@ -46,8 +49,8 @@ const nativeTravel = new NativeWindowTravel({
   readPosition: () => ({ x: window.screenX, y: window.screenY }),
   changed: value => motion.syncTravel(value),
 });
-let sharing = false, pendingScreen = false, loading = false, transcriptLoading = false, latestHistory, previousPhase = 'idle';
-let didMove = false, threadTitle, settingsView, settingsValues, decisionPanel, workView;
+let sharing = false, pendingScreen = false, transcriptLoading = false, previousPhase = 'idle';
+let didMove = false, settingsView, settingsValues, decisionPanel, workView;
 let previousControlPanel = 'none';
 // F229 stays quiet: a history ID change has no provenance for proactive text.
 const ambientPanel = active => active ? 'actions' : 'none';
@@ -60,6 +63,12 @@ if (!window.clowderCompanion) {
 } else {
   nativeTravel.start();
   const client = createCompanionClient(window.clowderCompanion);
+  let conversationSurface;
+  const visibleText = new VisibleTextPresentation({
+    document,
+    report: observation => client.observePresentation(observation),
+    transcript, callTranscript, bubble, callBubble,
+  });
   const controls = bindPetControls(client, {
     error: (error, context) => status(context?.panel === 'settings' && error?.code === 'invalid_request' ? '这个 Host 版本还不支持猫旁设置。原来的设置入口会继续保留。' : explainError(error)),
     moved: (dx, dy) => {
@@ -72,16 +81,23 @@ if (!window.clowderCompanion) {
       setTimeout(() => motion.setDockedEdge(detectDockedEdge(window, $('pet').getBoundingClientRect())), 100);
     },
     changed: panel => {
+      visibleText.retry();
+      if (previousControlPanel === 'chat' && panel !== 'chat') conversationSurface.closeHistory();
+      if (panel === 'chat' && conversationSurface.controller.mode === 'idle') conversationSurface.openHistory();
       if (panel === 'chat') foldedChatUnread.open();
       else if (previousControlPanel === 'chat' && conversation?.active) foldedChatUnread.fold();
       previousControlPanel = panel;
-      if (panel === 'chat') void readHistory();
+      if (panel === 'chat') void conversationSurface.readHistory();
       if (panel === 'transcript') void readCallTranscript();
       if (panel === 'decisions') void decisionPanel?.read();
       if (panel === 'settings') void settingsView?.open();
       if (panel === 'menu') void workView?.load();
     },
     action(kind) {
+      if (kind === 'write') {
+        conversationSurface.openWrite(conversation.active);
+      }
+      if (kind === 'history') conversationSurface.openHistory();
       if (kind === 'begin') { transcript.reset(); callTranscript.reset(); bubble.reset(); callBubble.clear(); void conversation.begin(); void controls.show('none'); }
       if (kind === 'listen') { transcript.reset(); callTranscript.reset(); bubble.reset(); callBubble.clear(); void conversation.begin('receive_only'); void controls.show('none'); }
       if (kind === 'stop') void conversation.end();
@@ -137,18 +153,17 @@ if (!window.clowderCompanion) {
   }
   const conversation = new CompanionConversation({
     client, createPeer: callback => new VoicePeer(callback, client), stopScreen: () => screen.stop(),
+    delivery: ({ text, receipt }) => conversationSurface.controller.recordDelivery(receipt, text),
     transcript: event => {
       if (event.type === 'turn-done') {
         transcript.finish(event); callTranscript.finish(event); bubble.finish(event.role); callBubble.finish(event.role);
-        void readHistory(); void readCallTranscript();
+        void conversationSurface.readHistory(); void readCallTranscript();
       } else if (event.type === 'typed') {
         callTranscript.appendTyped(event);
-        void readHistory(); void readCallTranscript();
+        void conversationSurface.readHistory(); void readCallTranscript();
       } else {
-        transcript.append(event.role, event.text, true);
-        callTranscript.appendLive(event);
         const speaker = event.role === 'assistant' ? conversation.identity?.displayName ?? '猫猫' : '语音';
-        bubble.append(event.role, event.text, speaker); callBubble.append(event.role, event.text, speaker);
+        visibleText.appendLive(event, speaker);
         void controls.setAmbient('actions');
       }
     },
@@ -156,7 +171,7 @@ if (!window.clowderCompanion) {
       const showFailure = value.failed && previousPhase !== 'idle' && value.phase === 'idle';
       const justConnected = previousPhase !== 'talking' && value.phase === 'talking';
       if (previousPhase !== 'idle' && value.phase === 'idle') {
-        transcript.reset(); callTranscript.reset(); bubble.reset(); callBubble.clear(); void readHistory();
+        transcript.reset(); callTranscript.reset(); bubble.reset(); callBubble.clear(); void conversationSurface.readHistory();
         foldedChatUnread.reset();
         if (controls.panel === 'transcript') void controls.show('menu');
       }
@@ -165,7 +180,7 @@ if (!window.clowderCompanion) {
       const receiveOnly = value.audioMode === 'receive_only';
       const canReceiveOnly = value.identity?.audio?.supportedModes?.includes('receive_only') === true;
       const retryReceiveOnly = value.failed && value.retryMode === 'receive_only' && canReceiveOnly;
-      callTranscript.setCall(value.callId);
+      visibleText.setCall(value.callId);
       void controls.setAmbient(ambientPanel(active));
       $('begin').hidden = active; $('begin').disabled = !value.identity;
       $('listen').hidden = active || !canReceiveOnly; $('listen').disabled = !value.identity;
@@ -191,7 +206,8 @@ if (!window.clowderCompanion) {
         : value.phase === 'connecting' ? '正在请求麦克风'
           : value.recovering ? `正在等待原连接恢复 · ${value.muted ? '麦克风已静音' : '麦克风已开启'}`
             : value.muted ? '麦克风已静音' : active ? '麦克风已开启' : '';
-      for (const id of ['actions', 'chat-call-context', 'transcript-call-context', 'menu-call-context', 'settings-call-context']) $(id).dataset.connectionState = value.recovering ? 'recovering' : active ? 'connected' : 'idle';
+      const connectionState = value.recovering ? 'recovering' : value.phase === 'talking' ? 'connected' : value.phase === 'connecting' ? 'connecting' : 'idle';
+      for (const id of ['actions', 'chat-call-context', 'transcript-call-context', 'menu-call-context', 'settings-call-context']) $(id).dataset.connectionState = connectionState;
       setTexts(['call-state', 'chat-call-state', 'transcript-call-state', 'menu-call-state', 'settings-call-state'], callState);
       setTexts(['call-mic-state', 'chat-call-mic-state', 'transcript-call-mic-state', 'menu-call-mic-state', 'settings-call-mic-state'], microphoneState);
       setHidden(['chat-call-context', 'transcript-call-context', 'menu-call-context', 'settings-call-context'], !active);
@@ -214,6 +230,7 @@ if (!window.clowderCompanion) {
       $('menu-hide-label').textContent = active ? '结束通话并隐藏' : '隐藏';
       const identity = value.identity;
       const companionIdentity = currentCompanionIdentity(identity);
+      renderMeetingStatus(document, meetingStatus.update(active ? identity?.meeting : null));
       const nativeSnapshot = normalizeNativeWork(identity?.nativeWork);
       const workLabel = nativeWorkLabel(nativeSnapshot, identity?.nativeActivity);
       const workState = workLabel ? `${identity.displayName}${workLabel}` : '';
@@ -229,7 +246,8 @@ if (!window.clowderCompanion) {
         $('menu-play').hidden = identity.skin !== 'xianxian-codex';
         const companionLabel = companionIdentity
           ? `${companionIdentity.title} · ${companionIdentity.partnerLabel}` : identity.displayName;
-        $('chat-name').textContent = threadTitle ?? companionLabel; $('menu-name').textContent = companionLabel;
+        $('chat-name').textContent = conversationSurface?.threadTitle ?? companionLabel; $('menu-name').textContent = companionLabel;
+        $('message').placeholder = `发消息给${identity.displayName}…`;
         $('call-message').placeholder = `发消息给${identity.displayName}…`;
         $('pet').setAttribute('aria-label', `${companionLabel}：点击或右键打开名片，拖动移动`);
         $('documents').querySelector('.menu-status').textContent = identity.documentsAllowed ? '开启' : '暂停';
@@ -247,6 +265,15 @@ if (!window.clowderCompanion) {
       if (showFailure) void controls.show('menu');
     },
   });
+  conversationSurface = new ConversationSurface({
+    document, client, controls, bubble, transcript,
+    getCallPhase: () => conversation.phase,
+    onAccepted: () => motion.signal('received'),
+    retryLegacy: text => conversation.send(text),
+    status,
+    scheduleInterval: setInterval,
+    cancelInterval: clearInterval,
+  });
   settingsView = new SettingsView({
     document,
     client,
@@ -280,21 +307,6 @@ if (!window.clowderCompanion) {
     motion,
     proactivePolicy: () => settingsValues?.proactivePolicy,
   });
-  async function readHistory() {
-    if (loading) return;
-    loading = true;
-    try {
-      const history = await client.readConversation();
-      $('history').textContent = history.hasMore ? '在 Cat Café 中查看更早聊天 ↗' : '在 Cat Café 中查看完整聊天 ↗';
-      threadTitle = history.threadTitle;
-      $('chat-name').textContent = threadTitle;
-      bubble.load(history.messages);
-      void controls.setAmbient(ambientPanel(conversation.phase !== 'idle'));
-      const serialized = JSON.stringify(history.messages);
-      if (serialized !== latestHistory) { latestHistory = serialized; transcript.load(history.messages); }
-    } catch { $('chat-status').textContent = '聊天记录暂未更新 · 正在说的话仍会显示'; }
-    finally { loading = false; }
-  }
   async function readCallTranscript() {
     if (transcriptLoading || !conversation.active || !conversation.callId) return;
     transcriptLoading = true;
@@ -302,41 +314,36 @@ if (!window.clowderCompanion) {
     try {
       const reply = await client.readTranscript();
       if (!conversation.active || conversation.callId !== callId) return;
-      callTranscript.load(reply);
+      if (!visibleText.loadTranscript(callId, reply, conversation.identity?.displayName)) return;
       foldedChatUnread.receiveMany(reply.rows);
-      callBubble.loadTranscript(reply.rows, conversation.identity?.displayName);
       $('call-transcript-status').textContent = reply.hasMore ? '这里只显示这次通话最近的内容' : '';
     } catch {
       if (conversation.active && conversation.callId === callId) $('call-transcript-status').textContent = '字幕暂未更新 · 实时内容仍会继续显示';
     } finally { transcriptLoading = false; }
   }
   $('share-badge').onclick = toggleScreen;
-  $('history').onclick = () => void client.openConversation().then(result => {
-    if (result.delivery === 'unconfirmed') status('打开聊天尚未确认 · 请从 Clowder 查看');
-  }).catch(error => status(explainError(error)));
-  async function submitMessage(event, inputId) {
-    event.preventDefault(); const input = $(inputId); const text = input.value;
+  async function submitCallMessage(event) {
+    event.preventDefault(); const input = $('call-message'); const text = input.value;
     if (await conversation.send(text)) {
       motion.signal('received');
       if (input.value === text) input.value = '';
-      latestHistory = undefined; void readHistory();
+      void conversationSurface.readHistory();
     }
   }
-  $('compose').onsubmit = event => submitMessage(event, 'message');
-  $('call-compose').onsubmit = event => submitMessage(event, 'call-message');
+  $('call-compose').onsubmit = submitCallMessage;
   $('call-unread').onclick = () => callTranscript.scrollToBottom();
   const unsubscribe = client.subscribe(event => {
     if (event.kind === 'media-stopped') void conversation.hostStopped(event.reason);
     if (event.kind === 'view-dismiss') controls.dismiss();
   });
   const statusMonitor = setInterval(() => void conversation.refresh(), 1000);
-  const historyMonitor = setInterval(() => void readHistory(), 3000);
   const transcriptMonitor = setInterval(() => void readCallTranscript(), 1000);
   const decisionMonitor = setInterval(() => { if (controls.panel !== 'decisions') void decisionPanel.read(); }, 15000);
   const workMonitor = setInterval(() => { if (controls.panel === 'menu') void workView.load(); }, 15000);
-  void conversation.refresh(); void readHistory(); void decisionPanel.read(); void settingsView.controller.load(); void controls.show('none');
+  void conversation.refresh(); conversationSurface.start(); void decisionPanel.read(); void settingsView.controller.load(); void controls.show('none');
   window.addEventListener('beforeunload', () => {
-    clearInterval(statusMonitor); clearInterval(historyMonitor); clearInterval(transcriptMonitor); clearInterval(decisionMonitor); clearInterval(workMonitor);
-    unsubscribe(); nativeTravel.close(); motion.close(); void conversation.end();
+    clearInterval(statusMonitor); clearInterval(transcriptMonitor); clearInterval(decisionMonitor); clearInterval(workMonitor);
+    conversationSurface.close();
+    unsubscribe(); visibleText.close(); nativeTravel.close(); motion.close(); void conversation.end();
   });
 }

package/renderer/transcript-view.mjs

@@ -20,6 +20,7 @@ export class TranscriptView {
     }
     row.textContent = (row.textContent + text).slice(-16000);
     if (pinned) this.container.scrollTop = this.container.scrollHeight;
+    return row;
   }
   finish({ role, transcript, turnId }) {
     if (!['user', 'assistant'].includes(role) || (turnId && this.completed.has(turnId))) return;

package/renderer/visible-text-receipt.mjs

@@ -0,0 +1,241 @@
+const MAX_PENDING = 64;
+const MAX_REPORT_ATTEMPTS = 3;
+const REPORT_RETRY_DELAYS_MS = [1_000, 2_000];
+const CLIPPING_OVERFLOW = new Set(['auto', 'clip', 'hidden', 'scroll']);
+
+function referenceKey(reference) {
+  if (reference?.kind === 'message' && typeof reference.messageId === 'string')
+    return `message:${reference.messageId}`;
+  if (reference?.kind === 'native-item' && typeof reference.nativeItemId === 'string')
+    return `native-item:${reference.nativeItemId}`;
+  return null;
+}
+
+function intersect(bounds, rect) {
+  return {
+    left: Math.max(bounds.left, rect.left),
+    top: Math.max(bounds.top, rect.top),
+    right: Math.min(bounds.right, rect.right),
+    bottom: Math.min(bounds.bottom, rect.bottom),
+  };
+}
+
+function nodeIsVisible(document, node) {
+  if (document.visibilityState !== 'visible' || node?.isConnected !== true) return false;
+  const view = document.defaultView;
+  if (!view) return false;
+  const rects = node.getClientRects?.();
+  if (!rects || rects.length === 0) return false;
+  const viewport = {
+    left: 0,
+    top: 0,
+    right: document.documentElement?.clientWidth ?? view.innerWidth,
+    bottom: document.documentElement?.clientHeight ?? view.innerHeight,
+  };
+  let visible = intersect(viewport, node.getBoundingClientRect());
+  for (let current = node; current; current = current.parentElement) {
+    if (current.hidden || current.getAttribute?.('aria-hidden') === 'true') return false;
+    const style = view.getComputedStyle?.(current);
+    if (style?.display === 'none' || style?.visibility === 'hidden' || style?.visibility === 'collapse'
+      || style?.opacity === '0') return false;
+    const clipsX = CLIPPING_OVERFLOW.has(style?.overflowX ?? style?.overflow);
+    const clipsY = CLIPPING_OVERFLOW.has(style?.overflowY ?? style?.overflow);
+    if ((clipsX || clipsY) && current !== node) {
+      const bounds = current.getBoundingClientRect?.();
+      if (bounds) visible = intersect(visible, {
+        left: clipsX ? bounds.left : visible.left,
+        right: clipsX ? bounds.right : visible.right,
+        top: clipsY ? bounds.top : visible.top,
+        bottom: clipsY ? bounds.bottom : visible.bottom,
+      });
+    }
+  }
+  return visible.right > visible.left && visible.bottom > visible.top;
+}
+
+function currentNodes(candidate) {
+  try {
+    const nodes = typeof candidate.nodes === 'function' ? candidate.nodes() : candidate.nodes;
+    return Array.isArray(nodes) ? nodes.filter(Boolean) : [];
+  } catch {
+    return [];
+  }
+}
+
+function errorCode(value) {
+  return typeof value?.code === 'string' ? value.code : undefined;
+}
+
+/**
+ * Reports the first assistant text that reaches a visible renderer surface.
+ * A post-mutation animation frame plus DOM/document/viewport checks is not a
+ * claim that the compositor painted, the window was unoccluded, or a person read it.
+ */
+export class FirstVisibleTextReceipt {
+  constructor({ document, report, now = Date.now, frame } = {}) {
+    this.document = document;
+    this.report = report;
+    this.now = now;
+    this.frame = frame ?? document?.defaultView?.requestAnimationFrame?.bind(document.defaultView);
+    this.pending = new Map();
+    this.rejected = new Set();
+    this.onVisibilityChange = () => this.retry();
+    this.onGeometryChange = () => this.retry();
+    document?.addEventListener?.('visibilitychange', this.onVisibilityChange);
+    document?.addEventListener?.('scroll', this.onGeometryChange, true);
+    document?.defaultView?.addEventListener?.('resize', this.onGeometryChange);
+  }
+
+  setCall(callId) {
+    callId ??= null;
+    if (callId === this.callId) return;
+    this.callId = callId;
+    this.pending.clear();
+    this.rejected.clear();
+    this.reported = false;
+    this.exhausted = false;
+  }
+
+  observe({ callId, reference, nodes } = {}) {
+    const key = referenceKey(reference);
+    if (!key || !callId || callId !== this.callId || this.reported || this.exhausted
+      || this.rejected.has(key)
+      || (!Array.isArray(nodes) && typeof nodes !== 'function')) return false;
+    const existing = this.pending.get(key);
+    const candidate = existing ?? { callId, reference, nodes, attempts: 0, retryAt: 0 };
+    candidate.nodes = nodes;
+    if (!existing && currentNodes(candidate).length === 0) return false;
+    this.pending.set(key, candidate);
+    while (this.pending.size > MAX_PENDING) this.pending.delete(this.pending.keys().next().value);
+    this.retry();
+    return true;
+  }
+
+  retry() {
+    if (this.scheduled || this.inFlight || this.reported || this.exhausted || !this.callId || this.pending.size === 0
+      || typeof this.frame !== 'function') return;
+    this.scheduled = true;
+    this.frame(() => {
+      this.scheduled = false;
+      void this.flush();
+    });
+  }
+
+  async flush() {
+    if (this.inFlight || this.reported || this.exhausted || !this.callId) return;
+    const entry = [...this.pending].find(([, candidate]) => currentNodes(candidate).some(node =>
+      nodeIsVisible(this.document, node)));
+    if (!entry) return;
+    const [key, candidate] = entry;
+    if (candidate.retryAt > this.now()) return;
+    const callId = this.callId;
+    const attempt = { callId, key };
+    this.inFlight = attempt;
+    candidate.attempts += 1;
+    let continueWithNext = false;
+    try {
+      const reply = await this.report({
+        event: 'first_text_visible', observedAt: this.now(), callId, reference: candidate.reference,
+      });
+      if (reply?.kind === 'ok' && this.callId === callId && this.pending.get(key) === candidate) {
+        this.reported = true;
+        this.pending.clear();
+      } else if (reply?.kind !== 'ok') {
+        continueWithNext = this.rejectCandidate(callId, key, candidate, reply);
+      }
+    } catch (error) {
+      continueWithNext = this.rejectCandidate(callId, key, candidate, error);
+    } finally {
+      if (this.inFlight === attempt) {
+        this.inFlight = undefined;
+        if (this.callId !== callId || continueWithNext) this.retry();
+      }
+    }
+  }
+
+  rejectCandidate(callId, key, candidate, failure) {
+    if (this.callId !== callId || this.pending.get(key) !== candidate) return false;
+    if (errorCode(failure) === 'invalid_request') {
+      this.pending.delete(key);
+      if (this.rejected.size >= MAX_PENDING) {
+        this.exhausted = true;
+        this.pending.clear();
+        return false;
+      }
+      this.rejected.add(key);
+      return true;
+    }
+    if (candidate.attempts >= MAX_REPORT_ATTEMPTS) {
+      // The first visible candidate was plausible but the Host never confirmed it.
+      // Preserve that result as unknown instead of reporting later text as the first.
+      this.exhausted = true;
+      this.pending.clear();
+      return false;
+    }
+    candidate.retryAt = this.now() + REPORT_RETRY_DELAYS_MS[candidate.attempts - 1];
+    return false;
+  }
+
+  close() {
+    this.document?.removeEventListener?.('visibilitychange', this.onVisibilityChange);
+    this.document?.removeEventListener?.('scroll', this.onGeometryChange, true);
+    this.document?.defaultView?.removeEventListener?.('resize', this.onGeometryChange);
+    this.pending.clear();
+    this.rejected.clear();
+  }
+}
+
+/** Owns the official companion's four subtitle projections and their receipt nodes. */
+export class VisibleTextPresentation {
+  constructor({ document, report, transcript, callTranscript, bubble, callBubble }) {
+    this.receipt = new FirstVisibleTextReceipt({ document, report });
+    Object.assign(this, { transcript, callTranscript, bubble, callBubble });
+  }
+
+  setCall(callId) {
+    this.callTranscript.setCall(callId);
+    this.receipt.setCall(callId);
+  }
+
+  appendLive(event, speaker) {
+    const reference = event.itemId
+      ? { kind: 'native-item', nativeItemId: event.itemId }
+      : undefined;
+    const stableNodes = [
+      this.transcript.append(event.role, event.text, true),
+      this.callTranscript.appendLive(event),
+    ];
+    this.bubble.append(event.role, event.text, speaker, reference);
+    this.callBubble.append(event.role, event.text, speaker, reference);
+    if (event.role === 'assistant' && reference && typeof event.text === 'string'
+      && event.text.trim().length > 0) this.receipt.observe({
+      callId: event.callId,
+      reference,
+      nodes: () => [
+        ...stableNodes,
+        ...this.bubble.rowsForReference(reference),
+        ...this.callBubble.rowsForReference(reference),
+      ],
+    });
+  }
+
+  loadTranscript(callId, reply, companionName) {
+    if (!this.callTranscript.load(reply)) return false;
+    this.callBubble.loadTranscript(reply.rows, companionName);
+    for (const row of reply.rows) {
+      if (row.role !== 'assistant' || row.text.trim().length === 0) continue;
+      const reference = { kind: 'message', messageId: row.messageId };
+      const transcriptRow = this.callTranscript.rowForMessage(row.messageId);
+      this.receipt.observe({
+        callId,
+        reference,
+        nodes: () => [transcriptRow, ...this.callBubble.rowsForReference(reference)],
+      });
+    }
+    this.receipt.retry();
+    return true;
+  }
+
+  retry() { this.receipt.retry(); }
+  close() { this.receipt.close(); }
+}