heyquilt 0.3.141 → 0.3.143

package/src/relay-webhooks.js

@@ -12,9 +12,10 @@
 // connects there with a subscription. Each keeps its own inbox state (what the webhook has told),
 // separate from quilt_inbox's, so a POSTed event still shows there when the agent asks.
 import { scanInbox } from './inbox.js'
+import { callEntries, liveCallOf } from './voice.js'
 import { hostedIn, newHosted } from './hosted-agents.js'
 import { readTasks } from './tasks.js'
-import { deliverWebhook, makeSubscription, webhookPayload, publicWebhookHost, WEBHOOK_EVENTS } from './webhooks.js'
+import { deliverWebhook, makeSubscription, webhookPayload, publicWebhookHost, takesEvent, WEBHOOK_EVENTS } from './webhooks.js'
 import { withContext } from './conversation.js'
 
 // A subscription with `coalesce` turns a burst into one POST: events are gathered for SETTLE_MS,
@@ -31,7 +32,13 @@ const inContext = (room, me, events) => {
   return withContext(events, all.filter((m) => m && m.id && (!m.to || m.to === me || m.by === me)), { names: [me], agent: true })
 }
 
-const scan = (doc, name, state) => scanInbox({ messages: chatFor(doc, name), tasks: readTasks(doc.getMap('tasks')), reader: { name, asAi: false, agent: true } }, state)
+const scan = (doc, name, state, room = null) => scanInbox({
+  messages: chatFor(doc, name),
+  tasks: readTasks(doc.getMap('tasks')),
+  // The call going on, which every agent of the session is in while it is transcribed.
+  voice: room && room.awareness ? callEntries(doc, { names: [name], call: liveCallOf(room.awareness), members: room.memberList ? room.memberList() : [] }) : [],
+  reader: { name, asAi: false, agent: true }
+}, state)
 
 /** The payload for a burst (oldest first): the first event exactly as one POST carries it, with every one of them in `events`. */
 export function batchPayload (payloads) {
@@ -106,14 +113,14 @@ export function hostedWebhooks ({ hosted, saveHosted, log = () => {}, fetch, del
       const name = (!at.hosted && (room.meta.members[account]?.name || r.name)) || sub.name
       // A session the subscription has not taken stock of yet (joined while the room was not loaded,
       // or an entry from before this relay): what is already there is not news.
-      if (r.webhookState === undefined) { r.webhookState = scan(room.doc, name, null).state; changed = true; continue }
-      const got = scan(room.doc, name, r.webhookState)
+      if (r.webhookState === undefined) { r.webhookState = scan(room.doc, name, null, room).state; changed = true; continue }
+      const got = scan(room.doc, name, r.webhookState, room)
       r.webhookState = got.state
       changed = true
       if (!got.events.length) continue
       const payloads = inContext(room, name, got.events)
         .map((e) => webhookPayload(e, { room: room.name, to: name, workspace: room.meta.workspace || '' }))
-        .filter((p) => sub.events.includes(p.event))
+        .filter((p) => takesEvent(sub.events, p.event))
       if (!payloads.length) continue
       if (sub.coalesce) queue(account, room.name, payloads)
       else for (const p of payloads) post(account, sub, p)
@@ -128,6 +135,9 @@ export function hostedWebhooks ({ hosted, saveHosted, log = () => {}, fetch, del
     const run = () => { try { check(room) } catch (err) { log(`[${room.name}] webhooks: ${err.message}`) } }
     room.doc.getArray('chat').observe(run)
     room.doc.getMap('tasks').observeDeep(run)
+    // What is said in a call to an agent in it.
+    room.doc.getArray('voiceLines').observe(run)
+    room.doc.getMap('voiceAgents').observe(run)
   }
 
   const restock = (h) => {

package/src/server.js

@@ -4804,9 +4804,11 @@ export function startServer ({ port = 4321, host = '0.0.0.0', dataDir = null, lo
     if (!publicKey) return reject(socket, 400, withUpdateSteps(NEEDS_UPDATE, { ascii: true }))
     const key = parsePublicKey(publicKey)
     if (!person || person.length > MAX_NAME || !key) return reject(socket, 400, 'Bad name or identity key')
-    // Every session keeps its files in branch documents: an app that can't sync them is told to update.
+    // An app that can't sync branch documents or squares is told to update. Apps from R3 on (0.3.143) say
+    // `quilts` and no longer `branches` (Seam's plans/quilt-source-control/16 flag 9); every app with
+    // `quilts` syncs branch documents too, so either one is enough.
     const features = String(url.searchParams.get('features') || '').split(',')
-    if (!features.includes('branches')) return reject(socket, 400, withUpdateSteps(NEEDS_UPDATE, { ascii: true }))
+    if (!features.includes('branches') && !features.includes('quilts')) return reject(socket, 400, withUpdateSteps(NEEDS_UPDATE, { ascii: true }))
     if (roomEnded(name)) return reject(socket, 410, ENDED_MESSAGE)
     const ip = clientIp(req)
     const starter = pass ? account : ip

package/src/session-follow.js

@@ -380,7 +380,9 @@ export class SquareFollower {
         log: (m) => { if (s.log) s.log(`🧵 ${m}`) },
         started: (b) => { here.starting = b },
         // Who changed what goes on as on the branch: this folder's edits on record as its member's.
-        onEdits: s.squareEdits ? (list, doc, held) => { if (this.here === here) s.squareEdits(list, doc, held) } : null
+        onEdits: s.squareEdits ? (list, doc, held) => { if (this.here === here) s.squareEdits(list, doc, held) } : null,
+        // A file the square wrote to disk: the app redraws its Files list and an open file (as for a change its watcher heard).
+        onWrite: (p) => { if (this.here === here && s.emit) s.emit('file-changed', { path: p }) }
       })
       if (this.here !== here) { await block.close().catch(() => {}); return }
       if (empty) s.joinedEmpty = false

package/src/session.js

@@ -57,7 +57,7 @@ import { RELAY_NAME as RELAY_BY, isRelay } from './ui/chat.js'
 import { cleanTyping, TYPING_MS, AGENT_TYPING_MS } from './ui/chat.js'
 import { DEFAULT_KEY } from './branchdocs.js'
 import { boxEntry } from './boxes.js'
-import { cleanVoice, newCallId, voiceView, inVoice, transcribeSetting, setTranscribe, addLine, readLines, MAX_IN_CALL, CALL_FULL } from './voice.js'
+import { cleanVoice, newCallId, voiceView, inVoice, transcribeSetting, setTranscribe, addLine, readLines, MAX_IN_CALL, CALL_FULL, callAgents, setAgentLeft, notInCall, callEntries, agentSays } from './voice.js'
 import { toolInUse } from './agents/common.js'
 import { squareShort } from '@heyquilt/seam/lib/quilts/errors.js'
 
@@ -885,6 +885,12 @@ export class Session extends EventEmitter {
       this.scheduleStatusWrite()
     })
     this.activity.observe(() => this.scheduleStatusWrite())
+    // An agent in a call is woken by what is said to it there.
+    if (this.kind === 'agent') {
+      const heard = (ev, tr) => this.scanInbox({ quiet: tr.origin === LOCAL, personas: false })
+      this.doc.getArray('voiceLines').observe(heard)
+      this.doc.getMap('voiceAgents').observe(heard)
+    }
     this.tasks.observe((ev, tr) => {
       this.scanInbox({ quiet: tr.origin === LOCAL })
       this.scheduleStatusWrite()
@@ -3137,6 +3143,35 @@ export class Session extends EventEmitter {
     return readLines(this.doc, opts)
   }
 
+  /** The call going on now (someone is in it): its id, or ''. */
+  liveCall () {
+    const v = this.voice()
+    return v.in || v.peers.length ? v.call : ''
+  }
+
+  /**
+   * An agent leaves the call going on (`leave`), or comes back to it. While transcripts are on every
+   * agent of the session is in it, muted until someone says its name.
+   */
+  voiceJoin ({ leave = false } = {}) {
+    if (this.kind !== 'agent') throw new Error(`In this session you are ${this.name}: people join calls in the Quilt app. The session's agents (and "${this.name}'s AI") are in every transcribed call.`)
+    const call = this.liveCall()
+    if (!call) throw new Error('There is no call going on in this session.')
+    setAgentLeft(this.doc, { call, name: this.name, leave })
+    if (leave) return { in: false, call }
+    const why = notInCall(this.doc, { call, name: this.name, members: this.members })
+    if (why) throw new Error(why)
+    const v = this.voice()
+    return { in: true, call, me: this.name, people: v.peers.map((p) => p.name), agents: v.agents.map((a) => a.name) }
+  }
+
+  /** What an agent in the call says out loud: a line every app in the call speaks in its voice. */
+  voiceSay (text) {
+    if (this.kind !== 'agent') throw new Error('Only agents speak in a call through Quilt.')
+    if (!this.mayTalk()) throw new Error(TALK_REFUSED)
+    return agentSays(this.doc, { call: this.liveCall(), name: this.name, text, members: this.members })
+  }
+
   // --------------------------------------------------------------- typing --
 
   /**
@@ -3470,6 +3505,8 @@ export class Session extends EventEmitter {
       events = this.inboxTracker.scan({
         messages,
         tasks,
+        // An agent in the session's call: what is said to it there.
+        voice: this.kind === 'agent' && this.conn ? callEntries(this.doc, { names: [this.name], call: this.liveCall(), members: this.members }) : [],
         reader: this.inboxReader()
       }, { quiet: quiet || !this.ready })
     } catch (err) {
@@ -5420,8 +5457,8 @@ export class Session extends EventEmitter {
         work: this.work,
         ...(myVoice && myVoice.me ? { voice: { muted: myVoice.muted, ...(myVoice.transcribing ? { transcribing: true } : {}), ...(myVoice.sharing ? { sharing: true } : {}) } } : {})
       },
-      // The session's call: whether the owner has transcripts on.
-      voice: { transcribe: transcribeSetting(this.doc, this.members).on },
+      // The session's call: whether the owner has transcripts on, and the agents in it.
+      voice: { transcribe: transcribeSetting(this.doc, this.members).on, agents: callAgents(this.doc, this.conn ? this.liveCall() : '', this.members) },
       peers,
       claims: [...this.claims.values()].sort((a, b) => a.ts - b.ts),
       tasks: this.taskList(),

package/src/source-live/live-block.js

@@ -68,13 +68,15 @@ export class LiveBlock {
    * since it last synced it; `skip(path)`: the session never syncs it (`skipDir(path)`: nor
    * anything in that folder); `record`: where block.json
    * goes; `reader`: a client for reads (square, tree, blobs, diff), else `client`; `log`;
+   * `onWrite(path)`: the block wrote (or deleted) that file on disk from the square, so whoever shows the
+   * folder hears of it without waiting on a file watcher (macOS's FSEvents drops events under load).
    * `onEdits(list, doc, held)`: the edits this folder made, as each scan takes them into the live
    * document `doc` ([{ path, kind: created|edited|deleted, detail, before?, after? }]), so the session
    * keeps them on record (who changed what) as it did on its branch; `held()` resolves once the relay
    * holds them.
    */
-  constructor ({ dir, client, quilt, square, paths = [''], relay, api = '', inPlace = null, skip = null, skipDir = null, record = null, reader = null, log = null, onEdits = null, started = null, stepMs = STEP_MS }) {
-    Object.assign(this, { dir: path.resolve(dir), client, quilt, squareRef: square, paths, relayOpts: relay, api, inPlace, skip, skipDir, record, reader: reader || client, logFn: log, onEdits, stepMs })
+  constructor ({ dir, client, quilt, square, paths = [''], relay, api = '', inPlace = null, skip = null, skipDir = null, record = null, reader = null, log = null, onEdits = null, onWrite = null, started = null, stepMs = STEP_MS }) {
+    Object.assign(this, { dir: path.resolve(dir), client, quilt, squareRef: square, paths, relayOpts: relay, api, inPlace, skip, skipDir, record, reader: reader || client, logFn: log, onEdits, onWrite, stepMs })
     // `started(block)`: told as it starts to open, so whoever waits for it can stop it part-way (abandon).
     if (started) started(this)
     this.known = {} // path -> { blob, mode, size, mtimeMs }: what is on disk as this block last read or wrote it
@@ -564,7 +566,7 @@ export class LiveBlock {
 
   write (p, want) {
     const abs = path.join(this.dir, ...p.split('/'))
-    if (!want) { fs.rmSync(abs, { force: true }); delete this.known[p]; this.lastText.delete(p); return }
+    if (!want) { fs.rmSync(abs, { force: true }); delete this.known[p]; this.lastText.delete(p); this.wrote(p); return }
     fs.mkdirSync(path.dirname(abs), { recursive: true })
     const tmp = path.join(path.dirname(abs), `.${path.basename(abs)}.quilt-tmp`)
     if (want.mode === 'symlink') { fs.rmSync(abs, { force: true }); fs.symlinkSync(want.bytes.toString('utf8'), abs) } else {
@@ -576,8 +578,11 @@ export class LiveBlock {
     if (this.doc && liveMaps(this.doc).files.has(p) && want.bytes.length <= MAX_LIVE_TEXT) this.lastText.set(p, { id: this.known[p].blob, text: want.bytes.toString('utf8') })
     else this.lastText.delete(p)
     this.saveSoon()
+    this.wrote(p)
   }
 
+  wrote (p) { if (this.onWrite) { try { this.onWrite(p) } catch {} } }
+
   /** Moves the block to the document's head (a commit by anyone in the square), then writes the overlay out. */
   async pull () {
     if (this.closed) return

package/src/status.js

@@ -46,7 +46,7 @@ export function renderStatus (st, { asAi = false, mentionYours = false } = {}) {
   }
   out.push('')
   const inCall = [...(st.me.voice ? [{ name: st.me.name, ...st.me.voice }] : []), ...st.peers.filter((p) => p.voice).map((p) => ({ name: p.name, ...p.voice }))]
-  const call = voiceStatusLines(inCall, !!(st.voice && st.voice.transcribe))
+  const call = voiceStatusLines(inCall, !!(st.voice && st.voice.transcribe), (st.voice && st.voice.agents) || [], st.me.kind === 'agent' ? st.me.name : '')
   if (call.length) out.push(...call, '')
   if (st.agents && st.agents.length) {
     out.push('## Agents and how Quilt wakes them')

package/src/ui/agent-guide.js

@@ -62,6 +62,7 @@ export function agentGuide ({ apiUrl = 'https://api.heyquilt.com', via = 'both'
     `- Read: \`quilt_read_messages\`${cli ? ` (CLI: \`quilt messages\`, or \`--all\` for history). With the CLI, every quilt answer also starts with anything new for you` : ''}.`,
     `- What wakes you (a direct message, a mention, a task) comes with the conversation before it: what you and its sender said to each other, directly or in the room, oldest first, and the message a reply answers (in \`quilt_inbox\`, in the webhook's "context", in a channel event, in your tool's hooks). "That", "it", "again" and "retry" mean the latest thing there. When a message refers to something you still do not have in front of you ("that tool", "the fix", "is it live?"), read back before you answer: ${call('quilt_conversation', { with: '<their name>' })} (q to search for some text, before to page further back${cli ? '; CLI: `quilt messages --with <name>`, `--grep <text>`, `--before <id>`' : ''}). Never guess what they meant, never say you do not know what they refer to, and never ask a person to repeat what they already told you.`,
     `- People also talk in voice calls in the Quilt app. \`quilt_status\` says who is in a call and whether it is transcribed. When it is, read what was said before acting on it: ${call('quilt_voice_transcript', {})}. When it is not, ask in chat what was decided.`,
+    `- While a call is transcribed, every agent of the session is in it, like a person but muted. When someone says your name you are unmuted and woken with what they said: answer out loud with ${call('quilt_voice_say', { text: '...' })}, in one to three short spoken sentences (lists, code and links go in chat). What is said in the next minute and a half wakes you too; then you mute again. Stay quiet when a line is not for you. ${call('quilt_voice_join', { leave: true })} leaves the call.`,
     '- A person and their AI share the person\'s name. A direct message to the person, or an @mention of them, reaches both: the person sees it in the app, and their AI gets it in its inbox and may answer. To give their AI work, assign a task to the person with to_ai: true. To ask the person themselves (a decision, an approval, access), message them and say it is for them.',
     '- Other agents are reached the same way, by their own name: an @mention or a direct message wakes them (through their inbox, their webhook or their next tool call). They answer you like they answer a person.',
     '- When to talk: when you start something, when you need something from someone, when you are blocked, and when you finish. Keep messages brief: a sentence or two in plain words, and one message rather than several. No play-by-play, recaps, logs or pasted code.',

package/src/ui/app.css

@@ -478,6 +478,9 @@ textarea.input { height: auto; padding: 10px 12px; resize: vertical; }
 .call-face.talk { box-shadow: 0 0 0 2px var(--panel), 0 0 0 4px var(--ok); }
 .call-muted { position: absolute; right: -4px; bottom: -3px; width: 12px; height: 12px; display: grid; place-items: center; border-radius: 50%; background: var(--panel); color: var(--bad); }
 .call-muted svg { width: 9px; height: 9px; }
+/* An agent in the call: the agent mark on its face, top right (its mute mark stays bottom right). */
+.call-ai { position: absolute; right: -4px; top: -4px; width: 12px; height: 12px; display: grid; place-items: center; border-radius: 50%; background: var(--violet); color: #fff; box-shadow: 0 0 0 1.5px var(--panel); }
+.call-ai svg { width: 8px; height: 8px; }
 .call-tx { display: inline-flex; align-items: center; gap: 5px; height: 22px; padding: 0 8px; border: 1px solid var(--border); border-radius: 999px; background: none; color: var(--muted); font: 500 11.5px var(--sans); cursor: pointer; white-space: nowrap; }
 .call-tx:hover, .call-tx[aria-expanded="true"] { background: var(--panel-2); }
 .call-tx .d { width: 6px; height: 6px; border-radius: 50%; background: var(--faint); }

package/src/ui/seam-model.js

@@ -17,12 +17,22 @@ const quiltName = (st) => (st.quilt && (st.quilt.slug || st.quilt.id)) || 'the q
 /** The square this folder follows, as Seam names it (`sq:` and the end of its id), or ''. */
 export const squareName = (st) => (st && st.square && st.square.short ? `sq:${st.square.short}` : '')
 
+/** The branch whose square this folder follows (`st.squares` is branch -> square name), or ''. */
+export function squareBranch (st) {
+  const name = squareName(st)
+  const hit = name && Object.entries(st.squares || {}).find(([, sq]) => sq === name)
+  return hit ? hit[0] : (name && st.branch) || ''
+}
+
+/** The square as a person reads it: its branch (a square's id means nothing to anyone), or its id when no branch is known. */
+const squareWords = (st) => squareBranch(st) || squareName(st)
+
 /**
  * The source control button at layout 3: { label, title, aria } for the square this folder follows, or
  * why it follows none yet. `label` '' leaves the button with its icon and count only.
  */
 export function seamChip (st) {
-  const name = squareName(st)
+  const name = squareWords(st)
   const s = st.square
   const label = name
   const title = st.moving ? movingCard(st).text
@@ -30,13 +40,13 @@ export function seamChip (st) {
     : st.readOnly && !(s && !s.opening && !s.problem)
     ? `Your edits here are not reaching anyone: ${st.readOnly.why}`
     : !s ? `This session's code is committed and merged in Seam (${quiltName(st)}), and this folder is not following a square yet.`
-      : s.problem ? `This folder's work belongs in square ${name}, but it can't follow it yet: ${s.problem}`
-        : s.opening ? `This folder is starting to follow square ${name}.`
-          : [`This folder's work is live in square ${name} of ${quiltName(st)}. Click to commit it or merge it onto main.`, ...catchUpWords(s)].join(' ')
-  // The number is the square's work not committed yet (everyone's on it): what a commit takes.
+      : s.problem ? `This folder's work belongs on ${name}, but it can't follow it yet: ${s.problem}`
+        : s.opening ? `This folder is starting to follow ${name}.`
+          : [`This folder's work is live on ${name} in ${quiltName(st)}, with everyone else's there. Click to commit it or merge it onto main.`, ...catchUpWords(s)].join(' ')
+  // The number is the square's work not committed yet (everyone's on it, not only yours): what a commit takes.
   const count = s && s.work ? s.work.count : 0
-  const countLine = count ? `${files(count)} not committed in this square.` : ''
-  return { label, title, aria: `Source control: ${name ? `square ${name}` : 'Seam'}${count ? `, ${files(count)} not committed` : ''}`, count, countLine }
+  const countLine = count ? `${files(count)} changed on ${name} and not committed yet, by everyone on it.` : ''
+  return { label, title, aria: `Source control: ${name || 'Seam'}${count ? `, ${files(count)} not committed` : ''}`, count, countLine }
 }
 
 const files = (n) => `${n} file${n === 1 ? '' : 's'}`
@@ -98,10 +108,10 @@ export function squareCard (st) {
  */
 export function squarePanelHtml (st, { busy = '' } = {}) {
   const s = st.square
-  const name = squareName(st)
+  const name = squareWords(st)
   const following = s && !s.opening && !s.problem
   const viewer = !!(st.access && st.access.role === 'viewer')
-  const head = `<div class="br-here sq-here"><div class="br-title">${name ? `Square <b>${esc(name)}</b> of ${esc(quiltName(st))}` : `Code committed and merged in Seam: <b>${esc(quiltName(st))}</b>`}</div>`
+  const head = `<div class="br-here sq-here"><div class="br-title"${squareName(st) ? ` title="Seam's square ${esc(squareName(st))}"` : ''}>${name ? `Live work on <b>${esc(name)}</b> in ${esc(quiltName(st))}` : `Code committed and merged in Seam: <b>${esc(quiltName(st))}</b>`}</div>`
   // While the session moves, nothing is committed or merged from here: the move is what it waits for.
   if (st.moving) return `${head}<div class="br-sub warn">${esc(movingCard(st).text)}</div></div>`
   if (st.squareless) {
@@ -112,13 +122,13 @@ export function squarePanelHtml (st, { busy = '' } = {}) {
     const why = st.readOnly ? st.readOnly.why : s ? s.line : 'This folder is not following a square yet: Quilt opens it by itself once the session says which.'
     return `${head}<div class="br-sub${st.readOnly || (s && s.problem) ? ' warn' : ''}">${esc(why)}</div></div>`
   }
-  const sub = '<div class="br-sub">Every edit here is live for everyone on this square. Commit it to keep a version; merge it to put it on main (GitHub gets main from Seam by itself).</div>'
+  const sub = `<div class="br-sub">Everyone on ${esc(name)} edits this together, live: the files below are everyone's edits, not only yours. Commit them to keep a version; merge them to put them on main (GitHub gets main from Seam by itself).</div>`
   const caught = catchUpWords(s)
   const work = (caught.length ? `<div class="br-sub${s.conflicted ? ' warn' : ''}">${esc(caught.join(' '))}</div>` : '') + workHtml(s.work)
   if (viewer) return `${head}${sub}${work}</div>`
   return `${head}${sub}${work}
     <form class="cp-commit" data-square-commit><input class="input" name="message" placeholder="What this commit is" aria-label="Commit message" maxlength="2000" required autocomplete="off"${busy ? ' disabled' : ''}><button type="submit" class="btn sm primary"${busy ? ' disabled' : ''}>${busy === 'commit' ? 'Committing…' : 'Commit'}</button></form>
-    <div class="sq-acts"><button type="button" class="btn sm" data-square-merge${busy ? ' disabled' : ''} title="Commits what is not committed yet, then merges ${esc(name)} onto main once its checks pass">${busy === 'merge' ? 'Merging…' : 'Merge onto main'}</button></div>
+    <div class="sq-acts"><button type="button" class="btn sm" data-square-merge${busy ? ' disabled' : ''} title="Commits what is not committed yet, then merges it onto main once its checks pass">${busy === 'merge' ? 'Merging…' : 'Merge onto main'}</button></div>
   </div>`
 }
 
@@ -201,7 +211,7 @@ export function squaresSideHtml (st) {
   add(mine, 'you')
   for (const p of st.peers || []) if (p && p.name && p.branch && squares[p.branch] && !(p.persona && p.of === (st.me && st.me.name))) add(squares[p.branch], p.mine ? 'your AI' : p.name)
   const n = st.square && st.square.work ? st.square.work.count : 0
-  const rows = [...groups].map(([sq, who]) => `<li class="sq-side-row${sq === mine ? ' on' : ''}"><div><b>${esc(sq)}</b>${branchOf[sq] ? ` <span class="hint">${esc(branchOf[sq])}</span>` : ''}</div><div class="hint">${esc(who.join(', '))}${sq === mine ? ` · ${n ? `${files(n)} not committed` : 'all committed'}` : ''}</div></li>`).join('')
+  const rows = [...groups].map(([sq, who]) => `<li class="sq-side-row${sq === mine ? ' on' : ''}"><div title="Seam's square ${esc(sq)}"><b>${esc(branchOf[sq] || sq)}</b></div><div class="hint">${esc(who.join(', '))}${sq === mine ? ` · ${n ? `${files(n)} not committed` : 'all committed'}` : ''}</div></li>`).join('')
   // The pane's own title says Squares already (session.js, as it says Boxes outside Seam): no second heading.
   return rows ? `<ul class="sq-side">${rows}</ul>` : ''
 }

package/src/ui/voice-call.js

@@ -33,6 +33,42 @@ export class VoiceCall {
     this.view = null
     this.sending = null
     this.again = false
+    this.agentTalking = null // the agent whose line is playing now
+    this.spoken = Promise.resolve() // agents' lines, played one after another
+  }
+
+  /**
+   * An agent's line (a WAV, base64), played after any before it. While it plays our microphone
+   * sends nothing to the transcript, so the agent's voice from our speakers is never written down
+   * as ours (and never wakes the agent again).
+   */
+  play (name, wav) {
+    this.spoken = this.spoken.then(() => this.playNow(name, wav)).catch((err) => console.warn('agent voice', err.message))
+    return this.spoken
+  }
+
+  async playNow (name, wav) {
+    if (!this.in || !this.ctx) return
+    const bytes = Uint8Array.from(atob(wav), (c) => c.charCodeAt(0))
+    const audio = await this.ctx.decodeAudioData(bytes.buffer)
+    if (!this.in) return
+    const src = this.ctx.createBufferSource()
+    src.buffer = audio
+    src.connect(this.ctx.destination)
+    // What we were saying goes to the transcript now, before the agent speaks.
+    this.segmenter?.end()
+    this.agentTalking = name
+    this.onChange()
+    await new Promise((resolve) => {
+      this.stopAgent = () => { try { src.stop() } catch {} resolve() }
+      src.onended = resolve
+      src.start()
+    })
+    // The room's echo dies down before the microphone counts again.
+    await new Promise((resolve) => setTimeout(resolve, 300))
+    this.stopAgent = null
+    this.agentTalking = null
+    this.onChange()
   }
 
   /** Joins: the microphone opens (the browser or system may ask first), then everyone already in is called. */
@@ -61,6 +97,7 @@ export class VoiceCall {
     this.in = false
     clearInterval(this.timer)
     clearInterval(this.retry)
+    this.stopAgent?.()
     this.stopShare()
     this.segmenter?.end()
     for (const id of [...this.links.keys()]) this.drop(id)
@@ -266,7 +303,7 @@ export class VoiceCall {
         this.clip({ audio: toBase64(wavBytes(pcm, 16000)), seconds: pcm.length / 16000 }).catch((err) => console.warn('voice clip', err.message))
       }
     })
-    this.worklet.port.onmessage = (e) => { if (this.transcribing && !this.muted) this.segmenter.push(e.data) }
+    this.worklet.port.onmessage = (e) => { if (this.transcribing && !this.muted && !this.agentTalking) this.segmenter.push(e.data) }
     this.ctx.createMediaStreamSource(this.stream).connect(this.worklet)
     // A node nobody listens to may never run: it goes on to the speakers, silenced.
     const hush = this.ctx.createGain()

package/src/ui/voice.js

@@ -11,7 +11,7 @@ import { screenShown, shareRefused } from './voice-model.js'
 import { playSound } from './sounds.js'
 
 const views = new Map() // session id -> the call as that session's server sees it
-const EMPTY = { peers: [], transcribe: false, recent: [], calls: [], since: 0, call: '' }
+const EMPTY = { peers: [], transcribe: false, recent: [], calls: [], since: 0, call: '', agents: [] }
 const FULL = 6 // voice.js MAX_IN_CALL: everyone connects to everyone, so a call holds six
 let live = null // { sid, call }: the call this window is in (one at a time)
 let shown = null // { id, who, chatChanged }: the session on screen
@@ -31,6 +31,7 @@ let selfClosed = false // our own screen's preview closed, until we stop sharing
 let selfVideo = null // our own screen's preview: one element, like screenVideo
 let drawn = '' // what the pinned call was last drawn from: redrawn only when that changes, so a click is never lost mid-press
 let callsDrawn = ''
+let heard = { call: '', since: 0, ids: new Set() } // agents' lines this window has played (or let pass) in the call it is in
 
 const NO_SCREEN = "Quilt can't see your screen. Allow it for Quilt in System Settings → Privacy & Security → Screen & System Audio Recording, then quit and reopen Quilt."
 const NO_MIC = "Quilt can't use your microphone. Allow it for Quilt in System Settings → Privacy & Security → Microphone, then try again."
@@ -91,7 +92,7 @@ export function voiceEvent (id, view) {
     if (view.in && !(live && live.sid === id)) { api('POST', `/api/sessions/${id}/voice`, { in: false }).then((v) => voiceEvent(id, v), () => {}); return }
   }
   // The call we're in: it plays the join and leave sounds itself (onPeople), so a relay blip plays neither.
-  if (live && live.sid === id) live.call.update(view)
+  if (live && live.sid === id) { live.call.update(view); speakAgents(id, view) }
   if (shown && shown.id === id) {
     renderCall()
     // A call ended: its card joins the chat. The call going on now changes every few seconds and
@@ -128,6 +129,7 @@ async function join (id, transcribing) {
       onPeople: (sound) => playSound(sound, { force: true })
     })
     live = { sid: id, call }
+    heard = { call: '', since: Date.now(), ids: new Set() }
     await call.join({ transcribing })
     playSound('callIn', { force: true })
   } catch (err) {
@@ -229,14 +231,47 @@ async function onClick (e) {
   }
 }
 
-// A face in the call; `key` ('me' or a presence id) lets its speaking ring change without redrawing anything.
+/**
+ * An agent's lines in the call we're in, said since we joined: each spoken in its voice (quilt-api,
+ * Fluxions), one after another. Without voices on the server the line still shows in the call.
+ */
+function speakAgents (id, view) {
+  if (!live || live.sid !== id || !live.call.in) return
+  if (heard.call !== view.call) heard = { call: view.call, since: heard.since, ids: new Set() }
+  for (const l of view.recent) {
+    if (!l.spoken || heard.ids.has(l.id)) continue
+    heard.ids.add(l.id)
+    if (l.ts < heard.since - 2000) continue
+    const call = live.call
+    api('GET', `/api/sessions/${id}/voice/speech?line=${encodeURIComponent(l.id)}`).then((r) => {
+      if (live && live.call === call && r && r.audio) call.play(l.by, r.audio)
+    }, (err) => {
+      // Once a call: voices not set up, or the month's share heard.
+      if (!heard.told && (err.status === 503 || err.status === 429 || err.status === 404)) { heard.told = true; toast(err.status === 404 ? "Agents' voices aren't on Quilt's servers yet: their lines show in the call." : err.message) }
+    })
+  }
+}
+
+// A face in the call; `key` ('me', a presence id, or 'agent:<name>') lets its speaking ring change without redrawing anything.
 const face = (key, name, color, muted) => `<span class="call-face" data-face="${key}" title="${esc(name)}${muted ? ' (muted)' : ''}">${avatar(name, color)}${muted ? `<span class="call-muted">${I.micOff}</span>` : ''}</span>`
+// An agent in the call: like a person, with the agent mark, muted until someone says its name; its ring lights while its line plays.
+const agentFace = (a, color, now) => {
+  const muted = !(a.until > now)
+  return `<span class="call-face agent" data-face="agent:${esc(a.name)}" title="${esc(a.name)}, an agent: ${muted ? 'muted until someone says its name' : 'listening'}">${avatar(a.name, color)}<span class="call-ai" aria-hidden="true">${I.bot}</span>${muted ? `<span class="call-muted">${I.micOff}</span>` : ''}</span>`
+}
+// An agent unmuted by its name mutes again on its own: draw the call again when that happens.
+let unmuteTimer = null
+function redrawWhenMuted (agents, now) {
+  clearTimeout(unmuteTimer)
+  const next = Math.min(...agents.map((a) => a.until).filter((u) => u > now))
+  if (Number.isFinite(next)) unmuteTimer = setTimeout(renderCall, next - now + 50)
+}
 const minutes = (from, to = Date.now()) => { const m = Math.round((to - from) / 60000); return m < 1 ? 'under a minute' : `${m} min` }
 const names = (list) => list.length <= 2 ? list.join(' and ') : `${list.slice(0, -1).join(', ')} and ${list[list.length - 1]}`
 
 /** One line of what someone said. */
 function lineHtml (l, colorOf) {
-  return `<div class="call-line">${avatar(l.by, colorOf(l.by))}<div class="call-said"><div class="call-whosaid"><b>${esc(l.by)}</b><span>${esc(clock(l.ts))}</span></div><div>${esc(l.text)}</div></div></div>`
+  return `<div class="call-line">${avatar(l.by, colorOf(l.by))}<div class="call-said"><div class="call-whosaid"><b>${esc(l.by)}</b>${l.spoken ? '<span class="ai-tag">AI</span>' : ''}<span>${esc(clock(l.ts))}</span></div><div>${esc(l.text)}</div></div></div>`
 }
 
 /** Rings on whoever speaks now, without redrawing (it changes many times a second). */
@@ -244,7 +279,7 @@ function paintTalk () {
   if (!live || !shown || live.sid !== shown.id) return
   for (const el of document.querySelectorAll('#call-pin [data-face]')) {
     const k = el.dataset.face
-    el.classList.toggle('talk', live.call.speaking(k === 'me' ? 'me' : Number(k)))
+    el.classList.toggle('talk', k.startsWith('agent:') ? live.call.agentTalking === k.slice(6) : live.call.speaking(k === 'me' ? 'me' : Number(k)))
   }
 }
 
@@ -274,7 +309,11 @@ export function renderCall () {
   // Closing the preview lasts until we stop sharing; the next share shows it again.
   if (!c?.sharing) selfClosed = false
   renderSelf(c)
-  const key = JSON.stringify([id, here, joining, asking, options, owner, elsewhere, view.transcribe, others.map((p) => [p.id, p.name, p.color, p.muted, p.transcribing, p.sharing]), me.name, me.color, c && [c.muted, c.transcribing, c.sharing], selfClosed, showing, minutes(view.since || Date.now())])
+  const now = Date.now()
+  const agents = view.agents || []
+  redrawWhenMuted(agents, now)
+  const agentFaces = agents.map((a) => agentFace(a, colorOf(a.name), now)).join('')
+  const key = JSON.stringify([id, here, joining, asking, options, owner, elsewhere, view.transcribe, others.map((p) => [p.id, p.name, p.color, p.muted, p.transcribing, p.sharing]), agents.map((a) => [a.name, a.until > now]), me.name, me.color, c && [c.muted, c.transcribing, c.sharing], selfClosed, showing, minutes(view.since || Date.now())])
   if (key === drawn && (pin.hidden || pin.childElementCount)) { paintLines(pin, lines); paintTalk(); return }
   drawn = key
   linesDrawn = null
@@ -287,7 +326,7 @@ export function renderCall () {
       <p class="cp-note">This call is transcribed: what you say is written down for everyone in the session, agents included. Audio is never kept.</p>
       <div class="cp-two"><button type="button" class="btn sm primary" data-call-join="transcribed">${others.length ? 'Join' : 'Start'}, transcribed</button><button type="button" class="btn sm" data-call-join="plain">${others.length ? 'Join' : 'Start'}, not transcribed</button></div>`
   } else if (!here) {
-    pin.innerHTML = `<div class="cp-top"><span class="call-dot"></span><span class="call-faces">${others.map((p) => face(p.id, p.name, p.color, p.muted)).join('')}</span>
+    pin.innerHTML = `<div class="cp-top"><span class="call-dot"></span><span class="call-faces">${others.map((p) => face(p.id, p.name, p.color, p.muted)).join('')}${agentFaces}</span>
         <span class="cp-what"><b>${esc(names(others.map((p) => p.name)))} ${others.length === 1 ? 'is' : 'are'} in a call</b><span class="call-sub">${[dur, view.transcribe ? 'transcribed' : '', ...others.filter((p) => p.sharing).map((p) => `${esc(p.name)} sharing a screen`)].filter(Boolean).join(' · ')}</span></span>
         <button type="button" class="btn sm primary cp-end" data-call-join="ask" ${elsewhere ? 'disabled title="Leave your other call first"' : others.length >= FULL ? 'disabled title="This call is full: calls hold six people for now."' : ''}>Join</button></div>`
   } else {
@@ -305,7 +344,7 @@ export function renderCall () {
         <button type="button" class="call-ic cp-end" data-call-share aria-pressed="${c.sharing}" title="${c.sharing ? 'Stop sharing your screen' : 'Share your screen'}" aria-label="${c.sharing ? 'Stop sharing your screen' : 'Share your screen'}">${c.sharing ? I.screenOff : I.screenShare}</button>
         <button type="button" class="call-ic" data-call-mute aria-pressed="${c.muted}" title="${c.muted ? 'Unmute' : 'Mute'}" aria-label="${c.muted ? 'Unmute' : 'Mute'}">${c.muted ? I.micOff : I.mic}</button>
         <button type="button" class="call-ic bad" data-call-leave title="Leave the call" aria-label="Leave the call">${I.hangUp}</button></div>
-      <div class="cp-row"><span class="call-faces">${face('me', me.name, me.color, c.muted)}${others.map((p) => face(p.id, p.name, p.color, p.muted)).join('')}</span>${tag}</div>
+      <div class="cp-row"><span class="call-faces">${face('me', me.name, me.color, c.muted)}${others.map((p) => face(p.id, p.name, p.color, p.muted)).join('')}${agentFaces}</span>${tag}</div>
       ${options ? `<div class="cp-opts">${myChoice}${ownerChoice}</div>` : ''}
       ${c.sharing ? `<div class="cp-share">${I.screenShare}<span class="cp-says">Sharing your screen</span>${selfClosed ? '<button type="button" class="btn sm ghost" data-self-share="open" title="Show what you share, in the corner">Show</button>' : ''}<button type="button" class="btn sm" data-call-share>Stop sharing</button></div>` : ''}
       ${others.filter((p) => p.sharing).map((p) => `<div class="cp-share">${I.screenShare}<span class="cp-says">${esc(p.name)} is sharing a screen</span>${p.id === showing ? '<span class="call-sub">in the middle</span>' : `<button type="button" class="btn sm" data-screen-show="${p.id}">Show</button>`}</div>`).join('')}

package/src/ui-server.js

@@ -20,7 +20,7 @@ import { autowork, autoConfig, publicConfig, AUTO_MODES } from './autowork.js'
 import { migrateDir } from './legacy.js'
 import { writePrivateJson } from './private-file.js'
 import { autoToolFor } from './autowork-tools.js'
-import { readAccount, saveAccount, clearAccount, clearAccountIf, resumeAccount, startLink, waitForLink, fetchMe, signOut, revokeToken, accountFromProfile, renameSession, createAgentInvite, listAgents, agentDetails, listAccessTypes, listCollaborators, listGrants, putGrant, deleteGrant, inviteToSession, listSessionInvites, cancelSessionInvite, listWorkspaces, listOrgs, createWorkspace, getWorkspace, updateWorkspace, deleteWorkspace, putWorkspaceMember, removeWorkspaceMember, getAgentPlacement, putAgentPlacement, listOrgAgents, putWorkspaceAgent, deleteWorkspaceAgent, createWorkspaceAgentInvite, excludeSessionAgent, listExcludedSessionAgents, listSessionAgents, includeSessionAgent, setSessionWorkspace, moveSessionToWorkspace, listWorkspaceInvites, inviteToWorkspace, cancelWorkspaceInvite, listMyInvites, answerMyInvite, announceSessionStarted, announceWhenReported, listWorkspaceFiles, createWorkspaceFile, confirmWorkspaceFile, workspaceFileDownload, updateWorkspaceFile, deleteWorkspaceFile, createWorkspaceFolder, listWorkspaceFileVersions, listSandboxKeys, createSandboxKey, rotateSandboxKey, revokeSandboxKey, removeSandboxKey, aiVmFollow, aiVmUnfollow, aiVmStatus, aiVmConnect, aiVmUpdate, aiVmSignIn, aiVmWake, aiVmDisconnect, voiceIce, transcribeVoice, apiUrl } from './account.js'
+import { readAccount, saveAccount, clearAccount, clearAccountIf, resumeAccount, startLink, waitForLink, fetchMe, signOut, revokeToken, accountFromProfile, renameSession, createAgentInvite, listAgents, agentDetails, listAccessTypes, listCollaborators, listGrants, putGrant, deleteGrant, inviteToSession, listSessionInvites, cancelSessionInvite, listWorkspaces, listOrgs, createWorkspace, getWorkspace, updateWorkspace, deleteWorkspace, putWorkspaceMember, removeWorkspaceMember, getAgentPlacement, putAgentPlacement, listOrgAgents, putWorkspaceAgent, deleteWorkspaceAgent, createWorkspaceAgentInvite, excludeSessionAgent, listExcludedSessionAgents, listSessionAgents, includeSessionAgent, setSessionWorkspace, moveSessionToWorkspace, listWorkspaceInvites, inviteToWorkspace, cancelWorkspaceInvite, listMyInvites, answerMyInvite, announceSessionStarted, announceWhenReported, listWorkspaceFiles, createWorkspaceFile, confirmWorkspaceFile, workspaceFileDownload, updateWorkspaceFile, deleteWorkspaceFile, createWorkspaceFolder, listWorkspaceFileVersions, listSandboxKeys, createSandboxKey, rotateSandboxKey, revokeSandboxKey, removeSandboxKey, aiVmFollow, aiVmUnfollow, aiVmStatus, aiVmConnect, aiVmUpdate, aiVmSignIn, aiVmWake, aiVmDisconnect, voiceIce, transcribeVoice, speakVoice, apiUrl } from './account.js'
 import { aiName } from './persona.js'
 import { effectiveAccess, builtinType } from './session-access.js'
 import { cleanSessionName, BAD_SESSION_NAME, SESSION_NAME_MAX } from './session-name.js'
@@ -248,7 +248,7 @@ export async function startUi ({ port = 7420, onShutdown, preview = false, repor
   // below — it returns earlier — so it needs no entry here.)
   const UNRECORDED = new Set(['POST /api/report'])
   // Sent many times a call (call setup, each clip of speech): only kept when slow or failed.
-  const QUIET_WHEN_OK = new Set(['POST /api/sessions/:id/voice', 'POST /api/sessions/:id/voice/clip'])
+  const QUIET_WHEN_OK = new Set(['POST /api/sessions/:id/voice', 'POST /api/sessions/:id/voice/clip', 'GET /api/sessions/:id/voice/speech'])
   function recordRoute (key, { startedAt, status, body, error }) {
     // The reporter's own `enabled()` is the single gate on the "report" setting.
     if (UNRECORDED.has(key)) return
@@ -517,6 +517,7 @@ export async function startUi ({ port = 7420, onShutdown, preview = false, repor
     s.conn.awareness.on('change', pushVoice)
     s.doc.getMap('voiceSettings').observe(pushVoice)
     s.doc.getArray('voiceLines').observe(pushVoice)
+    s.doc.getMap('voiceAgents').observe(pushVoice)
     // The person's cloud AI, when they connected one, joins the sessions they join.
     followWithCloudAi(entry.run.invite, log)
     if (workspace && mode === 'create') {
@@ -694,6 +695,23 @@ export async function startUi ({ port = 7420, onShutdown, preview = false, repor
     return { line, used: out.used, cap: out.cap }
   }
 
+  // An agent's line in the call we're in, as speech from quilt-api (Fluxions): asked once per line,
+  // kept a little while so the window asking again (a reload) costs nothing.
+  const spoken = new Map() // line id -> Promise<{ audio }>
+  async function voiceSpeech (s, lineId) {
+    const v = s.voice()
+    if (!v.in) throw httpError(409, 'You are not in the call.')
+    const line = s.voiceTranscript({ call: v.call, limit: 1000 }).lines.find((l) => l.id === lineId)
+    if (!line || !line.spoken || !v.agents.some((a) => a.name === line.by)) throw httpError(404, 'That is not an agent\'s line in this call.')
+    if (!spoken.has(line.id)) {
+      const got = asAccount((token) => speakVoice({ token, text: line.text, by: line.by })).then((r) => ({ audio: r.audio, voice: r.voice }))
+      spoken.set(line.id, got)
+      got.catch(() => spoken.delete(line.id))
+      if (spoken.size > 50) spoken.delete(spoken.keys().next().value)
+    }
+    return spoken.get(line.id)
+  }
+
   // Agent invites and the list of your agents come from the accounts API, as this computer's account.
   // A 401 there is checked against /v1/me before it counts: only a token the API no longer knows
   // signs the app out (an older API that doesn't take the app's token for these yet just errors).
@@ -1033,6 +1051,8 @@ export async function startUi ({ port = 7420, onShutdown, preview = false, repor
     'POST /api/sessions/:id/voice/clip': (b, id) => voiceClip(get(id), b),
     'GET /api/sessions/:id/voice/transcript': (b, id, url) => get(id).voiceTranscript({ call: url.searchParams.get('call') || '' }),
     'GET /api/voice/ice': () => voiceServers(),
+    // An agent's line in the call, as speech.
+    'GET /api/sessions/:id/voice/speech': (b, id, url) => voiceSpeech(get(id), url.searchParams.get('line') || ''),
     // The composer says while someone types (and stops when they clear it or send).
     'POST /api/sessions/:id/typing': (b, id) => get(id).setTyping(b.on !== false, { to: b.to || null }),
     'POST /api/sessions/:id/claim': (b, id) => get(id).claim(b.pattern, b.note),