heyquilt 0.3.141 → 0.3.143

package/src/voice.js

@@ -3,6 +3,8 @@
 // The audio (and a shared screen) goes from app window to app window, never through the relay or this
 // process. What people say, when the owner turned transcripts on and they agreed, is kept as
 // lines in the session's document for everyone, agents included, to read.
+// Agents join a call too: someone in it adds them (or one joins when asked), they hear it through
+// those lines, and what they say back is a line each app in the call speaks aloud (Fluxions, on quilt-api).
 import crypto from 'node:crypto'
 
 // One offer or answer with its gathered addresses is a few kilobytes; anything far bigger isn't one.
@@ -75,12 +77,15 @@ export function setTranscribe (doc, { on, by, now = Date.now() }) {
   doc.getMap('voiceSettings').set('transcribe', { on: !!on, by, ts: now })
 }
 
-/** One person's line of speech in a call's transcript. */
-export function addLine (doc, { call, by, text, now = Date.now() }) {
+/**
+ * One person's line of speech in a call's transcript, or an agent's (`spoken`: each app in the
+ * call plays it aloud in the agent's voice).
+ */
+export function addLine (doc, { call, by, text, spoken = false, now = Date.now() }) {
   const t = String(text || '').replace(/\s+/g, ' ').trim().slice(0, MAX_LINE)
   if (!t) return null
   if (typeof call !== 'string' || !CALL.test(call)) throw new Error('That is not a call.')
-  const line = { id: crypto.randomBytes(8).toString('hex'), call, by, text: t, ts: now }
+  const line = { id: crypto.randomBytes(8).toString('hex'), call, by, text: t, ts: now, ...(spoken ? { spoken: true } : {}) }
   const lines = doc.getArray('voiceLines')
   doc.transact(() => {
     lines.push([line])
@@ -110,6 +115,112 @@ export function readLines (doc, { call = '', since = 0, limit = 200 } = {}) {
   return { lines: picked.slice(-n), more: picked.length > n, calls: [...calls.values()].sort((a, b) => a.start - b.start) }
 }
 
+// ------------------------------------------------------------ agents in a call --
+// Every agent in the session is in a transcribed call, like a person, but muted: saying its name
+// unmutes it. It hears the call through the transcript, and while unmuted what is said wakes it
+// and it answers out loud; it mutes again once the conversation with it has gone quiet.
+
+// After an agent speaks, or is named, it stays unmuted this long: a conversation going on.
+export const AGENT_FOLLOW_MS = 90 * 1000
+// Lines of the call before one that wakes an agent, shown with it.
+export const AGENT_CONTEXT = 8
+export const AGENT_NEEDS_TRANSCRIPTS = 'Agents hear a call through its transcript, and this one is not transcribed: the session owner turns transcripts on.'
+
+/** Whether agent `name` left call `call` (quilt_voice_join leave). */
+const leftCall = (doc, call, name) => {
+  const v = doc.getMap('voiceAgents').get(name)
+  return !!(v && typeof v === 'object' && v.call === call && v.left === true)
+}
+
+/** Lines of call `call`, oldest first. */
+const callLines = (doc, call) => doc.getArray('voiceLines').toArray().filter((l) => l && l.call === call && typeof l.text === 'string')
+
+/** Until when an agent (`names`) is unmuted, from a call's lines (0: muted): AGENT_FOLLOW_MS after the latest line it said or that named it. */
+export function agentOpenUntil (lines, names) {
+  let until = 0
+  for (const l of lines) {
+    if (names.includes(l.by) || names.some((n) => saysName(l.text, n))) until = Math.max(until, (l.ts || 0) + AGENT_FOLLOW_MS)
+  }
+  return until
+}
+
+/**
+ * The agents in call `call` (the one going on): every agent member of the session while the call
+ * is transcribed, but those that left it. Each with `until`: unmuted until then (0: muted).
+ */
+export function callAgents (doc, call, members = []) {
+  if (!call || !transcribeSetting(doc, members).on) return []
+  const lines = callLines(doc, call)
+  return (members || []).filter((m) => m && m.kind === 'agent' && m.name && !leftCall(doc, call, m.name))
+    .map((m) => ({ name: m.name, until: agentOpenUntil(lines, [m.name]) }))
+    .sort((a, b) => a.name.localeCompare(b.name))
+}
+
+/** Agent `name` leaves call `call` (`leave`), or comes back to it. */
+export function setAgentLeft (doc, { call, name, leave = true, now = Date.now() }) {
+  if (leave) doc.getMap('voiceAgents').set(name, { call, left: true, ts: now })
+  else if (doc.getMap('voiceAgents').has(name)) doc.getMap('voiceAgents').delete(name)
+}
+
+/** Why agent `name` isn't in call `call` now, or '' when it is. */
+export function notInCall (doc, { call, name, members = [] }) {
+  if (!call) return 'There is no call going on in this session.'
+  if (!transcribeSetting(doc, members).on) return AGENT_NEEDS_TRANSCRIPTS
+  if (!(members || []).some((m) => m && m.kind === 'agent' && m.name === name)) return 'Only the session\'s agents are in its calls.'
+  if (leftCall(doc, call, name)) return 'You left this call: come back with quilt_voice_join.'
+  return ''
+}
+
+// A name as spoken words, loosely: "Daniel's AI" is "daniels ai".
+const words = (s) => String(s || '').toLowerCase().replace(/['’]/g, '').replace(/[^\p{L}\p{N}]+/gu, ' ').trim()
+
+/** Whether a line said out loud names `name` (its words in a row, or run together, any case or punctuation): "Moss, can you…", "GrokBot" for Grok-Bot. */
+export function saysName (text, name) {
+  const n = words(name)
+  if (!n) return false
+  const t = ` ${words(text)} `
+  return t.includes(` ${n} `) || t.includes(` ${n.replace(/ /g, '')} `)
+}
+
+/**
+ * What reaches agent `names[0]` from call `call` (the one going on), as inbox entries (inbox.js
+ * scanInbox `voice`): each line by someone else that names it, or comes while it is unmuted
+ * (AGENT_FOLLOW_MS after it spoke or was named). Each line carries the lines just before it.
+ */
+export function callEntries (doc, { names, call, members = [] }) {
+  const me = names && names[0]
+  if (!me || notInCall(doc, { call, name: me, members })) return []
+  const lines = callLines(doc, call)
+  const out = []
+  let open = 0 // until when what is said is for us
+  lines.forEach((l, i) => {
+    if (names.includes(l.by)) { open = l.ts + AGENT_FOLLOW_MS; return }
+    const named = names.some((n) => saysName(l.text, n))
+    if (named) open = l.ts + AGENT_FOLLOW_MS
+    else if (l.ts > open) return
+    out.push({ id: `v-${l.id}`, kind: 'voice', by: l.by, text: l.text, ts: l.ts, call, ...(named ? { named: true } : {}), said: lines.slice(Math.max(0, i - AGENT_CONTEXT), i).map((x) => ({ by: x.by, text: x.text, ts: x.ts })) })
+  })
+  return out
+}
+
+/** The call going on in a session (someone is in it), from presence: its id, or ''. */
+export function liveCallOf (awareness) {
+  for (const s of awareness.getStates().values()) {
+    const v = s && s.name ? cleanVoice(s.voice) : null
+    if (v && v.call) return v.call
+  }
+  return ''
+}
+
+/** Agent `name` says `text` out loud in call `call` (the one going on): its line, which every app in the call speaks. */
+export function agentSays (doc, { call, name, text, members = [] }) {
+  const why = notInCall(doc, { call, name, members })
+  if (why) throw new Error(why)
+  const line = addLine(doc, { call, by: name, text, spoken: true })
+  if (!line) throw new Error('Say something.')
+  return line
+}
+
 /** Transcript lines as plain text for an agent: one per line, with who and when. */
 export function renderLines ({ lines, more, calls }, { tz } = {}) {
   if (!lines.length) return calls.length ? 'No transcript lines match.' : 'Nothing has been transcribed in this session yet.'
@@ -150,6 +261,8 @@ export function voiceView ({ awareness, clientID, members = [], doc }) {
   // When the call began: its first person in. The chat shows it, and the calls before it.
   const starts = [...(me ? [me.since] : []), ...peers.map((p) => p.since)].filter((x) => x > 0)
   const said = doc ? readLines(doc, { limit: 1 }) : { calls: [] }
+  // A call is going on while anyone is in it; its agents leave with the last person.
+  const going = !!(me || peers.length)
   return {
     me: clientID,
     in: !!me,
@@ -163,7 +276,9 @@ export function voiceView ({ awareness, clientID, members = [], doc }) {
     // The latest lines of the call going on now, as each is written down.
     recent: doc && call ? readLines(doc, { call, limit: RECENT }).lines : [],
     // Every call that has a transcript: when, who spoke, how many lines.
-    calls: said.calls
+    calls: said.calls,
+    // The agents in the call going on now, and the session's agents who could join it.
+    agents: doc && going ? callAgents(doc, call, members) : []
   }
 }
 
@@ -177,15 +292,24 @@ export function inVoice (awareness, clientID) {
   return out
 }
 
+export const JOIN_DESCRIPTION = 'Leave the session\'s voice call, or come back to it. While the owner has transcripts on, you are in every call, like a person but muted: ' +
+  'when someone says your name you are unmuted, woken with what they said, and answer out loud with quilt_voice_say; you mute again a minute and a half after the conversation with you goes quiet. ' +
+  'leave: true to stop hearing the call until it ends.'
+
+export const SAY_DESCRIPTION = 'Say something out loud in the voice call: everyone in it hears it in your voice, and it goes in the transcript. ' +
+  'Talk the way people do in a call: one to three short sentences, no lists, no code, no markdown, no links (put those in chat with quilt_message). ' +
+  'Answer what was said to you; stay quiet when a line was not for you. [laugh], [sigh] and [hesitate] are spoken as sounds.'
+
 export const TRANSCRIPT_DESCRIPTION = 'Read what people said in the session\'s voice calls: the transcript lines, oldest first, with who said what and when. ' +
   'People talk in calls in the Quilt app; while the session owner has transcripts on, each person who agreed has their own speech written here. ' +
   'quilt_status says who is in a call now. Read the latest call before acting on something decided out loud, and ask in chat when a line is unclear.'
 
-/** Who is in a call, for an agent's status: a heading and a line each, or nothing when nobody is. */
-export function voiceStatusLines (people, transcribe) {
+/** Who is in a call, for an agent's status: a heading and a line each, or nothing when nobody is. `agents`: the agents in it ({ name }). */
+export function voiceStatusLines (people, transcribe, agents = [], me = '') {
   if (!people.length) return []
   return [
     `## In a voice call${transcribe ? ' (transcribed: quilt_voice_transcript reads it)' : ' (not transcribed: ask in chat what was decided)'}`,
-    ...people.map((p) => `- ${p.name}${p.muted ? ' (muted)' : ''}${p.sharing ? ' (sharing their screen)' : ''}${transcribe && !p.transcribing ? ' (not transcribed)' : ''}`)
+    ...people.map((p) => `- ${p.name}${p.muted ? ' (muted)' : ''}${p.sharing ? ' (sharing their screen)' : ''}${transcribe && !p.transcribing ? ' (not transcribed)' : ''}`),
+    ...agents.map((a) => `- ${a.name === me ? 'You' : a.name} (an agent, ${a.until > Date.now() ? 'unmuted' : 'muted until someone says its name'}${a.name === me ? '; answer out loud with quilt_voice_say' : ''})`)
   ]
 }

package/src/webhooks.js

@@ -16,8 +16,12 @@ import crypto from 'node:crypto'
 import dns from 'node:dns'
 import net from 'node:net'
 
-export const WEBHOOK_EVENTS = ['chat.mention', 'chat.dm', 'task.assigned', 'email.received']
-export const EVENT_OF_KIND = { mention: 'chat.mention', dm: 'chat.dm', task: 'task.assigned', email: 'email.received' }
+export const WEBHOOK_EVENTS = ['chat.mention', 'chat.dm', 'task.assigned', 'email.received', 'voice.said']
+export const EVENT_OF_KIND = { mention: 'chat.mention', dm: 'chat.dm', task: 'task.assigned', email: 'email.received', voice: 'voice.said' }
+
+/** Whether a subscription takes `event`. One made before voice.said that took every event then takes it too. */
+export const takesEvent = (events, event) => events.includes(event) ||
+  (event === 'voice.said' && ['chat.mention', 'chat.dm', 'task.assigned', 'email.received'].every((e) => events.includes(e)))
 export const MAX_URL = 2000
 export const MAX_SECRET = 200
 export const MIN_SECRET = 16
@@ -141,7 +145,7 @@ export function describeSubscription (sub, { showSecret = false } = {}) {
   if (!sub) return 'No webhook: Quilt only answers quilt_inbox when you ask.'
   const lines = [`Webhook: Quilt POSTs to ${sub.url} on ${sub.events.join(', ')}${sub.bearer ? ', with your bearer key in the Authorization header' : ''}${sub.coalesce ? ', a burst in one POST (its events in `events`)' : ''}${sub.local ? ' (sent from this computer: the relay cannot reach that URL)' : ''}.`]
   if (showSecret) lines.push(`Secret (shown once; check x-quilt-signature with it): ${sub.secret}`)
-  lines.push('Each POST is JSON ({ event, id, session, workspace, room, to, by, text, ts, task?, mail?, context? }; session: the session it is from, which you name as `session` in the tools you answer with, so an answer never lands in another session; workspace: { id } or null; mail: { id } of an email to the session, read with quilt_mail; context: the messages before this one between you and its sender, oldest first) with x-quilt-event, x-quilt-delivery, x-quilt-timestamp and ' +
+  lines.push('Each POST is JSON ({ event, id, session, workspace, room, to, by, text, ts, task?, mail?, call?, context? }; session: the session it is from, which you name as `session` in the tools you answer with, so an answer never lands in another session; workspace: { id } or null; mail: { id } of an email to the session, read with quilt_mail; call: { id, named?, before? } for voice.said, a line said to you in the session voice call (named: it says your name), answered out loud with quilt_voice_say; context: the messages before this one between you and its sender, oldest first) with x-quilt-event, x-quilt-delivery, x-quilt-timestamp and ' +
     'x-quilt-signature: sha256=HMAC-SHA256(secret, "<timestamp>.<body>"). Answer 2xx quickly; a failed POST is retried a few times. ' +
     'What you are told is still in quilt_inbox. quilt_webhook_test POSTs a webhook.test event now; stop with quilt_webhook_unsubscribe.')
   return lines.join('\n')
@@ -157,6 +161,11 @@ export function webhookPayload (e, { room = '', to = '', workspace = '' } = {})
   if (e.kind === 'email' && e.mail) p.mail = { id: String(e.mail) } // read it with quilt_mail (read: id)
   if (e.commit) p.commit = true // work this member asked to commit was committed: nothing to answer
   if (e.nudge) p.nudge = true // a reminder from Quilt (uncommitted work): nothing to answer
+  // Said to the agent in the session's voice call; answered with quilt_voice_say.
+  if (e.kind === 'voice') {
+    p.call = { id: String(e.call || ''), ...(e.named ? { named: true } : {}) }
+    if (Array.isArray(e.said) && e.said.length) p.call.before = e.said.map((l) => ({ by: l.by, text: l.text, ts: l.ts }))
+  }
   // The conversation before this with whoever it is from (conversation.js), so an agent woken
   // fresh knows what "that" refers to; `earlier` older messages are a quilt_conversation away.
   if (Array.isArray(e.context) && e.context.length) {