@soffits/dsh-matrix-bot 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/LICENSE +21 -0
- package/README.md +129 -2
- package/bin/dsh-matrix.js +132 -0
- package/cordis.patch.yml +9 -0
- package/icon.svg +6 -0
- package/index.js +4 -0
- package/locale/en.json +6 -0
- package/package.json +68 -4
- package/src/app.js +446 -0
- package/src/commands.js +459 -0
- package/src/config.js +67 -0
- package/src/constants.js +70 -0
- package/src/controls.js +214 -0
- package/src/events.js +251 -0
- package/src/helpers.js +179 -0
- package/src/inbound.js +150 -0
- package/src/matrix.js +507 -0
- package/src/models.js +191 -0
- package/src/operations.js +150 -0
- package/src/outbound.js +141 -0
- package/src/prompts.js +244 -0
- package/src/reactions.js +127 -0
- package/src/sender.js +165 -0
- package/src/sessions.js +390 -0
- package/src/spaces.js +278 -0
- package/src/spool.js +172 -0
- package/src/state.js +203 -0
- package/src/text.js +125 -0
- package/src/titles.js +174 -0
- package/src/turns.js +331 -0
- package/src/voice.js +126 -0
- package/tools.js +60 -0
package/src/turns.js
ADDED
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turn delivery: follow every session bound to a room, show the answer while
|
|
3
|
+
* it is written (one message edited in place), send the final answer when
|
|
4
|
+
* the turn ends, keep a typing notice up while it runs, then clear the
|
|
5
|
+
* working reactions of its messages.
|
|
6
|
+
*/
|
|
7
|
+
import { basename } from 'node:path'
|
|
8
|
+
import { REQUEST_PREFIX, STREAM_EDIT_MS, TYPING_MS } from './constants.js'
|
|
9
|
+
import { clip, localImagePaths, mayBeSilent, messageAnswer, splitText, stripSilent } from './text.js'
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The event id a request id of a Matrix prompt names.
|
|
13
|
+
* @param rpcId - `matrix:<room id>:<event id>`.
|
|
14
|
+
* @returns the event id, or undefined for other request ids.
|
|
15
|
+
*/
|
|
16
|
+
export function eventOfRequest(rpcId) {
|
|
17
|
+
if (typeof rpcId !== 'string' || !rpcId.startsWith(REQUEST_PREFIX)) return undefined
|
|
18
|
+
const start = rpcId.indexOf(':$')
|
|
19
|
+
return start === -1 ? undefined : rpcId.slice(start + 1)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Create the turn tracker.
|
|
24
|
+
* @param deps - `{ ctx, getStore, isRunning, setting, outbound, reactions, roomState, publishStatus, fireAndForget, messageOf, setTyping }`.
|
|
25
|
+
* @returns `{ onSessionEvent, onStream, currentTurn, dispose }`.
|
|
26
|
+
*/
|
|
27
|
+
export function createTurns({ ctx, getStore, isRunning, setting, outbound, reactions, roomState, publishStatus, fireAndForget, messageOf, setTyping }) {
|
|
28
|
+
const { queueOut, notice, sendText, editText, sendFiles, checkFiles, redact } = outbound
|
|
29
|
+
/**
|
|
30
|
+
* Turns of bound sessions by session id: `{ current, turns: Map<turn, record>,
|
|
31
|
+
* pendingOurs, pendingIds, lastOurs }`; `pendingIds` are Matrix messages
|
|
32
|
+
* taken before their turn started, `lastOurs` whether the last turn that
|
|
33
|
+
* ended was started from Matrix.
|
|
34
|
+
*/
|
|
35
|
+
const sessionTurns = new Map()
|
|
36
|
+
|
|
37
|
+
/** @returns the turn bookkeeping of a session. */
|
|
38
|
+
function turnsOf(sessionId) {
|
|
39
|
+
let entry = sessionTurns.get(sessionId)
|
|
40
|
+
if (entry === undefined) {
|
|
41
|
+
entry = { current: undefined, turns: new Map(), pendingOurs: undefined, pendingIds: [], lastOurs: false }
|
|
42
|
+
sessionTurns.set(sessionId, entry)
|
|
43
|
+
}
|
|
44
|
+
return entry
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** @returns the key that ties a turn's messages to it. */
|
|
48
|
+
function turnKey(sessionId, turn) {
|
|
49
|
+
return `${sessionId}\u0000${turn}`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** @returns the record of a session's running turn, or undefined. */
|
|
53
|
+
function currentTurn(sessionId) {
|
|
54
|
+
const entry = sessionTurns.get(sessionId)
|
|
55
|
+
return entry?.current === undefined ? undefined : entry.turns.get(entry.current)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** @returns whether a turn's output goes to its room. */
|
|
59
|
+
function delivered(turn) {
|
|
60
|
+
return turn !== undefined && (turn.ours || turn.goal || turn.mirrored)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
// ------------------------------------------------------------- typing
|
|
64
|
+
|
|
65
|
+
function startTyping(roomId, turn) {
|
|
66
|
+
if (turn.typing !== undefined) return
|
|
67
|
+
const renew = () => { void setTyping(roomId, true, TYPING_MS).catch(() => {}) }
|
|
68
|
+
renew()
|
|
69
|
+
turn.typing = setInterval(renew, TYPING_MS - 5_000)
|
|
70
|
+
turn.typing.unref?.()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function stopTyping(roomId, turn) {
|
|
74
|
+
if (turn.typing === undefined) return
|
|
75
|
+
clearInterval(turn.typing)
|
|
76
|
+
turn.typing = undefined
|
|
77
|
+
void setTyping(roomId, false, 0).catch(() => {})
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// ---------------------------------------------------------- streaming
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Take one frame of an assistant stream: the text of the block being
|
|
84
|
+
* written is shown in the turn's live message.
|
|
85
|
+
* @param sessionId - the streaming session.
|
|
86
|
+
* @param frame - stream frame.
|
|
87
|
+
*/
|
|
88
|
+
function onStream(sessionId, frame) {
|
|
89
|
+
const store = getStore()
|
|
90
|
+
const roomId = store?.roomOfSession(sessionId)
|
|
91
|
+
const turn = currentTurn(sessionId)
|
|
92
|
+
if (roomId === undefined || turn === undefined || !delivered(turn) || setting('streaming', true) !== true) return
|
|
93
|
+
const chunk = frame?.type === 'chunk' ? frame.chunk : undefined
|
|
94
|
+
if (chunk?.type === 'block-start' && chunk.blockType === 'text') {
|
|
95
|
+
turn.stream.text = ''
|
|
96
|
+
return
|
|
97
|
+
}
|
|
98
|
+
if (chunk?.type !== 'text-delta' || typeof chunk.text !== 'string') return
|
|
99
|
+
turn.stream.text += chunk.text
|
|
100
|
+
scheduleFlush(roomId, turn)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function scheduleFlush(roomId, turn) {
|
|
104
|
+
const stream = turn.stream
|
|
105
|
+
if (stream.timer !== undefined || stream.closed) return
|
|
106
|
+
const wait = Math.max(0, stream.lastFlush + STREAM_EDIT_MS - Date.now())
|
|
107
|
+
stream.timer = setTimeout(() => {
|
|
108
|
+
stream.timer = undefined
|
|
109
|
+
void queueOut(roomId, () => flush(roomId, turn))
|
|
110
|
+
}, wait)
|
|
111
|
+
stream.timer.unref?.()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Show the streamed text: the first time as a message, then as edits of it. */
|
|
115
|
+
async function flush(roomId, turn) {
|
|
116
|
+
const stream = turn.stream
|
|
117
|
+
if (stream.closed) return
|
|
118
|
+
const text = stream.text.trim()
|
|
119
|
+
if (mayBeSilent(text) || text === stream.shown) return
|
|
120
|
+
const [visible] = splitText(text)
|
|
121
|
+
stream.lastFlush = Date.now()
|
|
122
|
+
if (stream.eventId === undefined) {
|
|
123
|
+
const [eventId] = await sendText(roomId, visible, { replyTo: replyTarget(roomId, turn) })
|
|
124
|
+
stream.eventId = eventId
|
|
125
|
+
} else {
|
|
126
|
+
await editText(roomId, stream.eventId, visible)
|
|
127
|
+
}
|
|
128
|
+
stream.shown = text
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// ------------------------------------------------------------- events
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* Follow every session bound to a room: collect each turn's answer and send
|
|
135
|
+
* it when the turn ends. Turns started from Matrix are always delivered;
|
|
136
|
+
* turns started elsewhere (Web UI, schedules) when `mirrorTurns` is on.
|
|
137
|
+
*/
|
|
138
|
+
function onSessionEvent(session, event) {
|
|
139
|
+
const store = getStore()
|
|
140
|
+
if (store === undefined || !isRunning()) return
|
|
141
|
+
const sessionId = session?.id
|
|
142
|
+
const roomId = store.roomOfSession(sessionId)
|
|
143
|
+
if (roomId === undefined) return
|
|
144
|
+
const entry = turnsOf(sessionId)
|
|
145
|
+
const mirror = setting('mirrorTurns', true) === true
|
|
146
|
+
if (event.type === 'turn/start') {
|
|
147
|
+
const record = {
|
|
148
|
+
texts: [],
|
|
149
|
+
sentTexts: [],
|
|
150
|
+
ours: entry.pendingOurs !== undefined,
|
|
151
|
+
lastPrompt: entry.pendingOurs,
|
|
152
|
+
goal: false,
|
|
153
|
+
mirrored: mirror,
|
|
154
|
+
sent: false,
|
|
155
|
+
stopped: false,
|
|
156
|
+
typing: undefined,
|
|
157
|
+
stream: { text: '', shown: '', eventId: undefined, timer: undefined, lastFlush: 0, closed: false },
|
|
158
|
+
}
|
|
159
|
+
entry.pendingOurs = undefined
|
|
160
|
+
entry.current = event.data?.turn
|
|
161
|
+
entry.turns.set(event.data?.turn, record)
|
|
162
|
+
for (const eventId of entry.pendingIds) reactions.attach(eventId, turnKey(sessionId, event.data?.turn))
|
|
163
|
+
entry.pendingIds = []
|
|
164
|
+
if (delivered(record)) startTyping(roomId, record)
|
|
165
|
+
return
|
|
166
|
+
}
|
|
167
|
+
const record = entry.current === undefined ? undefined : entry.turns.get(entry.current)
|
|
168
|
+
if (event.type === 'user/message') {
|
|
169
|
+
const source = event.data?.source
|
|
170
|
+
const rpcId = typeof source?.rpcId === 'string' ? source.rpcId : ''
|
|
171
|
+
const eventId = source?.kind === 'user' ? eventOfRequest(rpcId) : undefined
|
|
172
|
+
if (eventId !== undefined) {
|
|
173
|
+
if (record === undefined) {
|
|
174
|
+
entry.pendingOurs = eventId
|
|
175
|
+
entry.pendingIds.push(eventId)
|
|
176
|
+
} else {
|
|
177
|
+
record.ours = true
|
|
178
|
+
record.lastPrompt = eventId
|
|
179
|
+
reactions.attach(eventId, turnKey(sessionId, entry.current))
|
|
180
|
+
startTyping(roomId, record)
|
|
181
|
+
}
|
|
182
|
+
return
|
|
183
|
+
}
|
|
184
|
+
if (rpcId.startsWith('matrix-claim:')) return
|
|
185
|
+
// A helper's report wakes its parent: that turn answers the room the delegating turn came from.
|
|
186
|
+
if ((source?.kind === 'subagent-settled' || source?.kind === 'agent-message') && record !== undefined) {
|
|
187
|
+
if (entry.lastOurs) {
|
|
188
|
+
record.ours = true
|
|
189
|
+
startTyping(roomId, record)
|
|
190
|
+
}
|
|
191
|
+
return
|
|
192
|
+
}
|
|
193
|
+
if (source?.kind === 'goal' && record !== undefined) record.goal = true
|
|
194
|
+
if (!mirror) return
|
|
195
|
+
const label = source?.kind === 'schedule' ? '⏰ Scheduled task' : source?.kind !== 'user' ? undefined : rpcId.startsWith('telegram') ? '✈️ From Telegram' : '🖥 From Web'
|
|
196
|
+
if (label === undefined) return
|
|
197
|
+
const content = Array.isArray(event.data?.content) ? event.data.content : []
|
|
198
|
+
const text = content.filter((block) => block?.type === 'text').map((block) => block.text).join('\n').trim()
|
|
199
|
+
const files = content.filter((block) => block?.type === 'image' || block?.type === 'file').length
|
|
200
|
+
if (text.length === 0 && files === 0) return
|
|
201
|
+
notice(roomId, `${label}:\n${clip(text, 3000)}${files === 0 ? '' : `\n📎 ${files}`}`)
|
|
202
|
+
return
|
|
203
|
+
}
|
|
204
|
+
if (event.type === 'assistant/message') {
|
|
205
|
+
const turn = entry.turns.get(event.data?.turn) ?? record
|
|
206
|
+
const answer = messageAnswer(event.data?.message ?? event.data)
|
|
207
|
+
if (turn !== undefined && answer.length > 0) turn.texts.push(answer)
|
|
208
|
+
return
|
|
209
|
+
}
|
|
210
|
+
if (event.type === 'deliverables/presented') {
|
|
211
|
+
const turn = entry.turns.get(event.data?.turn) ?? record
|
|
212
|
+
if (turn !== undefined && !delivered(turn)) return
|
|
213
|
+
const paths = (Array.isArray(event.data?.files) ? event.data.files : []).map((file) => file?.path).filter((path) => typeof path === 'string')
|
|
214
|
+
if (paths.length === 0) return
|
|
215
|
+
if (turn !== undefined) turn.sent = true
|
|
216
|
+
void queueOut(roomId, () => forwardFiles(roomId, paths))
|
|
217
|
+
return
|
|
218
|
+
}
|
|
219
|
+
if (event.type === 'goal/change') {
|
|
220
|
+
fireAndForget('goal-change', () => announceGoal(roomId, sessionId))
|
|
221
|
+
return
|
|
222
|
+
}
|
|
223
|
+
if (event.type === 'turn/end') {
|
|
224
|
+
const turn = event.data?.turn
|
|
225
|
+
const finished = entry.turns.get(turn)
|
|
226
|
+
if (finished !== undefined) {
|
|
227
|
+
entry.lastOurs = finished.ours
|
|
228
|
+
stopTyping(roomId, finished)
|
|
229
|
+
if (finished.stream.timer !== undefined) clearTimeout(finished.stream.timer)
|
|
230
|
+
finished.stream.timer = undefined
|
|
231
|
+
finished.stream.closed = true
|
|
232
|
+
}
|
|
233
|
+
entry.turns.delete(turn)
|
|
234
|
+
if (entry.current === turn) entry.current = undefined
|
|
235
|
+
// The reactions go once the answer is out, also when the turn failed or was stopped.
|
|
236
|
+
void queueOut(roomId, async () => {
|
|
237
|
+
try {
|
|
238
|
+
if (finished !== undefined) await deliverTurn(roomId, finished, event.data?.reason)
|
|
239
|
+
} finally {
|
|
240
|
+
await reactions.finishTurn(roomId, turnKey(sessionId, turn))
|
|
241
|
+
}
|
|
242
|
+
})
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Send files a session showed its user, with a note for those that cannot go. */
|
|
247
|
+
async function forwardFiles(roomId, paths, quiet = false) {
|
|
248
|
+
const failed = []
|
|
249
|
+
for (const path of paths) {
|
|
250
|
+
try {
|
|
251
|
+
await sendFiles(roomId, await checkFiles([path]))
|
|
252
|
+
} catch (error) {
|
|
253
|
+
if (!quiet) failed.push(`${basename(path)}: ${messageOf(error)}`)
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (failed.length > 0) await sendText(roomId, `⚠️ Could not send:\n${failed.join('\n')}`, { plain: true, notice: true })
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** @returns the prompt to reply to, when the room moved on since it was sent. */
|
|
260
|
+
function replyTarget(roomId, turn) {
|
|
261
|
+
const room = roomState(roomId)
|
|
262
|
+
return turn.ours && typeof turn.lastPrompt === 'string' && room.lastInbound !== turn.lastPrompt ? turn.lastPrompt : undefined
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Send a finished turn's answer into its room, finishing its live message. */
|
|
266
|
+
async function deliverTurn(roomId, turn, reason) {
|
|
267
|
+
const live = turn.stream.eventId
|
|
268
|
+
const dropLive = () => (live === undefined ? undefined : redact(roomId, live).catch(() => {}))
|
|
269
|
+
if (!delivered(turn)) return
|
|
270
|
+
if (turn.stopped || reason?.kind === 'aborted' || reason?.kind === 'blocked') {
|
|
271
|
+
await dropLive()
|
|
272
|
+
return
|
|
273
|
+
}
|
|
274
|
+
// Text the session already sent here with its files is not repeated.
|
|
275
|
+
const sent = new Set(turn.sentTexts.map(sameText))
|
|
276
|
+
const { text, silent } = stripSilent(turn.texts.filter((part) => !sent.has(sameText(part))).join('\n\n'))
|
|
277
|
+
if (reason?.kind === 'error' && text.length === 0) {
|
|
278
|
+
await dropLive()
|
|
279
|
+
await sendText(roomId, `❌ ${reason.error?.message ?? 'The turn failed.'}`, { plain: true, notice: true })
|
|
280
|
+
return
|
|
281
|
+
}
|
|
282
|
+
if (text.length > 0) {
|
|
283
|
+
const [first, ...rest] = splitText(text)
|
|
284
|
+
if (live === undefined) await sendText(roomId, text, { replyTo: replyTarget(roomId, turn) })
|
|
285
|
+
else {
|
|
286
|
+
if (first !== turn.stream.shown) await editText(roomId, live, first)
|
|
287
|
+
for (const part of rest) await sendText(roomId, part)
|
|
288
|
+
}
|
|
289
|
+
const images = localImagePaths(text)
|
|
290
|
+
if (images.length > 0) await forwardFiles(roomId, images, true)
|
|
291
|
+
await publishStatus({ lastReplyAt: Date.now() })
|
|
292
|
+
return
|
|
293
|
+
}
|
|
294
|
+
await dropLive()
|
|
295
|
+
if (turn.ours && !silent && !turn.sent && reason?.kind !== 'error') await sendText(roomId, '✅ Done.', { plain: true, notice: true })
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** @returns text reduced to its words, for comparing an answer with a send. */
|
|
299
|
+
function sameText(text) {
|
|
300
|
+
return String(text).replace(/\s+/gu, ' ').trim()
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/** Tell the room when its session's goal changes phase. */
|
|
304
|
+
async function announceGoal(roomId, sessionId) {
|
|
305
|
+
const goals = ctx.get('goals')
|
|
306
|
+
const agent = ctx.get('agents')?.get(sessionId)
|
|
307
|
+
if (goals === undefined || agent === undefined) return
|
|
308
|
+
const view = goals.get(agent)
|
|
309
|
+
const room = roomState(roomId)
|
|
310
|
+
const phase = view === undefined ? 'none' : view.phase
|
|
311
|
+
if (room.goalPhase === phase) return
|
|
312
|
+
const previous = room.goalPhase
|
|
313
|
+
room.goalPhase = phase
|
|
314
|
+
if (previous === undefined) return
|
|
315
|
+
if (phase === 'complete') await notice(roomId, `🎯 Goal complete: ${clip(view.objective, 300)}`)
|
|
316
|
+
else if (phase === 'blocked') await notice(roomId, `🎯 Goal blocked: ${clip(view.objective, 300)}`)
|
|
317
|
+
else if (phase === 'paused') await notice(roomId, '🎯 Goal paused. !goal resume continues it.')
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Stop timers. */
|
|
321
|
+
function dispose() {
|
|
322
|
+
for (const entry of sessionTurns.values()) {
|
|
323
|
+
for (const turn of entry.turns.values()) {
|
|
324
|
+
if (turn.typing !== undefined) clearInterval(turn.typing)
|
|
325
|
+
if (turn.stream.timer !== undefined) clearTimeout(turn.stream.timer)
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
return { onSessionEvent, onStream, currentTurn, dispose }
|
|
331
|
+
}
|
package/src/voice.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
/** Voice messages and audio files: decoding and transcription. */
|
|
2
|
+
import { spawn } from 'node:child_process'
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Wrap raw 16 kHz mono 16-bit PCM in the canonical 44-byte WAV header that
|
|
6
|
+
* DSH speech providers accept.
|
|
7
|
+
* @param pcm - little-endian samples.
|
|
8
|
+
* @returns WAV bytes.
|
|
9
|
+
*/
|
|
10
|
+
export function pcmToWave(pcm) {
|
|
11
|
+
const data = pcm.length % 2 === 0 ? pcm : pcm.subarray(0, pcm.length - 1)
|
|
12
|
+
const header = Buffer.alloc(44)
|
|
13
|
+
header.write('RIFF', 0, 'ascii')
|
|
14
|
+
header.writeUInt32LE(36 + data.length, 4)
|
|
15
|
+
header.write('WAVE', 8, 'ascii')
|
|
16
|
+
header.write('fmt ', 12, 'ascii')
|
|
17
|
+
header.writeUInt32LE(16, 16)
|
|
18
|
+
header.writeUInt16LE(1, 20)
|
|
19
|
+
header.writeUInt16LE(1, 22)
|
|
20
|
+
header.writeUInt32LE(16000, 24)
|
|
21
|
+
header.writeUInt32LE(32000, 28)
|
|
22
|
+
header.writeUInt16LE(2, 32)
|
|
23
|
+
header.writeUInt16LE(16, 34)
|
|
24
|
+
header.write('data', 36, 'ascii')
|
|
25
|
+
header.writeUInt32LE(data.length, 40)
|
|
26
|
+
return Buffer.concat([header, data])
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Run one program and collect its output.
|
|
31
|
+
* @param argv - program and arguments; no shell is involved.
|
|
32
|
+
* @param signal - cancellation.
|
|
33
|
+
* @param timeoutMs - upper bound for the run.
|
|
34
|
+
* @returns stdout bytes; rejects with stderr on a non-zero exit.
|
|
35
|
+
*/
|
|
36
|
+
export function runProgram(argv, signal, timeoutMs) {
|
|
37
|
+
return new Promise((resolve, reject) => {
|
|
38
|
+
if (signal?.aborted) {
|
|
39
|
+
reject(signal.reason ?? new Error('aborted'))
|
|
40
|
+
return
|
|
41
|
+
}
|
|
42
|
+
const child = spawn(argv[0], argv.slice(1), { stdio: ['ignore', 'pipe', 'pipe'] })
|
|
43
|
+
const out = []
|
|
44
|
+
const err = []
|
|
45
|
+
let settled = false
|
|
46
|
+
const finish = (error, value) => {
|
|
47
|
+
if (settled) return
|
|
48
|
+
settled = true
|
|
49
|
+
clearTimeout(timer)
|
|
50
|
+
signal?.removeEventListener('abort', abort)
|
|
51
|
+
if (error) reject(error)
|
|
52
|
+
else resolve(value)
|
|
53
|
+
}
|
|
54
|
+
const abort = () => {
|
|
55
|
+
child.kill('SIGKILL')
|
|
56
|
+
finish(new Error(`${argv[0]} was cancelled`))
|
|
57
|
+
}
|
|
58
|
+
const timer = setTimeout(() => {
|
|
59
|
+
child.kill('SIGKILL')
|
|
60
|
+
finish(new Error(`${argv[0]} took longer than ${Math.round(timeoutMs / 1000)} s`))
|
|
61
|
+
}, timeoutMs)
|
|
62
|
+
signal?.addEventListener('abort', abort, { once: true })
|
|
63
|
+
child.stdout.on('data', (chunk) => out.push(chunk))
|
|
64
|
+
child.stderr.on('data', (chunk) => err.push(chunk))
|
|
65
|
+
child.on('error', (error) => finish(new Error(`${argv[0]} could not start: ${error.message}`)))
|
|
66
|
+
child.on('close', (code) => {
|
|
67
|
+
if (code === 0) finish(undefined, Buffer.concat(out))
|
|
68
|
+
else finish(new Error(`${argv[0]} exited with ${code}: ${Buffer.concat(err).toString('utf8').trim().slice(-400)}`))
|
|
69
|
+
})
|
|
70
|
+
})
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Transcribe one recording.
|
|
75
|
+
* @param options - `{ mode, path, language, command, ffmpeg, speech, signal, timeoutMs }`:
|
|
76
|
+
* `mode` is `dsh` (the DSH speech-to-text service, `speech`), `command`
|
|
77
|
+
* (`command` argv, `{file}` replaced by `path`, transcript on stdout) or `off`.
|
|
78
|
+
* @returns `{ text, by }`; rejects when transcription fails.
|
|
79
|
+
*/
|
|
80
|
+
export async function transcribe(options) {
|
|
81
|
+
const { mode, path, signal } = options
|
|
82
|
+
const timeoutMs = options.timeoutMs ?? 180_000
|
|
83
|
+
if (mode === 'command') {
|
|
84
|
+
const command = Array.isArray(options.command) ? options.command.filter((part) => typeof part === 'string') : []
|
|
85
|
+
if (command.length === 0) throw new Error('no transcription command is configured')
|
|
86
|
+
const argv = command.some((part) => part.includes('{file}'))
|
|
87
|
+
? command.map((part) => part.replaceAll('{file}', path))
|
|
88
|
+
: [...command, path]
|
|
89
|
+
const text = (await runProgram(argv, signal, timeoutMs)).toString('utf8').trim()
|
|
90
|
+
return { text, by: 'the configured transcription command' }
|
|
91
|
+
}
|
|
92
|
+
if (mode !== 'dsh') throw new Error('transcription is turned off')
|
|
93
|
+
const speech = options.speech
|
|
94
|
+
if (speech === undefined || typeof speech.resolve !== 'function') throw new Error('the DSH speech-to-text service is not enabled')
|
|
95
|
+
const pcm = await runProgram(
|
|
96
|
+
[options.ffmpeg || 'ffmpeg', '-nostdin', '-hide_banner', '-loglevel', 'error', '-i', path, '-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', 'pipe:1'],
|
|
97
|
+
signal,
|
|
98
|
+
timeoutMs,
|
|
99
|
+
)
|
|
100
|
+
if (pcm.length === 0) throw new Error('the recording has no audio')
|
|
101
|
+
const spec = speech.resolve({
|
|
102
|
+
audio: pcmToWave(pcm),
|
|
103
|
+
...(typeof options.language === 'string' && options.language.length > 0 ? { language: options.language } : {}),
|
|
104
|
+
})
|
|
105
|
+
const result = await speech.transcribe(spec, signal ?? new AbortController().signal)
|
|
106
|
+
const provider = spec.provider?.info?.name ?? spec.provider?.info?.id
|
|
107
|
+
return { text: String(result?.text ?? '').trim(), by: `DSH speech-to-text${provider ? ` (${provider})` : ''}` }
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The text the agent reads next to a spoken recording.
|
|
112
|
+
* @param media - `{ kind, name, seconds }`.
|
|
113
|
+
* @param outcome - `{ path, text, by, error }`.
|
|
114
|
+
* @returns prompt text.
|
|
115
|
+
*/
|
|
116
|
+
export function transcriptNote(media, outcome) {
|
|
117
|
+
const length = media.seconds === undefined ? '' : `, ${Math.floor(media.seconds / 60)}:${String(media.seconds % 60).padStart(2, '0')}`
|
|
118
|
+
const where = typeof outcome.path === 'string' ? `the file is attached and saved at ${outcome.path}` : 'the file is attached'
|
|
119
|
+
if (outcome.error !== undefined) {
|
|
120
|
+
return `[Matrix ${media.kind} "${media.name}"${length}: not transcribed (${outcome.error}); ${where}]`
|
|
121
|
+
}
|
|
122
|
+
if (outcome.text.length === 0) {
|
|
123
|
+
return `[Matrix ${media.kind} "${media.name}"${length}: ${outcome.by} heard no speech; ${where}]`
|
|
124
|
+
}
|
|
125
|
+
return `[Matrix ${media.kind} "${media.name}"${length}, transcribed by ${outcome.by}; the transcript may contain mistakes, and ${where} to check it]\n${outcome.text}`
|
|
126
|
+
}
|
package/tools.js
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Model-facing tool for the Matrix bot. A separate plugin entry from the
|
|
3
|
+
* bot (index.js): the bot is looked up as the `matrixRooms` service at call
|
|
4
|
+
* time, so reloading or restarting the bot never unregisters the tool.
|
|
5
|
+
*/
|
|
6
|
+
import z from '@deepseek-ai/schemastery'
|
|
7
|
+
import { SERVICE_NAME } from './src/constants.js'
|
|
8
|
+
|
|
9
|
+
export const name = 'matrix-bot-tools'
|
|
10
|
+
export const inject = ['tools']
|
|
11
|
+
export const Config = z.object({})
|
|
12
|
+
|
|
13
|
+
/** JSON Schema of the send tool's arguments. */
|
|
14
|
+
export const SEND_SCHEMA = {
|
|
15
|
+
type: 'object',
|
|
16
|
+
properties: {
|
|
17
|
+
room: { type: 'string', description: 'Matrix room id (starts with !); each message from a room names it in its sender notice, and !id in the room shows it.' },
|
|
18
|
+
text: { type: 'string', description: 'Message text in Markdown.' },
|
|
19
|
+
files: { type: 'array', items: { type: 'string' }, description: 'Absolute paths of files to send.' },
|
|
20
|
+
reply_to: { type: 'string', description: 'Event id to reply to (or to react to, with reaction).' },
|
|
21
|
+
reaction: { type: 'string', description: 'An emoji to react to reply_to with instead of sending a message.' },
|
|
22
|
+
},
|
|
23
|
+
required: ['room'],
|
|
24
|
+
additionalProperties: false,
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The send tool.
|
|
29
|
+
* @param getService - returns the running bot's service, or undefined.
|
|
30
|
+
* @returns a tool definition.
|
|
31
|
+
*/
|
|
32
|
+
export function createSendTool(getService) {
|
|
33
|
+
return {
|
|
34
|
+
name: 'matrix_send',
|
|
35
|
+
description: 'Send a message, files or a reaction into a Matrix room of the DSH Matrix bot, addressed by its room id. Not for answering in the room this session is bound to: the final answer of the turn is delivered there by itself, so text-only sends into that room are refused. Use it to reach another room, to send files, or to react to a message. The message is also recorded in the session bound to that room; no turn is started.',
|
|
36
|
+
parameters: SEND_SCHEMA,
|
|
37
|
+
output: { schema: { type: 'object' }, render: (_args, value) => [{ type: 'text', text: JSON.stringify(value, null, 2) }] },
|
|
38
|
+
async execute(args, exec) {
|
|
39
|
+
const service = getService()
|
|
40
|
+
if (service === undefined) return { code: 'unavailable', message: 'The Matrix bot is not running.' }
|
|
41
|
+
const sessionId = exec?.agent?.session?.id ?? exec?.agent?.id
|
|
42
|
+
try {
|
|
43
|
+
return await service.send({
|
|
44
|
+
to: args?.room,
|
|
45
|
+
text: typeof args?.text === 'string' ? args.text : '',
|
|
46
|
+
files: Array.isArray(args?.files) ? args.files : [],
|
|
47
|
+
...(args?.reply_to === undefined ? {} : { replyTo: args.reply_to }),
|
|
48
|
+
...(typeof args?.reaction === 'string' ? { reaction: args.reaction } : {}),
|
|
49
|
+
from: typeof sessionId === 'string' ? { sessionId, label: `session ${sessionId}` } : undefined,
|
|
50
|
+
})
|
|
51
|
+
} catch (error) {
|
|
52
|
+
return { code: 'failed', message: error?.message ?? String(error) }
|
|
53
|
+
}
|
|
54
|
+
},
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function apply(ctx) {
|
|
59
|
+
ctx.tools.register(createSendTool(() => ctx.get(SERVICE_NAME)))
|
|
60
|
+
}
|