dsh-realtime-agent 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.d.ts +33 -0
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +40 -0
- package/lib/index.js.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +72 -0
package/lib/index.d.ts
CHANGED
|
@@ -38,6 +38,37 @@ declare module '@deepseek-ai/cordis' {
|
|
|
38
38
|
'realtime-agent/delegation'(request: DelegationRequest): DelegationAnswer | undefined | Promise<DelegationAnswer | undefined>;
|
|
39
39
|
/** A session-scoped failure the adapter contained rather than throwing. */
|
|
40
40
|
'realtime-agent/error'(error: Error): void;
|
|
41
|
+
/**
|
|
42
|
+
* Output audio: PCM16 in the session's declared output format.
|
|
43
|
+
*
|
|
44
|
+
* Emitted once per provider delta, in order, with no buffering. A consumer that cannot keep up is
|
|
45
|
+
* expected to drop frames rather than queue them — a queue that grows while nothing drains it
|
|
46
|
+
* presents first as latency and then as an unbounded allocation.
|
|
47
|
+
*/
|
|
48
|
+
'realtime-agent/audio'(pcm16: Uint8Array): void;
|
|
49
|
+
/**
|
|
50
|
+
* Input audio: capture frames to write to the open session.
|
|
51
|
+
*
|
|
52
|
+
* The host-side half of the microphone path. A capture device belongs in a client half, which
|
|
53
|
+
* cannot reach this context directly, so frames arrive here as an event and the agent keeps the
|
|
54
|
+
* only object that can write them. Emitted with no session open, this is dropped — not buffered.
|
|
55
|
+
*/
|
|
56
|
+
'realtime-agent/mic'(pcm16: Uint8Array): void;
|
|
57
|
+
/**
|
|
58
|
+
* Open the voice session.
|
|
59
|
+
*
|
|
60
|
+
* Emitted by a transport when an authenticated client arrives, so that connecting a microphone is
|
|
61
|
+
* enough to be heard — with no profile option and no dependence on a model choosing to call
|
|
62
|
+
* `voice_start`. The session belongs to the agent, so the transport asks rather than opens one itself.
|
|
63
|
+
*/
|
|
64
|
+
'realtime-agent/start'(): void;
|
|
65
|
+
/**
|
|
66
|
+
* Close the voice session.
|
|
67
|
+
*
|
|
68
|
+
* Emitted by a transport when its last client goes away, including when the transport itself is
|
|
69
|
+
* disposed — a session outliving the microphone that asked for it is a socket nobody is listening to.
|
|
70
|
+
*/
|
|
71
|
+
'realtime-agent/stop'(): void;
|
|
41
72
|
}
|
|
42
73
|
}
|
|
43
74
|
export * from './types.ts';
|
|
@@ -86,6 +117,8 @@ export interface HandlerDeps {
|
|
|
86
117
|
readonly onClosed: () => void;
|
|
87
118
|
/** Where a session-scoped failure is reported. */
|
|
88
119
|
readonly onSessionError: (error: Error) => void;
|
|
120
|
+
/** Where output audio is delivered. Called once per provider delta. */
|
|
121
|
+
readonly onAudio: (pcm16: Uint8Array) => void;
|
|
89
122
|
}
|
|
90
123
|
/**
|
|
91
124
|
* Build the session handlers.
|
package/lib/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,MAAM,MAAM,0BAA0B,CAAA;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAClD,OAAO,KAAK,EAAsB,eAAe,EAAE,uBAAuB,EAAsB,MAAM,cAAc,CAAA;AACpH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AAEpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAE1F,OAAO,QAAQ,qBAAqB,CAAC;IACnC,UAAU,MAAM;QACd;;;;;;WAMG;QACH,2BAA2B,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,GAAG,SAAS,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAA;QAC7H,2EAA2E;QAC3E,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,MAAM,MAAM,0BAA0B,CAAA;AAC7C,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAClD,OAAO,KAAK,EAAsB,eAAe,EAAE,uBAAuB,EAAsB,MAAM,cAAc,CAAA;AACpH,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AAEpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAE1F,OAAO,QAAQ,qBAAqB,CAAC;IACnC,UAAU,MAAM;QACd;;;;;;WAMG;QACH,2BAA2B,CAAC,OAAO,EAAE,iBAAiB,GAAG,gBAAgB,GAAG,SAAS,GAAG,OAAO,CAAC,gBAAgB,GAAG,SAAS,CAAC,CAAA;QAC7H,2EAA2E;QAC3E,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAA;QAC1C;;;;;;WAMG;QACH,sBAAsB,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAA;QAC/C;;;;;;WAMG;QACH,oBAAoB,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAA;QAC7C;;;;;;WAMG;QACH,sBAAsB,IAAI,IAAI,CAAA;QAC9B;;;;;WAKG;QACH,qBAAqB,IAAI,IAAI,CAAA;KAC9B;CACF;AAED,cAAc,YAAY,CAAA;AAC1B,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AACpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAE,KAAK,aAAa,EAAE,MAAM,YAAY,CAAA;AAErE,wDAAwD;AACxD,eAAO,MAAM,IAAI,mBAAmB,CAAA;AAEpC,sFAAsF;AACtF,eAAO,MAAM,MAAM,UAAwB,CAAA;AAE3C;;;;;;GAMG;AACH,eAAO,MAAM,MAAM;;;;;;;;;;;;;;;;aAQjB,CAAA;AAEF,yGAAyG;AACzG,MAAM,WAAW,WAAW;IAC1B,wCAAwC;IACxC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAA;IACrC,mFAAmF;IACnF,QAAQ,CAAC,OAAO,EAAE,MAAM,eAAe,GAAG,SAAS,CAAA;IACnD,gCAAgC;IAChC,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAA;IAC7B,gCAAgC;IAChC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,MAAM,IAAI,CAAA;IAC7B,kDAAkD;IAClD,QAAQ,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;IAC/C,uEAAuE;IACvE,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,CAAA;CAC9C;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,WAAW,GAAG,uBAAuB,CAmBzE;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAgGrE"}
|
package/lib/index.js
CHANGED
|
@@ -69,6 +69,7 @@ export function createHandlers(deps) {
|
|
|
69
69
|
void answerDelegation(session, delegation, deps.transcript.lines(), deps.ask, deps.timeoutMs)
|
|
70
70
|
.catch(deps.onSessionError);
|
|
71
71
|
},
|
|
72
|
+
onAudio: (pcm16) => { deps.onAudio(pcm16); },
|
|
72
73
|
onClosed: () => { deps.onClosed(); },
|
|
73
74
|
onError: (error) => { deps.onSessionError(error); },
|
|
74
75
|
};
|
|
@@ -88,6 +89,7 @@ export function apply(ctx, config) {
|
|
|
88
89
|
timeoutMs: config.delegationTimeoutMs,
|
|
89
90
|
onClosed: () => { session = undefined; },
|
|
90
91
|
onSessionError: (error) => { ctx.emit('realtime-agent/error', error); },
|
|
92
|
+
onAudio: (pcm16) => { ctx.emit('realtime-agent/audio', pcm16); },
|
|
91
93
|
});
|
|
92
94
|
/**
|
|
93
95
|
* Open a session, or return the one already open.
|
|
@@ -128,6 +130,44 @@ export function apply(ctx, config) {
|
|
|
128
130
|
yield () => { for (const dispose of disposers)
|
|
129
131
|
dispose(); };
|
|
130
132
|
}, 'realtime-agent.tools');
|
|
133
|
+
// The microphone seam. Output audio leaves through a handler the adapter already calls; input had no
|
|
134
|
+
// such route, because the object that captures it — a client half — cannot reach this context. So the
|
|
135
|
+
// frames arrive as an event, and the agent keeps the only object that can write them.
|
|
136
|
+
ctx.effect(function* () {
|
|
137
|
+
const dispose = ctx.on('realtime-agent/mic', (pcm16) => {
|
|
138
|
+
const current = session;
|
|
139
|
+
// No session: a capture device that starts before `voice_start` must not throw from inside a
|
|
140
|
+
// listener, and must not buffer either — see the event's own contract.
|
|
141
|
+
if (current === undefined)
|
|
142
|
+
return;
|
|
143
|
+
try {
|
|
144
|
+
current.sendAudio(pcm16);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
// A write into a session that closed a moment ago is a race, not a fault, and the adapter's own
|
|
148
|
+
// close handler decides what it means. Reporting beats throwing from a listener, which is the
|
|
149
|
+
// worst place to raise one.
|
|
150
|
+
ctx.emit('realtime-agent/error', error);
|
|
151
|
+
}
|
|
152
|
+
});
|
|
153
|
+
yield () => { dispose(); };
|
|
154
|
+
}, 'realtime-agent.mic');
|
|
155
|
+
// Session requests from the transport. The route knows when an authenticated client connects and the
|
|
156
|
+
// agent owns the session, so one event is the whole of the wiring between them.
|
|
157
|
+
ctx.effect(function* () {
|
|
158
|
+
const disposers = [
|
|
159
|
+
ctx.on('realtime-agent/start', () => {
|
|
160
|
+
// Fire-and-forget for the same reason `autoStart` is: a listener has no caller to catch a
|
|
161
|
+
// rejection, and the failure is reported on the bus rather than thrown into one.
|
|
162
|
+
void open().catch((error) => { ctx.emit('realtime-agent/error', error); });
|
|
163
|
+
}),
|
|
164
|
+
ctx.on('realtime-agent/stop', () => {
|
|
165
|
+
void stop().catch((error) => { ctx.emit('realtime-agent/error', error); });
|
|
166
|
+
}),
|
|
167
|
+
];
|
|
168
|
+
yield () => { for (const dispose of disposers)
|
|
169
|
+
dispose(); };
|
|
170
|
+
}, 'realtime-agent.session-requests');
|
|
131
171
|
if (config.autoStart) {
|
|
132
172
|
// `apply` is synchronous, so a failed open cannot be thrown from it — it is reported on the bus
|
|
133
173
|
// instead. The plugin stays valid and a later start may succeed.
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,MAAM,MAAM,0BAA0B,CAAA;AAG7C,OAAO,EAAE,gBAAgB,EAAwB,MAAM,aAAa,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,MAAM,MAAM,0BAA0B,CAAA;AAG7C,OAAO,EAAE,gBAAgB,EAAwB,MAAM,aAAa,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AACjD,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAiDlD,cAAc,YAAY,CAAA;AAC1B,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,iBAAiB,EAAwB,MAAM,aAAa,CAAA;AACpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,oBAAoB,EAAsB,MAAM,YAAY,CAAA;AAErE,wDAAwD;AACxD,MAAM,CAAC,MAAM,IAAI,GAAG,gBAAgB,CAAA;AAEpC,sFAAsF;AACtF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;AAE3C;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,WAAW,CAAC,gDAAgD,CAAC;IAC9G,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,wBAAwB,CAAC;IAClF,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,mDAAmD,CAAC;IACvG,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,2CAA2C,CAAC;IACtG,SAAS,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,6CAA6C,CAAC;IACrG,mBAAmB,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,kCAAkC,CAAC;IACpG,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,gDAAgD,CAAC;CACjH,CAAC,CAAA;AAoBF;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAiB;IAC9C,OAAO;QACL,YAAY,EAAE,CAAC,QAA4B,EAAQ,EAAE;YACnD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAA;QACtE,CAAC;QACD,YAAY,EAAE,CAAC,UAA8B,EAAQ,EAAE;YACrD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAA;YAC9B,gGAAgG;YAChG,0FAA0F;YAC1F,IAAI,OAAO,KAAK,SAAS;gBAAE,OAAM;YACjC,4FAA4F;YAC5F,qFAAqF;YACrF,KAAK,gBAAgB,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC;iBAC1F,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;QAC/B,CAAC;QACD,OAAO,EAAE,CAAC,KAAiB,EAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA,CAAC,CAAC;QAC7D,QAAQ,EAAE,GAAS,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA,CAAC,CAAC;QACzC,OAAO,EAAE,CAAC,KAAY,EAAQ,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAA,CAAC,CAAC;KAChE,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,KAAK,CAAC,GAAY,EAAE,MAA2B;IAC7D,MAAM,UAAU,GAAG,IAAI,gBAAgB,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAClE,IAAI,OAAoC,CAAA;IAExC,MAAM,QAAQ,GAAG,cAAc,CAAC;QAC9B,UAAU;QACV,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO;QACtB,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,2BAA2B,EAAE,OAAO,CAAC;QAChE,SAAS,EAAE,MAAM,CAAC,mBAAmB;QACrC,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,GAAG,SAAS,CAAA,CAAC,CAAC;QACvC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAA,CAAC,CAAC;QACtE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAA,CAAC,CAAC;KAChE,CAAC,CAAA;IAEF;;;;;OAKG;IACH,MAAM,IAAI,GAAG,KAAK,IAA8B,EAAE;QAChD,MAAM,OAAO,GAAG,OAAO,CAAA;QACvB,IAAI,OAAO,KAAK,SAAS;YAAE,OAAO,OAAO,CAAA;QACzC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;YACxC,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,GAAG,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE;YAC5D,GAAG,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,YAAY,EAAE;YACjF,QAAQ;SACT,CAAC,CAAA;QACF,OAAO,GAAG,MAAM,CAAA;QAChB,OAAO,MAAM,CAAA;IACf,CAAC,CAAA;IAED;;;;;OAKG;IACH,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACrC,MAAM,OAAO,GAAG,OAAO,CAAA;QACvB,OAAO,GAAG,SAAS,CAAA;QACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAA;IACxB,CAAC,CAAA;IAED,oGAAoG;IACpG,kEAAkE;IAClE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAClB,MAAM,SAAS,GAAG,oBAAoB,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;aAClF,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAA;QACpD,MAAM,GAAG,EAAE,GAAG,KAAK,MAAM,OAAO,IAAI,SAAS;YAAE,OAAO,EAAE,CAAA,CAAC,CAAC,CAAA;IAC5D,CAAC,EAAE,sBAAsB,CAAC,CAAA;IAE1B,qGAAqG;IACrG,sGAAsG;IACtG,sFAAsF;IACtF,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAClB,MAAM,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,KAAiB,EAAE,EAAE;YACjE,MAAM,OAAO,GAAG,OAAO,CAAA;YACvB,6FAA6F;YAC7F,uEAAuE;YACvE,IAAI,OAAO,KAAK,SAAS;gBAAE,OAAM;YACjC,IAAI,CAAC;gBACH,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAC1B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,gGAAgG;gBAChG,8FAA8F;gBAC9F,4BAA4B;gBAC5B,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAc,CAAC,CAAA;YAClD,CAAC;QACH,CAAC,CAAC,CAAA;QACF,MAAM,GAAG,EAAE,GAAG,OAAO,EAAE,CAAA,CAAC,CAAC,CAAA;IAC3B,CAAC,EAAE,oBAAoB,CAAC,CAAA;IAExB,qGAAqG;IACrG,gFAAgF;IAChF,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;QAClB,MAAM,SAAS,GAAG;YAChB,GAAG,CAAC,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;gBAClC,0FAA0F;gBAC1F,iFAAiF;gBACjF,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;YAClF,CAAC,CAAC;YACF,GAAG,CAAC,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;gBACjC,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;YAClF,CAAC,CAAC;SACH,CAAA;QACD,MAAM,GAAG,EAAE,GAAG,KAAK,MAAM,OAAO,IAAI,SAAS;YAAE,OAAO,EAAE,CAAA,CAAC,CAAC,CAAA;IAC5D,CAAC,EAAE,iCAAiC,CAAC,CAAA;IAErC,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,gGAAgG;QAChG,iEAAiE;QACjE,KAAK,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAA,CAAC,CAAC,CAAC,CAAA;IAClF,CAAC;AACH,CAAC"}
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -41,6 +41,37 @@ declare module '@deepseek-ai/cordis' {
|
|
|
41
41
|
'realtime-agent/delegation'(request: DelegationRequest): DelegationAnswer | undefined | Promise<DelegationAnswer | undefined>
|
|
42
42
|
/** A session-scoped failure the adapter contained rather than throwing. */
|
|
43
43
|
'realtime-agent/error'(error: Error): void
|
|
44
|
+
/**
|
|
45
|
+
* Output audio: PCM16 in the session's declared output format.
|
|
46
|
+
*
|
|
47
|
+
* Emitted once per provider delta, in order, with no buffering. A consumer that cannot keep up is
|
|
48
|
+
* expected to drop frames rather than queue them — a queue that grows while nothing drains it
|
|
49
|
+
* presents first as latency and then as an unbounded allocation.
|
|
50
|
+
*/
|
|
51
|
+
'realtime-agent/audio'(pcm16: Uint8Array): void
|
|
52
|
+
/**
|
|
53
|
+
* Input audio: capture frames to write to the open session.
|
|
54
|
+
*
|
|
55
|
+
* The host-side half of the microphone path. A capture device belongs in a client half, which
|
|
56
|
+
* cannot reach this context directly, so frames arrive here as an event and the agent keeps the
|
|
57
|
+
* only object that can write them. Emitted with no session open, this is dropped — not buffered.
|
|
58
|
+
*/
|
|
59
|
+
'realtime-agent/mic'(pcm16: Uint8Array): void
|
|
60
|
+
/**
|
|
61
|
+
* Open the voice session.
|
|
62
|
+
*
|
|
63
|
+
* Emitted by a transport when an authenticated client arrives, so that connecting a microphone is
|
|
64
|
+
* enough to be heard — with no profile option and no dependence on a model choosing to call
|
|
65
|
+
* `voice_start`. The session belongs to the agent, so the transport asks rather than opens one itself.
|
|
66
|
+
*/
|
|
67
|
+
'realtime-agent/start'(): void
|
|
68
|
+
/**
|
|
69
|
+
* Close the voice session.
|
|
70
|
+
*
|
|
71
|
+
* Emitted by a transport when its last client goes away, including when the transport itself is
|
|
72
|
+
* disposed — a session outliving the microphone that asked for it is a socket nobody is listening to.
|
|
73
|
+
*/
|
|
74
|
+
'realtime-agent/stop'(): void
|
|
44
75
|
}
|
|
45
76
|
}
|
|
46
77
|
|
|
@@ -86,6 +117,8 @@ export interface HandlerDeps {
|
|
|
86
117
|
readonly onClosed: () => void
|
|
87
118
|
/** Where a session-scoped failure is reported. */
|
|
88
119
|
readonly onSessionError: (error: Error) => void
|
|
120
|
+
/** Where output audio is delivered. Called once per provider delta. */
|
|
121
|
+
readonly onAudio: (pcm16: Uint8Array) => void
|
|
89
122
|
}
|
|
90
123
|
|
|
91
124
|
/**
|
|
@@ -108,6 +141,7 @@ export function createHandlers(deps: HandlerDeps): RealtimeSessionHandlers {
|
|
|
108
141
|
void answerDelegation(session, delegation, deps.transcript.lines(), deps.ask, deps.timeoutMs)
|
|
109
142
|
.catch(deps.onSessionError)
|
|
110
143
|
},
|
|
144
|
+
onAudio: (pcm16: Uint8Array): void => { deps.onAudio(pcm16) },
|
|
111
145
|
onClosed: (): void => { deps.onClosed() },
|
|
112
146
|
onError: (error: Error): void => { deps.onSessionError(error) },
|
|
113
147
|
}
|
|
@@ -129,6 +163,7 @@ export function apply(ctx: Context, config: RealtimeAgentConfig): void {
|
|
|
129
163
|
timeoutMs: config.delegationTimeoutMs,
|
|
130
164
|
onClosed: () => { session = undefined },
|
|
131
165
|
onSessionError: (error) => { ctx.emit('realtime-agent/error', error) },
|
|
166
|
+
onAudio: (pcm16) => { ctx.emit('realtime-agent/audio', pcm16) },
|
|
132
167
|
})
|
|
133
168
|
|
|
134
169
|
/**
|
|
@@ -171,6 +206,43 @@ export function apply(ctx: Context, config: RealtimeAgentConfig): void {
|
|
|
171
206
|
yield () => { for (const dispose of disposers) dispose() }
|
|
172
207
|
}, 'realtime-agent.tools')
|
|
173
208
|
|
|
209
|
+
// The microphone seam. Output audio leaves through a handler the adapter already calls; input had no
|
|
210
|
+
// such route, because the object that captures it — a client half — cannot reach this context. So the
|
|
211
|
+
// frames arrive as an event, and the agent keeps the only object that can write them.
|
|
212
|
+
ctx.effect(function* () {
|
|
213
|
+
const dispose = ctx.on('realtime-agent/mic', (pcm16: Uint8Array) => {
|
|
214
|
+
const current = session
|
|
215
|
+
// No session: a capture device that starts before `voice_start` must not throw from inside a
|
|
216
|
+
// listener, and must not buffer either — see the event's own contract.
|
|
217
|
+
if (current === undefined) return
|
|
218
|
+
try {
|
|
219
|
+
current.sendAudio(pcm16)
|
|
220
|
+
} catch (error) {
|
|
221
|
+
// A write into a session that closed a moment ago is a race, not a fault, and the adapter's own
|
|
222
|
+
// close handler decides what it means. Reporting beats throwing from a listener, which is the
|
|
223
|
+
// worst place to raise one.
|
|
224
|
+
ctx.emit('realtime-agent/error', error as Error)
|
|
225
|
+
}
|
|
226
|
+
})
|
|
227
|
+
yield () => { dispose() }
|
|
228
|
+
}, 'realtime-agent.mic')
|
|
229
|
+
|
|
230
|
+
// Session requests from the transport. The route knows when an authenticated client connects and the
|
|
231
|
+
// agent owns the session, so one event is the whole of the wiring between them.
|
|
232
|
+
ctx.effect(function* () {
|
|
233
|
+
const disposers = [
|
|
234
|
+
ctx.on('realtime-agent/start', () => {
|
|
235
|
+
// Fire-and-forget for the same reason `autoStart` is: a listener has no caller to catch a
|
|
236
|
+
// rejection, and the failure is reported on the bus rather than thrown into one.
|
|
237
|
+
void open().catch((error: Error) => { ctx.emit('realtime-agent/error', error) })
|
|
238
|
+
}),
|
|
239
|
+
ctx.on('realtime-agent/stop', () => {
|
|
240
|
+
void stop().catch((error: Error) => { ctx.emit('realtime-agent/error', error) })
|
|
241
|
+
}),
|
|
242
|
+
]
|
|
243
|
+
yield () => { for (const dispose of disposers) dispose() }
|
|
244
|
+
}, 'realtime-agent.session-requests')
|
|
245
|
+
|
|
174
246
|
if (config.autoStart) {
|
|
175
247
|
// `apply` is synchronous, so a failed open cannot be thrown from it — it is reported on the bus
|
|
176
248
|
// instead. The plugin stays valid and a later start may succeed.
|