@moureau/pipeflow 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -17
- package/dist/cjs/client/client.js +2 -0
- package/dist/cjs/client/client.js.map +7 -0
- package/dist/cjs/client/index.js +2 -0
- package/dist/cjs/client/index.js.map +7 -0
- package/dist/cjs/client/json.js +2 -0
- package/dist/cjs/client/json.js.map +7 -0
- package/dist/cjs/client/protocol/websocket.js +2 -0
- package/dist/cjs/client/protocol/websocket.js.map +7 -0
- package/dist/cjs/client/protocol.js +2 -0
- package/dist/cjs/client/protocol.js.map +7 -0
- package/dist/cjs/conversations/conversation/conversation.js +1 -1
- package/dist/cjs/conversations/conversation/conversation.js.map +3 -3
- package/dist/cjs/conversations/conversations.js +1 -1
- package/dist/cjs/conversations/conversations.js.map +2 -2
- package/dist/cjs/conversations/orchestration/orchestrator/generation/generation.js +1 -1
- package/dist/cjs/conversations/orchestration/orchestrator/generation/generation.js.map +3 -3
- package/dist/cjs/conversations/orchestration/orchestrator/orchestrator.js +1 -1
- package/dist/cjs/conversations/orchestration/orchestrator/orchestrator.js.map +3 -3
- package/dist/cjs/conversations/orchestration/orchestrator/speech/speech.js +1 -1
- package/dist/cjs/conversations/orchestration/orchestrator/speech/speech.js.map +3 -3
- package/dist/cjs/conversations/orchestration/orchestrator/tools/tools.js +1 -1
- package/dist/cjs/conversations/orchestration/orchestrator/tools/tools.js.map +3 -3
- package/dist/cjs/conversations/orchestration/test-harness.js +1 -1
- package/dist/cjs/conversations/orchestration/test-harness.js.map +2 -2
- package/dist/cjs/pipeflow/pipeflow.js +1 -1
- package/dist/cjs/pipeflow/pipeflow.js.map +2 -2
- package/dist/cjs/providers/llm/adapters/openai-compatible.js +6 -6
- package/dist/cjs/providers/llm/adapters/openai-compatible.js.map +3 -3
- package/dist/cjs/providers/llm/types.js.map +2 -2
- package/dist/cjs/providers/stt/adapters/openrouter/openrouter.js +1 -1
- package/dist/cjs/providers/stt/adapters/openrouter/openrouter.js.map +3 -3
- package/dist/cjs/providers/tts/adapters/kokoro/kokoro.js +3 -3
- package/dist/cjs/providers/tts/adapters/kokoro/kokoro.js.map +3 -3
- package/dist/cjs/providers/tts/adapters/openrouter/openrouter.js +1 -1
- package/dist/cjs/providers/tts/adapters/openrouter/openrouter.js.map +3 -3
- package/dist/esm/client/client.js +2 -0
- package/dist/esm/client/client.js.map +7 -0
- package/dist/esm/client/index.js +2 -0
- package/dist/esm/client/index.js.map +7 -0
- package/dist/esm/client/json.js +2 -0
- package/dist/esm/client/json.js.map +7 -0
- package/dist/esm/client/protocol/websocket.js +2 -0
- package/dist/esm/client/protocol/websocket.js.map +7 -0
- package/dist/esm/client/protocol.js +1 -0
- package/dist/esm/client/protocol.js.map +7 -0
- package/dist/esm/conversations/conversation/conversation.js +1 -1
- package/dist/esm/conversations/conversation/conversation.js.map +3 -3
- package/dist/esm/conversations/conversations.js +1 -1
- package/dist/esm/conversations/conversations.js.map +2 -2
- package/dist/esm/conversations/orchestration/orchestrator/generation/generation.js +1 -1
- package/dist/esm/conversations/orchestration/orchestrator/generation/generation.js.map +3 -3
- package/dist/esm/conversations/orchestration/orchestrator/orchestrator.js +1 -1
- package/dist/esm/conversations/orchestration/orchestrator/orchestrator.js.map +3 -3
- package/dist/esm/conversations/orchestration/orchestrator/speech/speech.js +1 -1
- package/dist/esm/conversations/orchestration/orchestrator/speech/speech.js.map +3 -3
- package/dist/esm/conversations/orchestration/orchestrator/tools/tools.js +1 -1
- package/dist/esm/conversations/orchestration/orchestrator/tools/tools.js.map +3 -3
- package/dist/esm/conversations/orchestration/test-harness.js +1 -1
- package/dist/esm/conversations/orchestration/test-harness.js.map +2 -2
- package/dist/esm/pipeflow/pipeflow.js +1 -1
- package/dist/esm/pipeflow/pipeflow.js.map +2 -2
- package/dist/esm/providers/llm/adapters/openai-compatible.js +6 -6
- package/dist/esm/providers/llm/adapters/openai-compatible.js.map +3 -3
- package/dist/esm/providers/llm/types.js.map +2 -2
- package/dist/esm/providers/stt/adapters/openrouter/openrouter.js +1 -1
- package/dist/esm/providers/stt/adapters/openrouter/openrouter.js.map +3 -3
- package/dist/esm/providers/tts/adapters/kokoro/kokoro.js +3 -3
- package/dist/esm/providers/tts/adapters/kokoro/kokoro.js.map +3 -3
- package/dist/esm/providers/tts/adapters/openrouter/openrouter.js +1 -1
- package/dist/esm/providers/tts/adapters/openrouter/openrouter.js.map +3 -3
- package/dist/types/client/client.d.ts +186 -0
- package/dist/types/client/index.d.ts +31 -0
- package/dist/types/client/json.d.ts +13 -0
- package/dist/types/client/protocol/websocket.d.ts +56 -0
- package/dist/types/client/protocol.d.ts +46 -0
- package/dist/types/conversations/conversation/conversation.d.ts +45 -0
- package/dist/types/conversations/conversations.d.ts +24 -0
- package/dist/types/conversations/orchestration/orchestrator/generation/generation.d.ts +17 -0
- package/dist/types/conversations/orchestration/orchestrator/orchestrator.d.ts +25 -6
- package/dist/types/conversations/orchestration/orchestrator/speech/speech.d.ts +31 -2
- package/dist/types/conversations/orchestration/orchestrator/tools/tools.d.ts +41 -7
- package/dist/types/conversations/orchestration/test-harness.d.ts +4 -0
- package/dist/types/pipeflow/pipeflow.d.ts +8 -0
- package/dist/types/providers/llm/types.d.ts +4 -1
- package/dist/types/providers/stt/adapters/openrouter/openrouter.d.ts +103 -1
- package/dist/types/providers/tts/adapters/kokoro/kokoro.d.ts +1 -1
- package/dist/types/providers/tts/adapters/openrouter/openrouter.d.ts +26 -7
- package/package.json +7 -1
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pipeflow client SDK: talk to a Pipeflow server from a browser or Node.
|
|
3
|
+
*/
|
|
4
|
+
import type { AudioChunk, ConversationId, Generation, ToolCall, ToolCallResult, Turn, UserId } from "../conversations/types.js";
|
|
5
|
+
import type { TranscriptEntry } from "../conversations/transcription/transcription.js";
|
|
6
|
+
import type { Protocol } from "./protocol.js";
|
|
7
|
+
/** Track constraints used when opening the mic. */
|
|
8
|
+
export interface CaptureConstraints {
|
|
9
|
+
echoCancellation?: boolean;
|
|
10
|
+
noiseSuppression?: boolean;
|
|
11
|
+
autoGainControl?: boolean;
|
|
12
|
+
}
|
|
13
|
+
/** Optional mic auto-attach on `connect()`. */
|
|
14
|
+
export interface AudioInputOptions {
|
|
15
|
+
/** Auto-attach `getUserMedia` and stream mic audio into the connection. */
|
|
16
|
+
input?: boolean;
|
|
17
|
+
/**
|
|
18
|
+
* Gate or rewrite captured audio before it hits the wire: return a chunk
|
|
19
|
+
* to send it, or `null` to drop it (a VAD gate). May be async. Chunks are
|
|
20
|
+
* sent as the hook resolves and their `sequence` marks capture order, so
|
|
21
|
+
* the server can reorder them before STT. Applies only to captured
|
|
22
|
+
* audio; manual `sendAudio()` calls go out untouched.
|
|
23
|
+
*/
|
|
24
|
+
transform?: (chunk: AudioChunk) => AudioChunk | null | Promise<AudioChunk | null>;
|
|
25
|
+
/**
|
|
26
|
+
* Mic track constraints. Each defaults to `true`. Echo cancellation is the
|
|
27
|
+
* one that matters: without it the mic re-captures the agent's own reply.
|
|
28
|
+
*/
|
|
29
|
+
constraints?: CaptureConstraints;
|
|
30
|
+
}
|
|
31
|
+
export interface ReconnectOptions {
|
|
32
|
+
/** Maximum reconnect attempts after an unexpected close. `0` disables it. */
|
|
33
|
+
maxAttempts?: number;
|
|
34
|
+
/** Initial backoff in ms; doubled each attempt up to `maxBackoffMs`. */
|
|
35
|
+
initialBackoffMs?: number;
|
|
36
|
+
/** Backoff cap in ms. */
|
|
37
|
+
maxBackoffMs?: number;
|
|
38
|
+
}
|
|
39
|
+
export interface PipeflowClientOptions {
|
|
40
|
+
/** The wire protocol. */
|
|
41
|
+
protocol: Protocol;
|
|
42
|
+
/** Audio capture options. */
|
|
43
|
+
audio?: AudioInputOptions;
|
|
44
|
+
/** Auto-reconnect on unexpected close. */
|
|
45
|
+
reconnect?: ReconnectOptions;
|
|
46
|
+
/** Default user id for `sendText()` / `sendAudio()` when not provided. */
|
|
47
|
+
userId?: UserId;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Payloads emitted by `PipeflowClient` events, mirroring `ConversationEvents`
|
|
51
|
+
* for the events that travel the wire.
|
|
52
|
+
*/
|
|
53
|
+
export interface PipeflowClientEventMap {
|
|
54
|
+
start: {
|
|
55
|
+
conversationId: ConversationId;
|
|
56
|
+
};
|
|
57
|
+
stop: {
|
|
58
|
+
conversationId: ConversationId;
|
|
59
|
+
};
|
|
60
|
+
"audio-in": {
|
|
61
|
+
conversationId: ConversationId;
|
|
62
|
+
userId: UserId;
|
|
63
|
+
audio: AudioChunk;
|
|
64
|
+
};
|
|
65
|
+
audio: {
|
|
66
|
+
conversationId: ConversationId;
|
|
67
|
+
audio: AudioChunk;
|
|
68
|
+
};
|
|
69
|
+
"partial-transcript": {
|
|
70
|
+
conversationId: ConversationId;
|
|
71
|
+
userId: UserId;
|
|
72
|
+
text: string;
|
|
73
|
+
};
|
|
74
|
+
turn: {
|
|
75
|
+
conversationId: ConversationId;
|
|
76
|
+
turn: Turn;
|
|
77
|
+
};
|
|
78
|
+
transcript: {
|
|
79
|
+
conversationId: ConversationId;
|
|
80
|
+
entry: TranscriptEntry;
|
|
81
|
+
};
|
|
82
|
+
generation: {
|
|
83
|
+
conversationId: ConversationId;
|
|
84
|
+
generation: Generation;
|
|
85
|
+
};
|
|
86
|
+
"tool-call": {
|
|
87
|
+
conversationId: ConversationId;
|
|
88
|
+
call: ToolCall;
|
|
89
|
+
};
|
|
90
|
+
"tool-call-result": {
|
|
91
|
+
conversationId: ConversationId;
|
|
92
|
+
result: ToolCallResult;
|
|
93
|
+
};
|
|
94
|
+
interrupt: {
|
|
95
|
+
conversationId: ConversationId;
|
|
96
|
+
};
|
|
97
|
+
error: {
|
|
98
|
+
conversationId?: ConversationId;
|
|
99
|
+
error: Error;
|
|
100
|
+
};
|
|
101
|
+
state: {
|
|
102
|
+
conversationId: ConversationId;
|
|
103
|
+
state: unknown;
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
export type PipeflowClientEvent = keyof PipeflowClientEventMap;
|
|
107
|
+
export type Listener<E extends PipeflowClientEvent> = (event: PipeflowClientEventMap[E]) => void;
|
|
108
|
+
/**
|
|
109
|
+
* Typed client for a Pipeflow server: `connect()`, then events flow while
|
|
110
|
+
* you send text and audio, then `disconnect()`.
|
|
111
|
+
*
|
|
112
|
+
* ```ts
|
|
113
|
+
* const client = new PipeflowClient({
|
|
114
|
+
* protocol: new WebSocketProtocol({ url: "wss://…/conversations/abc" }),
|
|
115
|
+
* audio: { input: true },
|
|
116
|
+
* });
|
|
117
|
+
* client.on("turn", (e) => console.log(e.turn.text));
|
|
118
|
+
* await client.connect();
|
|
119
|
+
* ```
|
|
120
|
+
*
|
|
121
|
+
* Mic capture is opt-in via `audio.input`. Playback is the caller's job:
|
|
122
|
+
* wire the `audio` event into your own `AudioContext` or `<audio>` queue,
|
|
123
|
+
* or override `playAudio()`.
|
|
124
|
+
*/
|
|
125
|
+
export declare class PipeflowClient {
|
|
126
|
+
private readonly protocol;
|
|
127
|
+
private readonly audio;
|
|
128
|
+
private readonly reconnect;
|
|
129
|
+
private readonly defaultUserId;
|
|
130
|
+
private readonly listeners;
|
|
131
|
+
private mediaStream;
|
|
132
|
+
private mediaRecorder;
|
|
133
|
+
private startMicPromise;
|
|
134
|
+
private reconnectAttempts;
|
|
135
|
+
private reconnectTimer;
|
|
136
|
+
private closedByUser;
|
|
137
|
+
private nextSequence;
|
|
138
|
+
constructor(options: PipeflowClientOptions);
|
|
139
|
+
/**
|
|
140
|
+
* Open the connection and, if `audio.input` is set, attach the mic.
|
|
141
|
+
* Subsequent calls are no-ops while the connection is open.
|
|
142
|
+
*/
|
|
143
|
+
connect(): Promise<void>;
|
|
144
|
+
/** Close the connection, release the mic, and disable reconnect. */
|
|
145
|
+
disconnect(): Promise<void>;
|
|
146
|
+
/**
|
|
147
|
+
* Send a finalized text turn. Equivalent to `conversation.send({ userId, text })`.
|
|
148
|
+
*/
|
|
149
|
+
sendText(text: string, options?: {
|
|
150
|
+
userId?: UserId;
|
|
151
|
+
}): void;
|
|
152
|
+
/**
|
|
153
|
+
* Send a raw audio chunk. `listen()` on the server is synchronous; call
|
|
154
|
+
* this for every chunk as it arrives. The chunk's `sequence` marks
|
|
155
|
+
* capture order for the server's reorder buffer.
|
|
156
|
+
*/
|
|
157
|
+
sendAudio(audio: AudioChunk, options?: {
|
|
158
|
+
userId?: UserId;
|
|
159
|
+
}): void;
|
|
160
|
+
/** Request an interrupt on the server. */
|
|
161
|
+
interrupt(options?: {
|
|
162
|
+
conversationId?: ConversationId;
|
|
163
|
+
}): void;
|
|
164
|
+
/** Resolve a tool call when `autoExecuteTools` is off on the server. */
|
|
165
|
+
resolveToolCall(result: ToolCallResult, options?: {
|
|
166
|
+
conversationId?: ConversationId;
|
|
167
|
+
}): void;
|
|
168
|
+
/**
|
|
169
|
+
* No-op playback hook. Override it, or listen to `audio` directly.
|
|
170
|
+
*/
|
|
171
|
+
playAudio(_audio: AudioChunk): void;
|
|
172
|
+
/** Subscribe to an event. Returns an unsubscribe function. */
|
|
173
|
+
on<E extends PipeflowClientEvent>(event: E, listener: Listener<E>): () => void;
|
|
174
|
+
/** Remove a previously added listener. */
|
|
175
|
+
off<E extends PipeflowClientEvent>(event: E, listener: Listener<E>): void;
|
|
176
|
+
private send;
|
|
177
|
+
private handleMessage;
|
|
178
|
+
private handleStatus;
|
|
179
|
+
private scheduleReconnect;
|
|
180
|
+
private clearReconnect;
|
|
181
|
+
/** Attach the mic once, even under concurrent `connect()` calls. */
|
|
182
|
+
private startMic;
|
|
183
|
+
private openMic;
|
|
184
|
+
private stopMic;
|
|
185
|
+
private emit;
|
|
186
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Browser/client SDK for talking to a Pipeflow server.
|
|
3
|
+
*
|
|
4
|
+
* The client is intentionally tiny: a typed event surface that mirrors
|
|
5
|
+
* `Conversation`, backed by a swappable wire protocol. It knows nothing about
|
|
6
|
+
* STT/LLM/TTS: it forwards your input and surfaces what comes back.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { PipeflowClient, WebSocketProtocol } from "@moureau/pipeflow/client";
|
|
10
|
+
*
|
|
11
|
+
* const client = new PipeflowClient({
|
|
12
|
+
* protocol: new WebSocketProtocol({
|
|
13
|
+
* url: "wss://api.example.com/conversations/abc",
|
|
14
|
+
* token: () => localStorage.getItem("jwt"),
|
|
15
|
+
* }),
|
|
16
|
+
* audio: { input: true },
|
|
17
|
+
* });
|
|
18
|
+
*
|
|
19
|
+
* client.on("transcript", (e) => render(e.entry.text));
|
|
20
|
+
* client.on("audio", (e) => client.playAudio(e.audio));
|
|
21
|
+
* await client.connect();
|
|
22
|
+
* await client.sendText("hello", { userId: "alice" });
|
|
23
|
+
* ```
|
|
24
|
+
*
|
|
25
|
+
* @packageDocumentation
|
|
26
|
+
*/
|
|
27
|
+
export { PipeflowClient } from "./client.js";
|
|
28
|
+
export type { PipeflowClientOptions, PipeflowClientEvent, PipeflowClientEventMap, Listener, ReconnectOptions, AudioInputOptions, CaptureConstraints, } from "./client.js";
|
|
29
|
+
export type { Protocol, ProtocolStatus, ProtocolMessage, ProtocolListener, StatusListener, } from "./protocol.js";
|
|
30
|
+
export { WebSocketProtocol } from "./protocol/websocket.js";
|
|
31
|
+
export type { WebSocketProtocolOptions, TokenProvider, } from "./protocol/websocket.js";
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal JSON value guard for outbound messages.
|
|
3
|
+
*
|
|
4
|
+
* Kept narrow on purpose: server payloads carry domain types (turns,
|
|
5
|
+
* transcripts, audio chunks) that aren't structurally typed as JSON values,
|
|
6
|
+
* so the wire uses `unknown`. Outbound payloads, however, must serialize;
|
|
7
|
+
* this guard catches values that JSON.stringify would silently drop
|
|
8
|
+
* (functions, `undefined`, `bigint`).
|
|
9
|
+
*/
|
|
10
|
+
export type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
11
|
+
[key: string]: JsonValue;
|
|
12
|
+
};
|
|
13
|
+
export declare function isJsonSerializable(value: unknown): boolean;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* WebSocket-backed implementation of the `Protocol` contract.
|
|
3
|
+
*
|
|
4
|
+
* The URL, auth token, and subprotocols are passed through to the underlying
|
|
5
|
+
* `WebSocket`. The token is resolved lazily on each `connect()` and appended
|
|
6
|
+
* to the URL as a `token` query parameter so the server can authenticate the
|
|
7
|
+
* upgrade (WebSocket has no `Authorization` header mechanism).
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* const protocol = new WebSocketProtocol({
|
|
11
|
+
* url: "wss://api.example.com/conversations/abc",
|
|
12
|
+
* token: () => localStorage.getItem("jwt"),
|
|
13
|
+
* });
|
|
14
|
+
* ```
|
|
15
|
+
*/
|
|
16
|
+
import type { Protocol, ProtocolListener, ProtocolMessage, ProtocolStatus, StatusListener } from "../protocol.js";
|
|
17
|
+
/**
|
|
18
|
+
* Resolves the connection token. Called on every `connect()` so the caller
|
|
19
|
+
* can rotate credentials without rebuilding the client. The resolved value
|
|
20
|
+
* is appended to the URL as `?token=…` (overwriting any existing value).
|
|
21
|
+
*/
|
|
22
|
+
export type TokenProvider = () => string | null | Promise<string | null>;
|
|
23
|
+
export interface WebSocketProtocolOptions {
|
|
24
|
+
/** Fully-qualified WebSocket URL, e.g. `wss://api.example.com/conversations/abc`. */
|
|
25
|
+
url: string;
|
|
26
|
+
/** Optional token resolver; the resolved value is appended as `?token=…`. */
|
|
27
|
+
token?: TokenProvider;
|
|
28
|
+
/** Subprotocols passed to the WebSocket constructor. */
|
|
29
|
+
protocols?: string | string[];
|
|
30
|
+
/**
|
|
31
|
+
* Inject a custom WebSocket constructor (for tests, polyfills, or tracing).
|
|
32
|
+
* Defaults to the global `WebSocket`.
|
|
33
|
+
*/
|
|
34
|
+
socketFactory?: (url: string, protocols?: string | string[]) => WebSocket;
|
|
35
|
+
}
|
|
36
|
+
export declare class WebSocketProtocol implements Protocol {
|
|
37
|
+
readonly url: string;
|
|
38
|
+
private readonly token?;
|
|
39
|
+
private readonly protocols?;
|
|
40
|
+
private readonly socketFactory;
|
|
41
|
+
private socket;
|
|
42
|
+
private _status;
|
|
43
|
+
private readonly messageListeners;
|
|
44
|
+
private readonly statusListeners;
|
|
45
|
+
constructor(options: WebSocketProtocolOptions);
|
|
46
|
+
get status(): ProtocolStatus;
|
|
47
|
+
connect(): Promise<void>;
|
|
48
|
+
close(): Promise<void>;
|
|
49
|
+
send(message: ProtocolMessage): void;
|
|
50
|
+
onMessage(listener: ProtocolListener): () => void;
|
|
51
|
+
onStatus(listener: StatusListener): () => void;
|
|
52
|
+
private readonly handleMessage;
|
|
53
|
+
private readonly handleClose;
|
|
54
|
+
private teardownSocket;
|
|
55
|
+
private setStatus;
|
|
56
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wire-level contract between a `PipeflowClient` and a Pipeflow server.
|
|
3
|
+
*
|
|
4
|
+
* A `Protocol` is responsible only for transport: it opens a connection,
|
|
5
|
+
* delivers `ProtocolMessage`s, and reports its status. The client sits on
|
|
6
|
+
* top and turns those messages into typed events.
|
|
7
|
+
*
|
|
8
|
+
* Swap protocols to change the wire (WebSocket today; SSE, HTTP, or an
|
|
9
|
+
* in-memory pair for tests tomorrow) without touching application code.
|
|
10
|
+
*/
|
|
11
|
+
/** Lifecycle status of a protocol connection. */
|
|
12
|
+
export type ProtocolStatus = "idle" | "connecting" | "open" | "closing" | "closed";
|
|
13
|
+
/**
|
|
14
|
+
* A message exchanged over a `Protocol`. Symmetric: both sides send and
|
|
15
|
+
* receive. `payload` is intentionally `unknown`: payloads are JSON-shaped
|
|
16
|
+
* server domain types (turns, transcripts, audio chunks, …) that the client
|
|
17
|
+
* decodes against its event map at the boundary.
|
|
18
|
+
*/
|
|
19
|
+
export interface ProtocolMessage {
|
|
20
|
+
type: string;
|
|
21
|
+
payload: unknown;
|
|
22
|
+
}
|
|
23
|
+
export type ProtocolListener = (message: ProtocolMessage) => void;
|
|
24
|
+
export type StatusListener = (status: ProtocolStatus) => void;
|
|
25
|
+
/**
|
|
26
|
+
* A bidirectional wire between a Pipeflow client and server.
|
|
27
|
+
*
|
|
28
|
+
* Implementations must be safe to subscribe to before `connect()`: messages
|
|
29
|
+
* received before the listener is attached are buffered or dropped per the
|
|
30
|
+
* implementation's policy, and a listener attached after `connect()` only sees
|
|
31
|
+
* subsequent messages. This mirrors the WebSocket contract.
|
|
32
|
+
*/
|
|
33
|
+
export interface Protocol {
|
|
34
|
+
/** Current lifecycle status. */
|
|
35
|
+
readonly status: ProtocolStatus;
|
|
36
|
+
/** Open the underlying connection. Idempotent: a second call is a no-op. */
|
|
37
|
+
connect(): Promise<void>;
|
|
38
|
+
/** Close the connection. Safe to call from any state. */
|
|
39
|
+
close(): Promise<void>;
|
|
40
|
+
/** Send a message. Throws if the connection is not `open`. */
|
|
41
|
+
send(message: ProtocolMessage): void;
|
|
42
|
+
/** Subscribe to incoming messages. Returns an unsubscribe function. */
|
|
43
|
+
onMessage(listener: ProtocolListener): () => void;
|
|
44
|
+
/** Subscribe to status transitions. Returns an unsubscribe function. */
|
|
45
|
+
onStatus(listener: StatusListener): () => void;
|
|
46
|
+
}
|
|
@@ -17,6 +17,29 @@ export interface ConversationOptions {
|
|
|
17
17
|
*/
|
|
18
18
|
stt?: STT;
|
|
19
19
|
tts?: TTS;
|
|
20
|
+
/**
|
|
21
|
+
* Execute the agents' tools automatically (default `true`), feeding each
|
|
22
|
+
* tool's result back into the model loop. Set `false` to resolve tool
|
|
23
|
+
* calls from your own backend via `resolveToolCall()`.
|
|
24
|
+
*/
|
|
25
|
+
autoExecuteTools?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* TTS synthesis concurrency (default 2): more in-flight requests let a
|
|
28
|
+
* multi-sentence reply synthesize in parallel, removing the gaps between
|
|
29
|
+
* sentences, at the cost of provider concurrency. See
|
|
30
|
+
* `SpeechPipelineOptions.maxConcurrentRequests`.
|
|
31
|
+
*/
|
|
32
|
+
maxConcurrentTtsRequests?: number;
|
|
33
|
+
/**
|
|
34
|
+
* Hold window (ms) for out-of-order audio (default 100). When `listen()`
|
|
35
|
+
* receives a chunk whose `sequence` skips ahead of the expected one, the
|
|
36
|
+
* chunk is held briefly so a packet still in flight can fill the gap —
|
|
37
|
+
* netcode-style reordering, like client-side prediction in multiplayer
|
|
38
|
+
* games. When the window expires the gap is skipped and the buffered
|
|
39
|
+
* chunks are released in order. In-order audio (the common case) is
|
|
40
|
+
* emitted immediately, with no added latency.
|
|
41
|
+
*/
|
|
42
|
+
audioReorderMs?: number;
|
|
20
43
|
}
|
|
21
44
|
export interface ConversationEvents {
|
|
22
45
|
start: {
|
|
@@ -119,9 +142,13 @@ export declare class Conversation {
|
|
|
119
142
|
private readonly persistence;
|
|
120
143
|
private readonly stt;
|
|
121
144
|
private readonly tts;
|
|
145
|
+
private readonly autoExecuteTools;
|
|
146
|
+
private readonly maxConcurrentTtsRequests;
|
|
122
147
|
private readonly listeners;
|
|
123
148
|
private readonly pendingToolCallsById;
|
|
124
149
|
private readonly subGenerations;
|
|
150
|
+
private readonly audioReorderMs;
|
|
151
|
+
private readonly audioReorder;
|
|
125
152
|
private nextAudioSequence;
|
|
126
153
|
constructor(options: ConversationOptions);
|
|
127
154
|
get status(): ConversationStatus;
|
|
@@ -137,11 +164,29 @@ export declare class Conversation {
|
|
|
137
164
|
/**
|
|
138
165
|
* Send an audio packet. Intentionally synchronous: it means "send this
|
|
139
166
|
* packet", not "wait for this utterance to finish".
|
|
167
|
+
*
|
|
168
|
+
* Pass the sender's `sequence` number to enable netcode-style reordering:
|
|
169
|
+
* chunks arriving ahead of a gap are held for `audioReorderMs` so an
|
|
170
|
+
* earlier packet still in flight can fill it, then released in order
|
|
171
|
+
* (gaps are skipped when the window expires). The orchestrator and STT
|
|
172
|
+
* always see the participant's audio in sequence order. Without a
|
|
173
|
+
* `sequence`, the chunk is emitted on arrival, as-is.
|
|
140
174
|
*/
|
|
141
175
|
listen(input: {
|
|
142
176
|
userId: UserId;
|
|
143
177
|
audio: Uint8Array;
|
|
178
|
+
sequence?: number;
|
|
144
179
|
}): void;
|
|
180
|
+
/** Emit an audio-in event with the conversation's own sequence numbering. */
|
|
181
|
+
private emitAudioIn;
|
|
182
|
+
/** Hold/drain logic for sequenced audio, keyed per participant. */
|
|
183
|
+
private reorderAudio;
|
|
184
|
+
/**
|
|
185
|
+
* Release everything a participant has buffered, in sequence order,
|
|
186
|
+
* skipping the gap that never filled.
|
|
187
|
+
*/
|
|
188
|
+
private releaseReorderBuffer;
|
|
189
|
+
private clearReorderTimer;
|
|
145
190
|
/**
|
|
146
191
|
* Send a finalized text turn, bypassing STT. Triggers the same pipeline as
|
|
147
192
|
* a transcribed utterance (routing, coordination, generation, resume) —
|
|
@@ -7,12 +7,35 @@ import type { TranscriptEntry } from "./transcription/transcription.js";
|
|
|
7
7
|
import { Conversation } from "./conversation/conversation.js";
|
|
8
8
|
export interface CreateConversationOptions {
|
|
9
9
|
agents?: Agent[];
|
|
10
|
+
/**
|
|
11
|
+
* Execute the agents' tools automatically (default `true`). Set `false`
|
|
12
|
+
* for this conversation to resolve tool calls from your own backend via
|
|
13
|
+
* `resolveToolCall()`. Overrides the `Conversations`-level default.
|
|
14
|
+
*/
|
|
15
|
+
autoExecuteTools?: boolean;
|
|
16
|
+
/**
|
|
17
|
+
* TTS synthesis concurrency (default 2). More in-flight requests let a
|
|
18
|
+
* multi-sentence reply synthesize in parallel, removing the gaps between
|
|
19
|
+
* sentences, at the cost of provider concurrency — some free TTS variants
|
|
20
|
+
* rate-limit concurrent requests.
|
|
21
|
+
*/
|
|
22
|
+
maxConcurrentTtsRequests?: number;
|
|
23
|
+
/**
|
|
24
|
+
* Hold window (ms) for out-of-order audio (default 100). See
|
|
25
|
+
* `ConversationOptions.audioReorderMs`.
|
|
26
|
+
*/
|
|
27
|
+
audioReorderMs?: number;
|
|
10
28
|
}
|
|
11
29
|
export interface ConversationsOptions {
|
|
12
30
|
persistence: Persistence;
|
|
13
31
|
/** Passed to conversations so `start()` can attach realtime processing. */
|
|
14
32
|
stt?: STT;
|
|
15
33
|
tts?: TTS;
|
|
34
|
+
/**
|
|
35
|
+
* Default for created conversations: auto-execute the agents' tools
|
|
36
|
+
* (default `true`). See `CreateConversationOptions.autoExecuteTools`.
|
|
37
|
+
*/
|
|
38
|
+
autoExecuteTools?: boolean;
|
|
16
39
|
}
|
|
17
40
|
/**
|
|
18
41
|
* The conversations API surface of a Pipeflow instance: create persistent
|
|
@@ -22,6 +45,7 @@ export declare class Conversations {
|
|
|
22
45
|
private readonly persistence;
|
|
23
46
|
private readonly stt;
|
|
24
47
|
private readonly tts;
|
|
48
|
+
private readonly autoExecuteTools;
|
|
25
49
|
constructor(options: ConversationsOptions);
|
|
26
50
|
/** Create a persistent conversation. Realtime execution is separate. */
|
|
27
51
|
create(options?: CreateConversationOptions): Promise<Conversation>;
|
|
@@ -16,6 +16,16 @@ export interface GenerationRequest {
|
|
|
16
16
|
maxToolIterations: number;
|
|
17
17
|
/** False once the run is stale (interrupt/stop); aborts the loop. */
|
|
18
18
|
isCurrent(): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Whether outgoing messages keep per-agent `name` fields on system and
|
|
21
|
+
* assistant messages. Multi-agent conversations need them so the model can
|
|
22
|
+
* tell which agent said what in the shared history. Single-agent
|
|
23
|
+
* conversations drop them: with one agent the names carry no information,
|
|
24
|
+
* and some providers render a message `name` as a role header ("Scout:")
|
|
25
|
+
* that weaker models imitate — every reply then starts by speaking the
|
|
26
|
+
* agent's name. Default true.
|
|
27
|
+
*/
|
|
28
|
+
agentNames?: boolean;
|
|
19
29
|
/** Streamed text; `textBefore` is the running text prior to this delta. */
|
|
20
30
|
onDelta?(delta: string, textBefore: string): void;
|
|
21
31
|
/** Hand tool calls to the application and resolve their results. */
|
|
@@ -28,6 +38,13 @@ export interface GenerationOutcome {
|
|
|
28
38
|
status: GenerationStatus;
|
|
29
39
|
/** The provider error, when `status` is "error". */
|
|
30
40
|
error?: unknown;
|
|
41
|
+
/**
|
|
42
|
+
* How many tool rounds were resolved before the run ended (only present
|
|
43
|
+
* when at least one). Lets callers retry a no-output failure safely:
|
|
44
|
+
* re-running a generation whose tools already executed would repeat their
|
|
45
|
+
* side effects.
|
|
46
|
+
*/
|
|
47
|
+
toolRounds?: number;
|
|
31
48
|
}
|
|
32
49
|
/**
|
|
33
50
|
* Owns the LLM tool loop shared by top-level generations and delegated
|
|
@@ -24,10 +24,19 @@ export interface OrchestratorOptions {
|
|
|
24
24
|
tts?: TTS;
|
|
25
25
|
/** Used to rehydrate conversation history on start. */
|
|
26
26
|
persistence?: Persistence;
|
|
27
|
-
/** How long to wait for
|
|
27
|
+
/** How long to wait for a tool call to resolve (auto-executed or app-managed). */
|
|
28
28
|
toolTimeoutMs?: number;
|
|
29
29
|
/** Safety bound on tool-call round trips per generation. */
|
|
30
30
|
maxToolIterations?: number;
|
|
31
|
+
/**
|
|
32
|
+
* Execute the agents' tools automatically (default `true`), feeding each
|
|
33
|
+
* tool's result — or a caught error — back into the model loop, like
|
|
34
|
+
* `Agent.run()` does. Set `false` for the application-managed contract:
|
|
35
|
+
* listen for `tool-call` events and resolve each call yourself with
|
|
36
|
+
* `conversation.resolveToolCall()`. The `tool-call` event fires in both
|
|
37
|
+
* modes, so apps can observe every call.
|
|
38
|
+
*/
|
|
39
|
+
autoExecuteTools?: boolean;
|
|
31
40
|
/**
|
|
32
41
|
* Additional coordinations the runtime can delegate to, registered by
|
|
33
42
|
* name (the key is the coordination's name).
|
|
@@ -38,12 +47,20 @@ export interface OrchestratorOptions {
|
|
|
38
47
|
temperature?: number;
|
|
39
48
|
maxTokens?: number;
|
|
40
49
|
/**
|
|
41
|
-
*
|
|
50
|
+
* How much conversation history each LLM request carries (default
|
|
42
51
|
* `{ maxTurns: 5, maxChars: 4000 }` — provider TTFT grows with input size,
|
|
43
52
|
* and a bounded window keeps requests in the fast regime). Pass `false` to
|
|
44
53
|
* always send the full history.
|
|
45
54
|
*/
|
|
46
55
|
historyWindow?: HistoryWindow | false;
|
|
56
|
+
/**
|
|
57
|
+
* How many TTS synthesis requests may be in flight at once (default 2).
|
|
58
|
+
* More lets the sentences of a multi-sentence reply synthesize in parallel
|
|
59
|
+
* (kills the gaps between them) at the cost of provider concurrency — some
|
|
60
|
+
* free TTS variants rate-limit concurrent requests. Passed through to the
|
|
61
|
+
* speech pipeline; see `SpeechPipelineOptions.maxConcurrentRequests`.
|
|
62
|
+
*/
|
|
63
|
+
maxConcurrentTtsRequests?: number;
|
|
47
64
|
}
|
|
48
65
|
/**
|
|
49
66
|
* The realtime conversation state machine and multi-agent coordinator.
|
|
@@ -56,13 +73,15 @@ export interface OrchestratorOptions {
|
|
|
56
73
|
* ```text
|
|
57
74
|
* audio-in ──► STT ──► turn ──► coordinator ──► agent ──► LLM ──► TTS ──► audio-out
|
|
58
75
|
* │
|
|
59
|
-
* └─► tool-call ──►
|
|
76
|
+
* └─► tool-call ──► tool runs ──► resume
|
|
60
77
|
* ```
|
|
61
78
|
*
|
|
62
79
|
* Deltas stream to TTS immediately (so the agent can narrate while a tool
|
|
63
|
-
* runs), tool calls pause the generation
|
|
64
|
-
*
|
|
65
|
-
*
|
|
80
|
+
* runs), tool calls pause the generation while the tool executes — the
|
|
81
|
+
* framework runs the agent's own tools by default, or the application
|
|
82
|
+
* resolves them when `autoExecuteTools` is off — and interruptions cancel
|
|
83
|
+
* the current generation, discarding any stale tool results or audio via a
|
|
84
|
+
* generation epoch.
|
|
66
85
|
*/
|
|
67
86
|
export declare class Orchestrator {
|
|
68
87
|
private readonly conversation;
|
|
@@ -10,6 +10,14 @@ export interface SpeechPipelineOptions {
|
|
|
10
10
|
* bump) or a stop drops queued and in-flight synthesis.
|
|
11
11
|
*/
|
|
12
12
|
isCurrent(epoch: number): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* How many synthesis requests may be in flight at once. Sentences are
|
|
15
|
+
* started as soon as a slot is free — the next one synthesizes while the
|
|
16
|
+
* current one is still streaming — so playback never waits for the whole
|
|
17
|
+
* reply, but a burst of many sentences can't flood the provider (free TTS
|
|
18
|
+
* variants rate-limit concurrent requests). Default 2.
|
|
19
|
+
*/
|
|
20
|
+
maxConcurrentRequests?: number;
|
|
13
21
|
}
|
|
14
22
|
/**
|
|
15
23
|
* Turns the LLM delta stream into speakable audio: expose the text stream
|
|
@@ -24,10 +32,15 @@ export declare class SpeechPipeline {
|
|
|
24
32
|
private readonly tts;
|
|
25
33
|
private readonly conversation;
|
|
26
34
|
private readonly isCurrent;
|
|
35
|
+
private readonly maxConcurrent;
|
|
27
36
|
private readonly chunker;
|
|
28
37
|
private speechEpoch;
|
|
29
38
|
private chain;
|
|
30
39
|
private audioSequence;
|
|
40
|
+
/** Flushed sentences still waiting for a synthesis slot. */
|
|
41
|
+
private readonly queue;
|
|
42
|
+
/** Synthesis requests started but not yet fully delivered. */
|
|
43
|
+
private inFlight;
|
|
31
44
|
constructor(options: SpeechPipelineOptions);
|
|
32
45
|
/**
|
|
33
46
|
* Stream an LLM delta to the application and the TTS pipeline. Empty
|
|
@@ -37,13 +50,29 @@ export declare class SpeechPipeline {
|
|
|
37
50
|
feed(delta: string, epoch: number): void;
|
|
38
51
|
/** Speak any remaining buffered text as a final chunk. */
|
|
39
52
|
flush(epoch: number): void;
|
|
40
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Queue a sentence for synthesis, starting it if a slot is free. The text
|
|
55
|
+
* order is preserved end to end: requests start in flush order, and delivery
|
|
56
|
+
* below is serialized through `chain`, so sentence N never overtakes N-1.
|
|
57
|
+
*/
|
|
41
58
|
speak(sentence: string, epoch: number): void;
|
|
59
|
+
/**
|
|
60
|
+
* Start synthesis for queued sentences up to the concurrency bound. A
|
|
61
|
+
* sentence's request is fired as soon as a slot is free — while the previous
|
|
62
|
+
* sentence is still streaming/playing — so the first piece is ready first
|
|
63
|
+
* and later pieces overlap it, without ever flooding the provider with one
|
|
64
|
+
* request per sentence of a long reply.
|
|
65
|
+
*/
|
|
66
|
+
private startNext;
|
|
42
67
|
/**
|
|
43
68
|
* Stop the current TTS playback and drop buffered text without cancelling
|
|
44
69
|
* the generation (barge-in, interrupt, stop).
|
|
45
70
|
*/
|
|
46
71
|
stop(): void;
|
|
47
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* Resolve when every queued and in-flight sentence has been synthesized and
|
|
74
|
+
* delivered. Unlike the delivery chain alone, this also waits for sentences
|
|
75
|
+
* still waiting on a synthesis slot.
|
|
76
|
+
*/
|
|
48
77
|
waitForIdle(): Promise<void>;
|
|
49
78
|
}
|