@juspay/neurolink 13.6.5 → 13.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -2
- package/dist/auth/sessionManager.js +8 -3
- package/dist/browser/neurolink.min.js +480 -460
- package/dist/cli/commands/serve.js +11 -1
- package/dist/cli/commands/transcribe.js +39 -3
- package/dist/constants/langfuse.d.ts +2 -0
- package/dist/constants/langfuse.js +2 -0
- package/dist/core/redisConversationMemoryManager.js +6 -16
- package/dist/factories/sttDescriptors.js +7 -0
- package/dist/index.d.ts +3 -2
- package/dist/index.js +4 -2
- package/dist/neurolink.js +33 -1
- package/dist/observability/exporters/langfuseExporter.d.ts +4 -0
- package/dist/observability/exporters/langfuseExporter.js +29 -8
- package/dist/observability/otlpEncoding.js +2 -2
- package/dist/server/abstract/baseServerAdapter.js +2 -0
- package/dist/server/adapters/expressAdapter.js +1 -1
- package/dist/server/adapters/fastifyAdapter.js +4 -1
- package/dist/server/adapters/honoAdapter.js +1 -1
- package/dist/server/adapters/koaAdapter.js +1 -1
- package/dist/server/routes/transcribeRoutes.d.ts +0 -8
- package/dist/server/routes/transcribeRoutes.js +130 -3
- package/dist/server/utils/validation.d.ts +409 -15
- package/dist/server/utils/validation.js +106 -4
- package/dist/server/websocket/frameQueue.d.ts +16 -0
- package/dist/server/websocket/frameQueue.js +55 -0
- package/dist/server/websocket/realtimeTranscription.d.ts +48 -0
- package/dist/server/websocket/realtimeTranscription.js +603 -0
- package/dist/server/websocket/transcribeWebSocket.d.ts +17 -1
- package/dist/server/websocket/transcribeWebSocket.js +263 -64
- package/dist/services/server/ai/observability/langfuseSetup.js +6 -1
- package/dist/telemetry/attributes.d.ts +1 -1
- package/dist/telemetry/attributes.js +84 -8
- package/dist/types/cli.d.ts +6 -0
- package/dist/types/exporter.d.ts +3 -1
- package/dist/types/server.d.ts +304 -2
- package/dist/types/stt.d.ts +38 -3
- package/dist/types/transcribe.d.ts +401 -5
- package/dist/utils/redis.d.ts +17 -0
- package/dist/utils/redis.js +31 -0
- package/dist/voice/audioLevel.d.ts +14 -0
- package/dist/voice/audioLevel.js +60 -0
- package/dist/voice/codeMix.d.ts +40 -0
- package/dist/voice/codeMix.js +282 -0
- package/dist/voice/correction/align.d.ts +12 -0
- package/dist/voice/correction/align.js +48 -0
- package/dist/voice/correction/guards.d.ts +39 -0
- package/dist/voice/correction/guards.js +122 -0
- package/dist/voice/correction/index.d.ts +25 -3
- package/dist/voice/correction/index.js +132 -4
- package/dist/voice/correction/rewrite.d.ts +64 -3
- package/dist/voice/correction/rewrite.js +295 -25
- package/dist/voice/eval/sttEvalLangfuse.d.ts +14 -0
- package/dist/voice/eval/sttEvalLangfuse.js +182 -0
- package/dist/voice/eval/sttEvalStore.d.ts +27 -0
- package/dist/voice/eval/sttEvalStore.js +397 -0
- package/dist/voice/index.js +3 -0
- package/dist/voice/languages.d.ts +34 -0
- package/dist/voice/languages.js +290 -0
- package/dist/voice/modelAssets.d.ts +28 -0
- package/dist/voice/modelAssets.js +113 -0
- package/dist/voice/output/render.d.ts +32 -0
- package/dist/voice/output/render.js +368 -0
- package/dist/voice/providers/OpenAISTT.d.ts +11 -1
- package/dist/voice/providers/OpenAISTT.js +57 -6
- package/dist/voice/session/transcriptionSession.d.ts +55 -0
- package/dist/voice/session/transcriptionSession.js +225 -0
- package/dist/voice/streaming/chunkedStreamAdapter.d.ts +7 -0
- package/dist/voice/streaming/chunkedStreamAdapter.js +711 -28
- package/dist/voice/streaming/endpointer.d.ts +50 -17
- package/dist/voice/streaming/endpointer.js +115 -19
- package/dist/voice/transcribe.d.ts +18 -1
- package/dist/voice/transcribe.js +459 -18
- package/dist/voice/transcribeStream.d.ts +18 -1
- package/dist/voice/transcribeStream.js +413 -16
- package/dist/voice/vad/assets.d.ts +36 -0
- package/dist/voice/vad/assets.js +142 -0
- package/dist/voice/vad/frameScorer.d.ts +28 -0
- package/dist/voice/vad/frameScorer.js +90 -0
- package/dist/voice/vad/sileroVad.d.ts +43 -0
- package/dist/voice/vad/sileroVad.js +178 -0
- package/dist/voice/whistle/assets.d.ts +3 -4
- package/dist/voice/whistle/assets.js +11 -95
- package/docs-site/static/search-index.json +21 -6
- package/models/silero-vad/README.md +19 -0
- package/models/silero-vad/silero_vad.onnx +0 -0
- package/package.json +4 -1
|
@@ -8,73 +8,49 @@
|
|
|
8
8
|
* 2. Then BINARY frames of PCM16LE mono audio at `streaming.sampleRate`
|
|
9
9
|
* (default 16000).
|
|
10
10
|
* 3. A TEXT frame `{"type":"end"}` (or closing the socket) ends the audio; the
|
|
11
|
-
* server flushes the last utterance.
|
|
11
|
+
* server flushes the last utterance. `{"type":"output","languages":[…]}`
|
|
12
|
+
* changes the output languages from the next utterance on; any other text
|
|
13
|
+
* frame ends the audio, as before.
|
|
12
14
|
*
|
|
13
15
|
* The server sends one TEXT frame per `TranscribeStreamEvent` (JSON), then
|
|
14
16
|
* closes with 1000. A bad config is answered with an `error` event and close
|
|
15
17
|
* code 1008.
|
|
16
18
|
*
|
|
19
|
+
* Sessions (one call, several speakers, listeners in their own languages):
|
|
20
|
+
* a speaker connects to the stream path with `?session=<id>&speaker=<name>`
|
|
21
|
+
* and uses the protocol above; a listener connects to the subscribe path
|
|
22
|
+
* (default `<path>/subscribe`) with `?session=<id>` and sends
|
|
23
|
+
* `{"type":"subscribe","languages":["es"],"speakers":[…]}` — again at any time
|
|
24
|
+
* to change them. Listeners receive every event of the speakers they follow,
|
|
25
|
+
* tagged with `speaker`, and only the `output` events of their languages; a
|
|
26
|
+
* session renders each utterance once into the union of its listeners'
|
|
27
|
+
* languages. A session lives while a speaker or a listener is connected.
|
|
28
|
+
*
|
|
29
|
+
* OpenAI Realtime transcription (default path `/v1/realtime`,
|
|
30
|
+
* `?intent=transcription`): the same pipeline behind OpenAI's protocol, for
|
|
31
|
+
* clients that already speak it (see realtimeTranscription.ts).
|
|
32
|
+
*
|
|
17
33
|
* The route adapters do not handle upgrades, so this attaches to the Node
|
|
18
34
|
* `http.Server` the caller owns, next to whatever else is listening there; it
|
|
19
35
|
* answers only upgrades on its own path.
|
|
20
36
|
*
|
|
21
37
|
* @module server/websocket/transcribeWebSocket
|
|
22
38
|
*/
|
|
39
|
+
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
|
40
|
+
import { join } from "node:path";
|
|
23
41
|
import { logger } from "../../utils/logger.js";
|
|
24
42
|
import { timingSafeEqualString } from "../voice/tokenCompare.js";
|
|
25
|
-
import { TranscribeStreamConfigSchema } from "../utils/validation.js";
|
|
43
|
+
import { TranscribeOutputControlSchema, TranscribeStreamConfigSchema, TranscribeSubscribeSchema, } from "../utils/validation.js";
|
|
44
|
+
import { fileModeStreamOptions } from "../../voice/transcribe.js";
|
|
45
|
+
import { TranscriptionSession } from "../../voice/session/transcriptionSession.js";
|
|
46
|
+
import { FrameQueue } from "./frameQueue.js";
|
|
47
|
+
import { resolveRealtimeProfiles, serveRealtimeTranscription, } from "./realtimeTranscription.js";
|
|
26
48
|
const DEFAULT_PATH = "/v1/audio/transcriptions/stream";
|
|
49
|
+
const DEFAULT_REALTIME_PATH = "/v1/realtime";
|
|
27
50
|
/** Audio waiting for the engine beyond this (about 8.5 min at 16 kHz) closes the socket. */
|
|
28
51
|
const MAX_QUEUED_BYTES = 16 * 1024 * 1024;
|
|
29
|
-
/**
|
|
30
|
-
|
|
31
|
-
frames = [];
|
|
32
|
-
waiting;
|
|
33
|
-
ended = false;
|
|
34
|
-
queuedBytes = 0;
|
|
35
|
-
push(frame) {
|
|
36
|
-
if (this.ended) {
|
|
37
|
-
return;
|
|
38
|
-
}
|
|
39
|
-
if (this.waiting) {
|
|
40
|
-
const resolve = this.waiting;
|
|
41
|
-
this.waiting = undefined;
|
|
42
|
-
resolve({ value: frame, done: false });
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
|
-
this.frames.push(frame);
|
|
46
|
-
this.queuedBytes += frame.length;
|
|
47
|
-
}
|
|
48
|
-
end() {
|
|
49
|
-
this.ended = true;
|
|
50
|
-
if (this.waiting) {
|
|
51
|
-
const resolve = this.waiting;
|
|
52
|
-
this.waiting = undefined;
|
|
53
|
-
resolve({ value: undefined, done: true });
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
[Symbol.asyncIterator]() {
|
|
57
|
-
return {
|
|
58
|
-
next: () => {
|
|
59
|
-
const frame = this.frames.shift();
|
|
60
|
-
if (frame) {
|
|
61
|
-
this.queuedBytes -= frame.length;
|
|
62
|
-
return Promise.resolve({ value: frame, done: false });
|
|
63
|
-
}
|
|
64
|
-
if (this.ended) {
|
|
65
|
-
return Promise.resolve({ value: undefined, done: true });
|
|
66
|
-
}
|
|
67
|
-
return new Promise((resolve) => {
|
|
68
|
-
this.waiting = resolve;
|
|
69
|
-
});
|
|
70
|
-
},
|
|
71
|
-
return: () => {
|
|
72
|
-
this.end();
|
|
73
|
-
return Promise.resolve({ value: undefined, done: true });
|
|
74
|
-
},
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
}
|
|
52
|
+
/** In file mode the whole recording may be queued at once: 512 MB is about 4.5 hours of 16 kHz PCM. */
|
|
53
|
+
const MAX_QUEUED_BYTES_FILE = 512 * 1024 * 1024;
|
|
78
54
|
/**
|
|
79
55
|
* `true` when the upgrade's `Origin` is on the allow-list, the list allows
|
|
80
56
|
* any origin, or the request carries no `Origin` (not a browser).
|
|
@@ -90,7 +66,7 @@ function isOriginAllowed(req, allowed) {
|
|
|
90
66
|
const wanted = origin.trim().toLowerCase();
|
|
91
67
|
return allowed.some((o) => o.trim().toLowerCase() === wanted);
|
|
92
68
|
}
|
|
93
|
-
function isAuthorized(req, token) {
|
|
69
|
+
function isAuthorized(req, token, allowSubprotocolKey = false) {
|
|
94
70
|
const tokens = (typeof token === "string" ? [token] : (token ?? [])).filter((t) => t.length > 0);
|
|
95
71
|
if (tokens.length === 0) {
|
|
96
72
|
return true;
|
|
@@ -108,7 +84,16 @@ function isAuthorized(req, token) {
|
|
|
108
84
|
catch {
|
|
109
85
|
urlToken = undefined;
|
|
110
86
|
}
|
|
111
|
-
|
|
87
|
+
// OpenAI's browser SDK cannot set headers, so it sends the key as the
|
|
88
|
+
// subprotocol "openai-insecure-api-key.<key>"; honoured on the realtime path.
|
|
89
|
+
const protocolToken = allowSubprotocolKey
|
|
90
|
+
? String(req.headers["sec-websocket-protocol"] ?? "")
|
|
91
|
+
.split(",")
|
|
92
|
+
.map((p) => p.trim())
|
|
93
|
+
.find((p) => p.startsWith("openai-insecure-api-key."))
|
|
94
|
+
?.slice("openai-insecure-api-key.".length)
|
|
95
|
+
: undefined;
|
|
96
|
+
const provided = headerToken ?? urlToken ?? protocolToken;
|
|
112
97
|
return (provided !== undefined &&
|
|
113
98
|
tokens.some((t) => timingSafeEqualString(provided, t)));
|
|
114
99
|
}
|
|
@@ -120,6 +105,26 @@ function pathOf(req) {
|
|
|
120
105
|
return "";
|
|
121
106
|
}
|
|
122
107
|
}
|
|
108
|
+
/** A query parameter of the upgrade URL, trimmed and length-capped; `undefined` when absent. */
|
|
109
|
+
function queryParam(req, name) {
|
|
110
|
+
try {
|
|
111
|
+
const value = new URL(req.url ?? "/", "http://localhost").searchParams
|
|
112
|
+
.get(name)
|
|
113
|
+
?.trim();
|
|
114
|
+
return value ? value.slice(0, 100) : undefined;
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return undefined;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
function parseJson(frame) {
|
|
121
|
+
try {
|
|
122
|
+
return JSON.parse(frame.toString("utf-8"));
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return undefined;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
123
128
|
/**
|
|
124
129
|
* Serve `transcribeStream()` on `path` (default
|
|
125
130
|
* `/v1/audio/transcriptions/stream`) of an existing HTTP server.
|
|
@@ -138,12 +143,46 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
138
143
|
// Loaded on use so `ws` stays off the package's static import graph.
|
|
139
144
|
const { WebSocketServer, WebSocket } = await import("ws");
|
|
140
145
|
const path = options.path ?? DEFAULT_PATH;
|
|
146
|
+
const subscribePath = options.subscribePath === false
|
|
147
|
+
? undefined
|
|
148
|
+
: (options.subscribePath ?? `${path}/subscribe`);
|
|
149
|
+
const realtimePath = options.realtimePath === false
|
|
150
|
+
? undefined
|
|
151
|
+
: (options.realtimePath ?? DEFAULT_REALTIME_PATH);
|
|
152
|
+
const realtimeProfiles = resolveRealtimeProfiles(options.realtimeProfiles);
|
|
141
153
|
const wss = new WebSocketServer({
|
|
142
154
|
noServer: true,
|
|
143
155
|
maxPayload: options.maxPayload ?? 1_048_576,
|
|
156
|
+
// OpenAI's browser SDK offers "realtime" alongside its key subprotocol.
|
|
157
|
+
handleProtocols: (protocols) => protocols.has("realtime") ? "realtime" : false,
|
|
144
158
|
});
|
|
159
|
+
// Sessions by id, alive while a speaker or a listener holds them.
|
|
160
|
+
const sessions = new Map();
|
|
161
|
+
const acquire = (id) => {
|
|
162
|
+
const entry = sessions.get(id) ?? {
|
|
163
|
+
session: new TranscriptionSession(neurolink, { id }),
|
|
164
|
+
refs: 0,
|
|
165
|
+
};
|
|
166
|
+
entry.refs++;
|
|
167
|
+
sessions.set(id, entry);
|
|
168
|
+
return entry.session;
|
|
169
|
+
};
|
|
170
|
+
const release = (id) => {
|
|
171
|
+
const entry = sessions.get(id);
|
|
172
|
+
if (!entry) {
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
entry.refs--;
|
|
176
|
+
if (entry.refs <= 0) {
|
|
177
|
+
entry.session.close();
|
|
178
|
+
sessions.delete(id);
|
|
179
|
+
}
|
|
180
|
+
};
|
|
145
181
|
const onUpgrade = (req, socket, head) => {
|
|
146
|
-
|
|
182
|
+
const requested = pathOf(req);
|
|
183
|
+
if (requested !== path &&
|
|
184
|
+
requested !== subscribePath &&
|
|
185
|
+
requested !== realtimePath) {
|
|
147
186
|
return; // Someone else's upgrade.
|
|
148
187
|
}
|
|
149
188
|
if (!isOriginAllowed(req, options.allowedOrigins)) {
|
|
@@ -151,7 +190,7 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
151
190
|
socket.destroy();
|
|
152
191
|
return;
|
|
153
192
|
}
|
|
154
|
-
if (!isAuthorized(req, options.authToken)) {
|
|
193
|
+
if (!isAuthorized(req, options.authToken, requested === realtimePath)) {
|
|
155
194
|
socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n");
|
|
156
195
|
socket.destroy();
|
|
157
196
|
return;
|
|
@@ -161,22 +200,164 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
161
200
|
});
|
|
162
201
|
};
|
|
163
202
|
server.on("upgrade", onUpgrade);
|
|
164
|
-
|
|
203
|
+
/** A listener: subscribe frames in, the session's events (filtered) out. */
|
|
204
|
+
const onListener = (client, req) => {
|
|
205
|
+
const id = queryParam(req, "session");
|
|
206
|
+
if (!id) {
|
|
207
|
+
client.send(JSON.stringify({
|
|
208
|
+
type: "error",
|
|
209
|
+
message: "A listener needs ?session=<id>",
|
|
210
|
+
recoverable: false,
|
|
211
|
+
}));
|
|
212
|
+
client.close(1008, "session required");
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
const session = acquire(id);
|
|
216
|
+
let subscription;
|
|
217
|
+
const deliver = (event) => {
|
|
218
|
+
if (client.readyState === WebSocket.OPEN) {
|
|
219
|
+
client.send(JSON.stringify(event));
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
client.on("message", (data) => {
|
|
223
|
+
const frame = Buffer.isBuffer(data)
|
|
224
|
+
? data
|
|
225
|
+
: Array.isArray(data)
|
|
226
|
+
? Buffer.concat(data)
|
|
227
|
+
: Buffer.from(data);
|
|
228
|
+
const parsed = TranscribeSubscribeSchema.safeParse(parseJson(frame));
|
|
229
|
+
if (!parsed.success) {
|
|
230
|
+
client.send(JSON.stringify({
|
|
231
|
+
type: "error",
|
|
232
|
+
message: 'Expected {"type":"subscribe","languages":[…],"speakers":[…]}',
|
|
233
|
+
recoverable: true,
|
|
234
|
+
}));
|
|
235
|
+
return;
|
|
236
|
+
}
|
|
237
|
+
if (subscription) {
|
|
238
|
+
subscription.update({
|
|
239
|
+
...(parsed.data.languages
|
|
240
|
+
? { languages: parsed.data.languages }
|
|
241
|
+
: {}),
|
|
242
|
+
...(parsed.data.speakers ? { speakers: parsed.data.speakers } : {}),
|
|
243
|
+
});
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
subscription = session.subscribe(parsed.data, deliver);
|
|
247
|
+
});
|
|
248
|
+
let released = false;
|
|
249
|
+
const leave = () => {
|
|
250
|
+
if (released) {
|
|
251
|
+
return;
|
|
252
|
+
}
|
|
253
|
+
released = true;
|
|
254
|
+
subscription?.close();
|
|
255
|
+
release(id);
|
|
256
|
+
};
|
|
257
|
+
client.on("close", leave);
|
|
258
|
+
client.on("error", leave);
|
|
259
|
+
};
|
|
260
|
+
wss.on("connection", (client, req) => {
|
|
261
|
+
if (subscribePath && pathOf(req) === subscribePath) {
|
|
262
|
+
onListener(client, req);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (realtimePath && pathOf(req) === realtimePath) {
|
|
266
|
+
serveRealtimeTranscription(client, req, neurolink, realtimeProfiles);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
165
269
|
const queue = new FrameQueue();
|
|
166
270
|
let started = false;
|
|
271
|
+
let fileMode = false;
|
|
272
|
+
// Debugging: `NEUROLINK_STT_RECORD_DIR` keeps every session's config,
|
|
273
|
+
// audio (PCM16LE) and events on disk, so a live session can be replayed
|
|
274
|
+
// and compared against another engine. Off unless set.
|
|
275
|
+
const recordRoot = process.env.NEUROLINK_STT_RECORD_DIR?.trim();
|
|
276
|
+
const recordDir = recordRoot
|
|
277
|
+
? join(recordRoot, `${new Date().toISOString().replace(/[:.]/g, "-")}-${Math.random().toString(36).slice(2, 6)}`)
|
|
278
|
+
: undefined;
|
|
279
|
+
const record = (name, data, append = true) => {
|
|
280
|
+
if (!recordDir) {
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
try {
|
|
284
|
+
mkdirSync(recordDir, { recursive: true });
|
|
285
|
+
(append ? appendFileSync : writeFileSync)(join(recordDir, name), data);
|
|
286
|
+
}
|
|
287
|
+
catch {
|
|
288
|
+
// Recording is best effort; it never affects the session.
|
|
289
|
+
}
|
|
290
|
+
};
|
|
291
|
+
// Output languages, changeable mid-stream by control frames.
|
|
292
|
+
let liveLanguages = [];
|
|
293
|
+
const sessionId = subscribePath ? queryParam(req, "session") : undefined;
|
|
294
|
+
// No name: the session assigns a unique one (speaker-1, speaker-2, …).
|
|
295
|
+
const speaker = queryParam(req, "speaker") ?? "";
|
|
167
296
|
const send = (event) => {
|
|
297
|
+
record("events.jsonl", `${JSON.stringify({ at: Date.now(), ...event })}\n`);
|
|
168
298
|
if (client.readyState !== WebSocket.OPEN) {
|
|
169
299
|
return false;
|
|
170
300
|
}
|
|
171
301
|
client.send(JSON.stringify(event));
|
|
172
302
|
return true;
|
|
173
303
|
};
|
|
304
|
+
const events = (config) => {
|
|
305
|
+
const options = {
|
|
306
|
+
...config,
|
|
307
|
+
audio: queue,
|
|
308
|
+
output: {
|
|
309
|
+
...config.output,
|
|
310
|
+
languages: { current: () => liveLanguages },
|
|
311
|
+
},
|
|
312
|
+
};
|
|
313
|
+
if (!sessionId) {
|
|
314
|
+
return neurolink.transcribeStream(options);
|
|
315
|
+
}
|
|
316
|
+
// A session speaker: events go to the session's listeners and back to
|
|
317
|
+
// this socket. The session is held until this speaker's audio ends.
|
|
318
|
+
const session = acquire(sessionId);
|
|
319
|
+
const buffered = [];
|
|
320
|
+
let wake;
|
|
321
|
+
let finished = false;
|
|
322
|
+
let failure;
|
|
323
|
+
void session
|
|
324
|
+
.addSpeaker(speaker, options, (event) => {
|
|
325
|
+
buffered.push(event);
|
|
326
|
+
wake?.();
|
|
327
|
+
})
|
|
328
|
+
.catch((error) => {
|
|
329
|
+
failure = error;
|
|
330
|
+
})
|
|
331
|
+
.finally(() => {
|
|
332
|
+
finished = true;
|
|
333
|
+
release(sessionId);
|
|
334
|
+
wake?.();
|
|
335
|
+
});
|
|
336
|
+
return {
|
|
337
|
+
[Symbol.asyncIterator]: async function* () {
|
|
338
|
+
for (;;) {
|
|
339
|
+
const next = buffered.shift();
|
|
340
|
+
if (next) {
|
|
341
|
+
yield next;
|
|
342
|
+
continue;
|
|
343
|
+
}
|
|
344
|
+
if (finished) {
|
|
345
|
+
if (failure) {
|
|
346
|
+
throw failure;
|
|
347
|
+
}
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
await new Promise((resolve) => {
|
|
351
|
+
wake = resolve;
|
|
352
|
+
});
|
|
353
|
+
wake = undefined;
|
|
354
|
+
}
|
|
355
|
+
},
|
|
356
|
+
};
|
|
357
|
+
};
|
|
174
358
|
const run = async (config) => {
|
|
175
359
|
try {
|
|
176
|
-
for await (const event of
|
|
177
|
-
...config,
|
|
178
|
-
audio: queue,
|
|
179
|
-
})) {
|
|
360
|
+
for await (const event of events(config)) {
|
|
180
361
|
if (!send(event)) {
|
|
181
362
|
break; // The client is gone; stop pulling.
|
|
182
363
|
}
|
|
@@ -187,7 +368,9 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
187
368
|
}
|
|
188
369
|
catch (error) {
|
|
189
370
|
const message = error instanceof Error ? error.message : String(error);
|
|
190
|
-
logger.warn("[TranscribeWebSocket] stream failed", {
|
|
371
|
+
logger.warn("[TranscribeWebSocket] stream failed", {
|
|
372
|
+
error: message,
|
|
373
|
+
});
|
|
191
374
|
send({ type: "error", message, recoverable: false });
|
|
192
375
|
if (client.readyState === WebSocket.OPEN) {
|
|
193
376
|
client.close(1011, "transcription failed");
|
|
@@ -230,12 +413,18 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
230
413
|
return;
|
|
231
414
|
}
|
|
232
415
|
started = true;
|
|
233
|
-
|
|
416
|
+
const { mode, ...rest } = config.data;
|
|
417
|
+
fileMode = mode === "file";
|
|
418
|
+
liveLanguages = rest.output?.languages ?? [];
|
|
419
|
+
record("config.json", JSON.stringify(config.data, null, 1), false);
|
|
420
|
+
void run(fileMode ? fileModeStreamOptions(rest) : rest);
|
|
234
421
|
return;
|
|
235
422
|
}
|
|
236
423
|
if (isBinary) {
|
|
424
|
+
record("audio.pcm", frame);
|
|
237
425
|
queue.push(frame);
|
|
238
|
-
if (queue.queuedBytes >
|
|
426
|
+
if (queue.queuedBytes >
|
|
427
|
+
(fileMode ? MAX_QUEUED_BYTES_FILE : MAX_QUEUED_BYTES)) {
|
|
239
428
|
send({
|
|
240
429
|
type: "error",
|
|
241
430
|
message: "Audio is arriving faster than it can be transcribed",
|
|
@@ -246,7 +435,13 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
246
435
|
}
|
|
247
436
|
return;
|
|
248
437
|
}
|
|
249
|
-
//
|
|
438
|
+
// `output` changes the languages; any other text frame ends the
|
|
439
|
+
// audio ({"type":"end"}).
|
|
440
|
+
const control = TranscribeOutputControlSchema.safeParse(parseJson(frame));
|
|
441
|
+
if (control.success) {
|
|
442
|
+
liveLanguages = control.data.languages;
|
|
443
|
+
return;
|
|
444
|
+
}
|
|
250
445
|
queue.end();
|
|
251
446
|
});
|
|
252
447
|
client.on("close", () => queue.end());
|
|
@@ -258,6 +453,10 @@ export async function attachTranscribeWebSocket(server, neurolink, options = {})
|
|
|
258
453
|
for (const client of wss.clients) {
|
|
259
454
|
client.terminate();
|
|
260
455
|
}
|
|
456
|
+
for (const { session } of sessions.values()) {
|
|
457
|
+
session.close();
|
|
458
|
+
}
|
|
459
|
+
sessions.clear();
|
|
261
460
|
wss.close();
|
|
262
461
|
},
|
|
263
462
|
};
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { DEFAULT_BASE_URL } from "../../../../constants/langfuse.js";
|
|
1
2
|
import { OTEL_LOG_PREFIX } from "../../../../constants/telemetry.js";
|
|
2
3
|
import { LangfuseExporter } from "../../../../observability/exporters/langfuseExporter.js";
|
|
3
4
|
import { logger } from "../../../../utils/logger.js";
|
|
@@ -90,11 +91,15 @@ export function createLangfuseExporter(config) {
|
|
|
90
91
|
logger.warn(`${OTEL_LOG_PREFIX} Langfuse is enabled but publicKey and secretKey are not both set in config or LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY; not exporting to Langfuse`);
|
|
91
92
|
return null;
|
|
92
93
|
}
|
|
94
|
+
const baseUrl = options.baseUrl || process.env.LANGFUSE_BASE_URL || undefined;
|
|
95
|
+
if (baseUrl === undefined) {
|
|
96
|
+
logger.warn(`${OTEL_LOG_PREFIX} Langfuse base URL is not set in observability.langfuse.baseUrl or LANGFUSE_BASE_URL; exporting to ${DEFAULT_BASE_URL}`);
|
|
97
|
+
}
|
|
93
98
|
const exporter = new LangfuseExporter({
|
|
94
99
|
enabled: true,
|
|
95
100
|
publicKey,
|
|
96
101
|
secretKey,
|
|
97
|
-
baseUrl
|
|
102
|
+
baseUrl,
|
|
98
103
|
// The released build labelled Langfuse traces from the option and
|
|
99
104
|
// LANGFUSE_TRACING_ENVIRONMENT / LANGFUSE_RELEASE only (the exporter reads
|
|
100
105
|
// those two itself); the longer chain feeds the OTLP resource alone.
|
|
@@ -137,7 +137,7 @@ export declare function recordsSpanContent(): boolean;
|
|
|
137
137
|
*/
|
|
138
138
|
export declare function isFinishedAttribute(value: string): boolean;
|
|
139
139
|
/**
|
|
140
|
-
* @description Returns a copy of a value with credentials masked: values under keys such as `access_token`, `password`, `authorization` or `set-cookie`, bearer tokens, and `name=value` credentials inside text. Purely numeric values after a credential-looking name (e.g. `max_tokens: 4096`) are left alone.
|
|
140
|
+
* @description Returns a copy of a value with credentials masked: values under keys such as `access_token`, `password`, `authorization` or `set-cookie`, bearer tokens, and `name=value` credentials inside text. Purely numeric values after a credential-looking name (e.g. `max_tokens: 4096`) are left alone. With `NEUROLINK_MASK_CREDENTIAL_ENTRIES=true` it also masks the value of a `{ name, value }` or `{ key, value }` entry whose name is such a key (a setting such as `max_tokens`, `password_policy` or `secret_version` is not, and the value is not consulted, so a numeric PIN is masked) and a credential inside an object key; unset, it does neither, as in the released build.
|
|
141
141
|
* @param value {JsonValue | object} - Content to redact; objects and arrays are walked recursively.
|
|
142
142
|
* @returns {JsonValue | object} A redacted copy, serializing exactly as the input when nothing in it is sensitive.
|
|
143
143
|
* @throws {Error} When the value cannot be walked; tracing callers use `spanContentAttribute`, which turns that into an omitted value.
|
|
@@ -227,7 +227,74 @@ function redactText(text) {
|
|
|
227
227
|
});
|
|
228
228
|
return redacted;
|
|
229
229
|
}
|
|
230
|
-
|
|
230
|
+
// Opt-in stricter masking (`NEUROLINK_MASK_CREDENTIAL_ENTRIES=true`): the value
|
|
231
|
+
// of a `{ name, value }` or `{ key, value }` entry whose name is a credential,
|
|
232
|
+
// and a credential inside an object key. The released masker reads keys and
|
|
233
|
+
// text only, so with the variable unset the output is exactly what it was.
|
|
234
|
+
function masksCredentialEntries() {
|
|
235
|
+
return (process.env.NEUROLINK_MASK_CREDENTIAL_ENTRIES?.trim().toLowerCase() ===
|
|
236
|
+
"true");
|
|
237
|
+
}
|
|
238
|
+
// A name that says how many, how long or what kind of thing rather than
|
|
239
|
+
// holding the credential (`max_tokens`, `password_policy`, `credential_type`,
|
|
240
|
+
// `secret_version`, `password_min_length`). A name that merely ends in `tokens`
|
|
241
|
+
// is not one: `access_tokens` and `refresh_tokens` are credentials, so only a
|
|
242
|
+
// bare `tokens` and the count-like names of a model's token budget and usage
|
|
243
|
+
// (`max_tokens`, `prompt_tokens`, `cache_read_input_tokens`, ...) are exempt.
|
|
244
|
+
// Decided on the name's segments, not with a pattern: a key comes from content
|
|
245
|
+
// and may be very long, so the check must stay linear in its length.
|
|
246
|
+
const MEASUREMENT_SUFFIXES = new Set([
|
|
247
|
+
"policy",
|
|
248
|
+
"type",
|
|
249
|
+
"algorithm",
|
|
250
|
+
"count",
|
|
251
|
+
"limit",
|
|
252
|
+
"version",
|
|
253
|
+
"length",
|
|
254
|
+
]);
|
|
255
|
+
const TOKEN_COUNT_PREFIXES = new Set([
|
|
256
|
+
"max",
|
|
257
|
+
"min",
|
|
258
|
+
"prompt",
|
|
259
|
+
"completion",
|
|
260
|
+
"input",
|
|
261
|
+
"output",
|
|
262
|
+
"total",
|
|
263
|
+
"cache",
|
|
264
|
+
"cached",
|
|
265
|
+
"reasoning",
|
|
266
|
+
"thinking",
|
|
267
|
+
"budget",
|
|
268
|
+
"context",
|
|
269
|
+
]);
|
|
270
|
+
function isMeasurementEntryName(name) {
|
|
271
|
+
const segments = snakeCased(name).split("_");
|
|
272
|
+
const last = segments[segments.length - 1];
|
|
273
|
+
if (segments.length === 1) {
|
|
274
|
+
return last === "tokens";
|
|
275
|
+
}
|
|
276
|
+
return (MEASUREMENT_SUFFIXES.has(last) ||
|
|
277
|
+
(last === "tokens" && TOKEN_COUNT_PREFIXES.has(segments[0])));
|
|
278
|
+
}
|
|
279
|
+
function snakeCased(name) {
|
|
280
|
+
return name
|
|
281
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
282
|
+
.replace(/[\s.-]+/g, "_")
|
|
283
|
+
.toLowerCase();
|
|
284
|
+
}
|
|
285
|
+
// Headers, environment variables and form fields are often lists of
|
|
286
|
+
// `{ name, value }` or `{ key, value }` entries; when the name is a credential
|
|
287
|
+
// the value is one, exactly as if it were stored under that key, whatever the
|
|
288
|
+
// value looks like (a numeric PIN is a credential). Only the name decides: a
|
|
289
|
+
// setting that merely mentions a credential is not one.
|
|
290
|
+
function credentialEntryName(node) {
|
|
291
|
+
const name = "name" in node ? node.name : "key" in node ? node.key : undefined;
|
|
292
|
+
if (typeof name !== "string" || !("value" in node) || !isSensitiveKey(name)) {
|
|
293
|
+
return undefined;
|
|
294
|
+
}
|
|
295
|
+
return isMeasurementEntryName(name) ? undefined : name;
|
|
296
|
+
}
|
|
297
|
+
function redactNode(value, ancestors, strict) {
|
|
231
298
|
if (typeof value === "string") {
|
|
232
299
|
return redactText(describeDataUrls(value));
|
|
233
300
|
}
|
|
@@ -243,25 +310,34 @@ function redactNode(value, ancestors) {
|
|
|
243
310
|
ancestors.add(value);
|
|
244
311
|
try {
|
|
245
312
|
if ("toJSON" in value && typeof value.toJSON === "function") {
|
|
246
|
-
return redactNode(value.toJSON(), ancestors);
|
|
313
|
+
return redactNode(value.toJSON(), ancestors, strict);
|
|
247
314
|
}
|
|
248
315
|
if (Array.isArray(value)) {
|
|
249
|
-
return value.map((entry) => redactNode(entry, ancestors));
|
|
316
|
+
return value.map((entry) => redactNode(entry, ancestors, strict));
|
|
250
317
|
}
|
|
251
318
|
// No prototype, so a key named __proto__ is data rather than an assignment.
|
|
252
319
|
const redacted = Object.create(null);
|
|
253
320
|
const mimeType = declaredMimeType(value);
|
|
321
|
+
const entryName = strict ? credentialEntryName(value) : undefined;
|
|
254
322
|
for (const [key, entry] of Object.entries(value)) {
|
|
255
|
-
|
|
323
|
+
// Opt-in: a key can carry a credential too, such as a URL with an
|
|
324
|
+
// api_key parameter used as a map key.
|
|
325
|
+
const safeKey = strict ? redactText(key) : key;
|
|
326
|
+
const sensitiveName = key === "value" && entryName !== undefined
|
|
327
|
+
? entryName
|
|
328
|
+
: isSensitiveKey(key)
|
|
329
|
+
? key
|
|
330
|
+
: undefined;
|
|
331
|
+
if (sensitiveName !== undefined &&
|
|
256
332
|
(typeof entry === "string" ||
|
|
257
333
|
(typeof entry === "object" && entry !== null))) {
|
|
258
|
-
redacted[
|
|
334
|
+
redacted[safeKey] = redactSensitiveEntry(sensitiveName, entry);
|
|
259
335
|
continue;
|
|
260
336
|
}
|
|
261
337
|
const placeholder = isBinaryKey(key)
|
|
262
338
|
? binaryPlaceholder(entry, mimeType)
|
|
263
339
|
: undefined;
|
|
264
|
-
redacted[
|
|
340
|
+
redacted[safeKey] = placeholder ?? redactNode(entry, ancestors, strict);
|
|
265
341
|
}
|
|
266
342
|
return redacted;
|
|
267
343
|
}
|
|
@@ -270,13 +346,13 @@ function redactNode(value, ancestors) {
|
|
|
270
346
|
}
|
|
271
347
|
}
|
|
272
348
|
/**
|
|
273
|
-
* @description Returns a copy of a value with credentials masked: values under keys such as `access_token`, `password`, `authorization` or `set-cookie`, bearer tokens, and `name=value` credentials inside text. Purely numeric values after a credential-looking name (e.g. `max_tokens: 4096`) are left alone.
|
|
349
|
+
* @description Returns a copy of a value with credentials masked: values under keys such as `access_token`, `password`, `authorization` or `set-cookie`, bearer tokens, and `name=value` credentials inside text. Purely numeric values after a credential-looking name (e.g. `max_tokens: 4096`) are left alone. With `NEUROLINK_MASK_CREDENTIAL_ENTRIES=true` it also masks the value of a `{ name, value }` or `{ key, value }` entry whose name is such a key (a setting such as `max_tokens`, `password_policy` or `secret_version` is not, and the value is not consulted, so a numeric PIN is masked) and a credential inside an object key; unset, it does neither, as in the released build.
|
|
274
350
|
* @param value {JsonValue | object} - Content to redact; objects and arrays are walked recursively.
|
|
275
351
|
* @returns {JsonValue | object} A redacted copy, serializing exactly as the input when nothing in it is sensitive.
|
|
276
352
|
* @throws {Error} When the value cannot be walked; tracing callers use `spanContentAttribute`, which turns that into an omitted value.
|
|
277
353
|
*/
|
|
278
354
|
export function redactSpanContent(value) {
|
|
279
|
-
return redactNode(value, new WeakSet());
|
|
355
|
+
return redactNode(value, new WeakSet(), masksCredentialEntries());
|
|
280
356
|
}
|
|
281
357
|
function redactSafely(value) {
|
|
282
358
|
try {
|
package/dist/types/cli.d.ts
CHANGED
|
@@ -1142,6 +1142,8 @@ export type ServeCommandArgs = {
|
|
|
1142
1142
|
cors?: boolean;
|
|
1143
1143
|
/** Browser origins allowed by CORS (`--cors-origin`, repeatable or comma-separated). */
|
|
1144
1144
|
corsOrigin?: string[];
|
|
1145
|
+
/** Largest raw upload (multipart / audio bodies) the server accepts, e.g. "64mb" (`--max-upload`). */
|
|
1146
|
+
maxUpload?: string;
|
|
1145
1147
|
rateLimit?: number;
|
|
1146
1148
|
swagger?: boolean;
|
|
1147
1149
|
config?: string;
|
|
@@ -1342,9 +1344,13 @@ export type CliTranscribeArgs = {
|
|
|
1342
1344
|
rewrite?: boolean;
|
|
1343
1345
|
transliterate?: "latin" | "native";
|
|
1344
1346
|
secondOpinion?: string;
|
|
1347
|
+
/** `--to`: output languages (comma-separated BCP-47 tags). */
|
|
1348
|
+
to?: string | string[];
|
|
1345
1349
|
fallback?: string;
|
|
1346
1350
|
context?: string;
|
|
1347
1351
|
stream?: boolean;
|
|
1352
|
+
/** With `--stream`: which speech detector finds utterances. */
|
|
1353
|
+
vad?: "auto" | "silero" | "energy";
|
|
1348
1354
|
format?: "text" | "json" | "srt" | "vtt";
|
|
1349
1355
|
timeout?: number;
|
|
1350
1356
|
quiet?: boolean;
|
package/dist/types/exporter.d.ts
CHANGED
|
@@ -306,7 +306,7 @@ export type SpanAttributeValue = SpanAttributes[string];
|
|
|
306
306
|
export type OtlpAnyValue = {
|
|
307
307
|
boolValue: boolean;
|
|
308
308
|
} | {
|
|
309
|
-
intValue:
|
|
309
|
+
intValue: number;
|
|
310
310
|
} | {
|
|
311
311
|
doubleValue: number;
|
|
312
312
|
} | {
|
|
@@ -383,6 +383,7 @@ export type OtlpTraceRequest = {
|
|
|
383
383
|
export type LangfuseDeliveryReport = {
|
|
384
384
|
delivered: number;
|
|
385
385
|
rejected: number;
|
|
386
|
+
refused: number;
|
|
386
387
|
droppedIds: string[];
|
|
387
388
|
error?: string;
|
|
388
389
|
};
|
|
@@ -392,6 +393,7 @@ export type LangfusePostOutcome = {
|
|
|
392
393
|
rejected: number;
|
|
393
394
|
} | {
|
|
394
395
|
accepted: false;
|
|
396
|
+
refused?: true;
|
|
395
397
|
};
|
|
396
398
|
/** @description What the Langfuse exporter adds to, masks in and removes from span attributes before sending. */
|
|
397
399
|
export type LangfuseAttributePolicy = Pick<LangfuseExporterConfig, "environment" | "release" | "mask"> & {
|