@fonoster/apiserver 0.25.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/applications/createGetFnUtil.d.ts +1 -1
- package/dist/applications/validation/assistantWithoutApiKeySchema.d.ts +18 -0
- package/dist/applications/validation/createValidationSchema.d.ts +44 -2
- package/dist/voice/VoiceDispatcher.js +1 -0
- package/dist/voice/client/AudioSocketHandler.d.ts +3 -0
- package/dist/voice/client/AudioSocketHandler.js +3 -1
- package/dist/voice/client/VoiceClientImpl.d.ts +8 -0
- package/dist/voice/client/VoiceClientImpl.js +36 -1
- package/dist/voice/filters/AudioFilterFactory.d.ts +35 -0
- package/dist/voice/filters/AudioFilterFactory.js +94 -0
- package/dist/voice/filters/aiCoustics/createAiCousticsFilter.d.ts +64 -0
- package/dist/voice/filters/aiCoustics/createAiCousticsFilter.js +252 -0
- package/dist/voice/filters/aiCoustics/types.d.ts +45 -0
- package/dist/voice/filters/aiCoustics/types.js +29 -0
- package/dist/voice/filters/createAudioFilterChain.d.ts +9 -0
- package/dist/voice/filters/createAudioFilterChain.js +171 -0
- package/dist/voice/filters/createAudioFilterSession.d.ts +15 -0
- package/dist/voice/filters/createAudioFilterSession.js +114 -0
- package/dist/voice/filters/index.d.ts +25 -0
- package/dist/voice/filters/index.js +41 -0
- package/dist/voice/filters/pcm.d.ts +21 -0
- package/dist/voice/filters/pcm.js +37 -0
- package/dist/voice/filters/types.d.ts +61 -0
- package/dist/voice/filters/types.js +26 -0
- package/dist/voice/handlers/createSetAudioFiltersHandler.d.ts +11 -0
- package/dist/voice/handlers/createSetAudioFiltersHandler.js +74 -0
- package/dist/voice/handlers/index.d.ts +1 -0
- package/dist/voice/handlers/index.js +1 -0
- package/dist/voice/stt/Deepgram.d.ts +26 -2
- package/dist/voice/stt/Deepgram.js +21 -6
- package/dist/voice/stt/types.d.ts +5 -3
- package/dist/voice/stt/types.js +5 -1
- package/dist/voice/transcribeOnConnection.d.ts +14 -1
- package/dist/voice/transcribeOnConnection.js +67 -2
- package/dist/voice/types/voice.d.ts +2 -0
- package/package.json +13 -10
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.aiCousticsPublicOptionsSchema = exports.aiCousticsOptionsSchema = exports.AI_COUSTICS_NAME = void 0;
|
|
13
|
+
exports.createAiCousticsFilter = createAiCousticsFilter;
|
|
14
|
+
/**
|
|
15
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
16
|
+
* http://github.com/fonoster/fonoster
|
|
17
|
+
*
|
|
18
|
+
* This file is part of Fonoster
|
|
19
|
+
*
|
|
20
|
+
* Licensed under the MIT License (the "License");
|
|
21
|
+
* you may not use this file except in compliance with
|
|
22
|
+
* the License. You may obtain a copy of the License at
|
|
23
|
+
*
|
|
24
|
+
* https://opensource.org/licenses/MIT
|
|
25
|
+
*
|
|
26
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
27
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
28
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
29
|
+
* See the License for the specific language governing permissions and
|
|
30
|
+
* limitations under the License.
|
|
31
|
+
*/
|
|
32
|
+
const fs_1 = require("fs");
|
|
33
|
+
const path_1 = require("path");
|
|
34
|
+
const logger_1 = require("@fonoster/logger");
|
|
35
|
+
const zod_1 = require("zod");
|
|
36
|
+
const types_1 = require("../types");
|
|
37
|
+
const types_2 = require("./types");
|
|
38
|
+
const AI_COUSTICS_NAME = "aiCoustics";
|
|
39
|
+
exports.AI_COUSTICS_NAME = AI_COUSTICS_NAME;
|
|
40
|
+
const PACKAGE = "@ai-coustics/aic-sdk";
|
|
41
|
+
const FULL_SCALE = 32767;
|
|
42
|
+
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
43
|
+
const aiCousticsOptionsSchema = zod_1.z.object({
|
|
44
|
+
// Voice Focus suppresses competing speech; Multi Speaker keeps every voice
|
|
45
|
+
model: zod_1.z.string().default("quail-vf-2.2-s-16khz"),
|
|
46
|
+
modelDir: zod_1.z.string().default("./models"),
|
|
47
|
+
// Exact .aicmodel file to load; overrides model/modelDir lookup
|
|
48
|
+
modelFile: zod_1.z.string().optional(),
|
|
49
|
+
// Download the model on first use when it isn't on disk yet
|
|
50
|
+
autoDownload: zod_1.z.boolean().default(true),
|
|
51
|
+
// Higher values suppress more, competing speech included
|
|
52
|
+
enhancementLevel: zod_1.z.number().min(0).max(1).default(1),
|
|
53
|
+
// Keeps the delay but stops enhancing; useful for A/B tests
|
|
54
|
+
bypass: zod_1.z.boolean().default(false),
|
|
55
|
+
// Name of the environment variable holding the license key. The key itself
|
|
56
|
+
// never goes in application config.
|
|
57
|
+
licenseEnv: zod_1.z.string().default("AIC_SDK_LICENSE")
|
|
58
|
+
});
|
|
59
|
+
exports.aiCousticsOptionsSchema = aiCousticsOptionsSchema;
|
|
60
|
+
// What a voice application may set. modelDir, modelFile, autoDownload and
|
|
61
|
+
// licenseEnv are deliberately absent: they read the media server's filesystem
|
|
62
|
+
// and environment, and applications are untrusted input. The model name is
|
|
63
|
+
// restricted to the vendor's id shape so it cannot walk the filesystem.
|
|
64
|
+
const aiCousticsPublicOptionsSchema = aiCousticsOptionsSchema
|
|
65
|
+
.pick({ enhancementLevel: true, bypass: true })
|
|
66
|
+
.extend({
|
|
67
|
+
model: zod_1.z
|
|
68
|
+
.string()
|
|
69
|
+
.regex(/^[a-z0-9]+(-[a-z0-9.]+)*$/, {
|
|
70
|
+
message: "Invalid model name"
|
|
71
|
+
})
|
|
72
|
+
.default("quail-vf-2.2-s-16khz")
|
|
73
|
+
})
|
|
74
|
+
// Reject anything else outright, so an application that sets a server-only
|
|
75
|
+
// option is told rather than having it quietly ignored
|
|
76
|
+
.strict();
|
|
77
|
+
exports.aiCousticsPublicOptionsSchema = aiCousticsPublicOptionsSchema;
|
|
78
|
+
function loadSdk() {
|
|
79
|
+
try {
|
|
80
|
+
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
81
|
+
return require(PACKAGE);
|
|
82
|
+
}
|
|
83
|
+
catch (_a) {
|
|
84
|
+
throw new Error(`${PACKAGE} is not installed. Run "npm install ${PACKAGE}" to use the ${AI_COUSTICS_NAME} filter.`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Downloaded files are named after the model with underscores, plus a content
|
|
88
|
+
// hash and model version, e.g. quail_vf_2_2_s_16khz_gf70x7zf_v14.aicmodel
|
|
89
|
+
function findModelFile(modelDir, model) {
|
|
90
|
+
if (!(0, fs_1.existsSync)(modelDir)) {
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
const prefix = model.replace(/[-.]/g, "_");
|
|
94
|
+
const match = (0, fs_1.readdirSync)(modelDir)
|
|
95
|
+
.filter((file) => file.startsWith(prefix) && file.endsWith(".aicmodel"))
|
|
96
|
+
.sort()
|
|
97
|
+
.pop();
|
|
98
|
+
return match ? (0, path_1.join)(modelDir, match) : undefined;
|
|
99
|
+
}
|
|
100
|
+
function toFloat32(frame) {
|
|
101
|
+
return Float32Array.from(frame, (sample) => sample / FULL_SCALE);
|
|
102
|
+
}
|
|
103
|
+
function toInt16(samples, length) {
|
|
104
|
+
return Int16Array.from({ length }, (_, i) => {
|
|
105
|
+
var _a;
|
|
106
|
+
const value = Math.round(((_a = samples[i]) !== null && _a !== void 0 ? _a : 0) * FULL_SCALE);
|
|
107
|
+
return Math.max(-32768, Math.min(32767, value));
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* ai-coustics speech enhancement (Quail Voice Focus by default), which keeps
|
|
112
|
+
* the primary speaker and suppresses noise and competing voices.
|
|
113
|
+
*
|
|
114
|
+
* The SDK works on fixed blocks of Float32 samples at the model's optimal
|
|
115
|
+
* size, so frames are re-chunked here and the output is delayed by one block.
|
|
116
|
+
* Processing runs on libuv worker threads, off the event loop.
|
|
117
|
+
*
|
|
118
|
+
* Until the model is loaded, audio passes through untouched rather than
|
|
119
|
+
* stalling the call.
|
|
120
|
+
*/
|
|
121
|
+
function createAiCousticsFilter(input = {}, deps = { loadSdk }) {
|
|
122
|
+
const options = aiCousticsOptionsSchema.parse(input);
|
|
123
|
+
const license = process.env[options.licenseEnv];
|
|
124
|
+
if (!license) {
|
|
125
|
+
throw new Error(`The ${AI_COUSTICS_NAME} filter needs a license key in ${options.licenseEnv}. Get one at https://developers.ai-coustics.com`);
|
|
126
|
+
}
|
|
127
|
+
let processor = null;
|
|
128
|
+
// close() can land while the model is still loading; without this the
|
|
129
|
+
// processor built afterwards would never be disposed
|
|
130
|
+
let closed = false;
|
|
131
|
+
let blockSize = 0;
|
|
132
|
+
let delaySamples = 0;
|
|
133
|
+
let pendingInput = new Float32Array(0);
|
|
134
|
+
let pendingOutput = new Float32Array(0);
|
|
135
|
+
let framesPassedThrough = 0;
|
|
136
|
+
let framesProcessed = 0;
|
|
137
|
+
let lastInputDb = -90;
|
|
138
|
+
let lastOutputDb = -90;
|
|
139
|
+
const append = (buffer, samples) => {
|
|
140
|
+
const next = new Float32Array(buffer.length + samples.length);
|
|
141
|
+
next.set(buffer);
|
|
142
|
+
next.set(samples, buffer.length);
|
|
143
|
+
return next;
|
|
144
|
+
};
|
|
145
|
+
// Runs on every frame, so it walks the samples in place rather than
|
|
146
|
+
// allocating a copy per call
|
|
147
|
+
const db = (samples, scale) => {
|
|
148
|
+
if (samples.length === 0) {
|
|
149
|
+
return -90;
|
|
150
|
+
}
|
|
151
|
+
const sum = samples.reduce((acc, s) => acc + (s / scale) * (s / scale), 0);
|
|
152
|
+
const rms = Math.sqrt(sum / samples.length);
|
|
153
|
+
return rms > 0 ? Math.max(-90, 20 * Math.log10(rms)) : -90;
|
|
154
|
+
};
|
|
155
|
+
const start = (() => __awaiter(this, void 0, void 0, function* () {
|
|
156
|
+
var _a;
|
|
157
|
+
const sdk = deps.loadSdk();
|
|
158
|
+
const existing = (_a = options.modelFile) !== null && _a !== void 0 ? _a : findModelFile(options.modelDir, options.model);
|
|
159
|
+
if (!existing && !options.autoDownload) {
|
|
160
|
+
throw new Error(`Model ${options.model} not found in ${options.modelDir}`);
|
|
161
|
+
}
|
|
162
|
+
if (!existing) {
|
|
163
|
+
logger.verbose("downloading ai-coustics model", {
|
|
164
|
+
model: options.model,
|
|
165
|
+
modelDir: options.modelDir
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
const file = existing !== null && existing !== void 0 ? existing : (yield sdk.Model.download(options.model, options.modelDir));
|
|
169
|
+
const model = sdk.Model.fromFile(file);
|
|
170
|
+
const optimalRate = model.getOptimalSampleRate();
|
|
171
|
+
if (optimalRate !== types_1.SAMPLE_RATE) {
|
|
172
|
+
throw new Error(`Model ${options.model} expects ${optimalRate} Hz; call audio is ${types_1.SAMPLE_RATE} Hz`);
|
|
173
|
+
}
|
|
174
|
+
blockSize = model.getOptimalBlockSize(types_1.SAMPLE_RATE);
|
|
175
|
+
processor = yield new sdk.ProcessorAsync(model, license).withConfig(types_1.SAMPLE_RATE, blockSize);
|
|
176
|
+
if (closed) {
|
|
177
|
+
processor.dispose();
|
|
178
|
+
processor = null;
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
const context = yield processor.getContext();
|
|
182
|
+
if (typeof (context === null || context === void 0 ? void 0 : context.setParameter) !== "function") {
|
|
183
|
+
throw new Error(`${PACKAGE} returned an unexpected processor context; the SDK version may be incompatible`);
|
|
184
|
+
}
|
|
185
|
+
context.setParameter(types_2.ENHANCEMENT_LEVEL_PARAMETER, options.enhancementLevel);
|
|
186
|
+
context.setParameter(types_2.BYPASS_PARAMETER, options.bypass ? 1 : 0);
|
|
187
|
+
delaySamples = context.getAudioDelay();
|
|
188
|
+
// One block of silence keeps output aligned with input from here on
|
|
189
|
+
pendingOutput = new Float32Array(blockSize);
|
|
190
|
+
logger.verbose("ai-coustics filter ready", {
|
|
191
|
+
model: options.model,
|
|
192
|
+
file,
|
|
193
|
+
blockSize,
|
|
194
|
+
delaySamples
|
|
195
|
+
});
|
|
196
|
+
}))();
|
|
197
|
+
// Surfaced by ready(); the chain disables the filter if it rejects
|
|
198
|
+
start.catch(() => undefined);
|
|
199
|
+
const processBlocks = () => __awaiter(this, void 0, void 0, function* () {
|
|
200
|
+
if (pendingInput.length < blockSize) {
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
const block = pendingInput.slice(0, blockSize);
|
|
204
|
+
pendingInput = pendingInput.slice(blockSize);
|
|
205
|
+
pendingOutput = append(pendingOutput, yield processor.process(block));
|
|
206
|
+
return processBlocks();
|
|
207
|
+
});
|
|
208
|
+
return {
|
|
209
|
+
name: AI_COUSTICS_NAME,
|
|
210
|
+
get delayMs() {
|
|
211
|
+
return delaySamples / (types_1.SAMPLE_RATE / 1000);
|
|
212
|
+
},
|
|
213
|
+
ready: () => start,
|
|
214
|
+
process(frame) {
|
|
215
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
216
|
+
if (!processor) {
|
|
217
|
+
framesPassedThrough++;
|
|
218
|
+
return frame;
|
|
219
|
+
}
|
|
220
|
+
lastInputDb = db(frame, FULL_SCALE);
|
|
221
|
+
pendingInput = append(pendingInput, toFloat32(frame));
|
|
222
|
+
yield processBlocks();
|
|
223
|
+
const output = toInt16(pendingOutput, frame.length);
|
|
224
|
+
pendingOutput = pendingOutput.slice(frame.length);
|
|
225
|
+
lastOutputDb = db(output, FULL_SCALE);
|
|
226
|
+
framesProcessed++;
|
|
227
|
+
return output;
|
|
228
|
+
});
|
|
229
|
+
},
|
|
230
|
+
getStats() {
|
|
231
|
+
return {
|
|
232
|
+
framesProcessed,
|
|
233
|
+
framesPassedThrough,
|
|
234
|
+
blockSize,
|
|
235
|
+
delaySamples,
|
|
236
|
+
enhancementLevel: options.enhancementLevel
|
|
237
|
+
};
|
|
238
|
+
},
|
|
239
|
+
inspect() {
|
|
240
|
+
return {
|
|
241
|
+
inputDb: lastInputDb,
|
|
242
|
+
outputDb: lastOutputDb,
|
|
243
|
+
reductionDb: lastOutputDb - lastInputDb
|
|
244
|
+
};
|
|
245
|
+
},
|
|
246
|
+
close() {
|
|
247
|
+
closed = true;
|
|
248
|
+
processor === null || processor === void 0 ? void 0 : processor.dispose();
|
|
249
|
+
processor = null;
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
declare const BYPASS_PARAMETER = 0;
|
|
20
|
+
declare const ENHANCEMENT_LEVEL_PARAMETER = 1;
|
|
21
|
+
type AiCousticsContext = {
|
|
22
|
+
setParameter(parameter: number, value: number): void;
|
|
23
|
+
getParameter(parameter: number): number;
|
|
24
|
+
getAudioDelay(): number;
|
|
25
|
+
reset(): void;
|
|
26
|
+
};
|
|
27
|
+
type AiCousticsProcessor = {
|
|
28
|
+
withConfig(sampleRate: number, blockSize: number, variableBlockSize?: boolean): Promise<AiCousticsProcessor>;
|
|
29
|
+
process(audio: Float32Array): Promise<Float32Array>;
|
|
30
|
+
getContext(): Promise<AiCousticsContext>;
|
|
31
|
+
dispose(): void;
|
|
32
|
+
};
|
|
33
|
+
type AiCousticsModel = {
|
|
34
|
+
getOptimalSampleRate(): number;
|
|
35
|
+
getOptimalBlockSize(sampleRate: number): number;
|
|
36
|
+
dispose?(): void;
|
|
37
|
+
};
|
|
38
|
+
type AiCousticsSdk = {
|
|
39
|
+
Model: {
|
|
40
|
+
fromFile(path: string): AiCousticsModel;
|
|
41
|
+
download(modelId: string, downloadDir: string): Promise<string>;
|
|
42
|
+
};
|
|
43
|
+
ProcessorAsync: new (model: AiCousticsModel, licenseKey: string) => AiCousticsProcessor;
|
|
44
|
+
};
|
|
45
|
+
export { AiCousticsContext, AiCousticsModel, AiCousticsProcessor, AiCousticsSdk, BYPASS_PARAMETER, ENHANCEMENT_LEVEL_PARAMETER };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
4
|
+
* http://github.com/fonoster/fonoster
|
|
5
|
+
*
|
|
6
|
+
* This file is part of Fonoster
|
|
7
|
+
*
|
|
8
|
+
* Licensed under the MIT License (the "License");
|
|
9
|
+
* you may not use this file except in compliance with
|
|
10
|
+
* the License. You may obtain a copy of the License at
|
|
11
|
+
*
|
|
12
|
+
* https://opensource.org/licenses/MIT
|
|
13
|
+
*
|
|
14
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
15
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
16
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
17
|
+
* See the License for the specific language governing permissions and
|
|
18
|
+
* limitations under the License.
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.ENHANCEMENT_LEVEL_PARAMETER = exports.BYPASS_PARAMETER = void 0;
|
|
22
|
+
// The slice of @ai-coustics/aic-sdk this filter uses. Declared here so the
|
|
23
|
+
// filter can be tested with a fake SDK, and so the real package stays an
|
|
24
|
+
// optional dependency.
|
|
25
|
+
// Values of the SDK's ProcessorParameter enum
|
|
26
|
+
const BYPASS_PARAMETER = 0;
|
|
27
|
+
exports.BYPASS_PARAMETER = BYPASS_PARAMETER;
|
|
28
|
+
const ENHANCEMENT_LEVEL_PARAMETER = 1;
|
|
29
|
+
exports.ENHANCEMENT_LEVEL_PARAMETER = ENHANCEMENT_LEVEL_PARAMETER;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { AudioFilter, AudioFilterChain } from "./types";
|
|
2
|
+
/**
|
|
3
|
+
* Runs audio frames through filters in order. The chain is fail-open: a
|
|
4
|
+
* filter that throws, rejects, or changes the frame length is disabled for
|
|
5
|
+
* the rest of the stream and its input is passed through, so caller audio is
|
|
6
|
+
* never dropped.
|
|
7
|
+
*/
|
|
8
|
+
declare function createAudioFilterChain(filters: AudioFilter[]): AudioFilterChain;
|
|
9
|
+
export { createAudioFilterChain };
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.createAudioFilterChain = createAudioFilterChain;
|
|
13
|
+
/**
|
|
14
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
15
|
+
* http://github.com/fonoster/fonoster
|
|
16
|
+
*
|
|
17
|
+
* This file is part of Fonoster
|
|
18
|
+
*
|
|
19
|
+
* Licensed under the MIT License (the "License");
|
|
20
|
+
* you may not use this file except in compliance with
|
|
21
|
+
* the License. You may obtain a copy of the License at
|
|
22
|
+
*
|
|
23
|
+
* https://opensource.org/licenses/MIT
|
|
24
|
+
*
|
|
25
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
26
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
27
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
28
|
+
* See the License for the specific language governing permissions and
|
|
29
|
+
* limitations under the License.
|
|
30
|
+
*/
|
|
31
|
+
const perf_hooks_1 = require("perf_hooks");
|
|
32
|
+
const logger_1 = require("@fonoster/logger");
|
|
33
|
+
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
34
|
+
// Keeps timing memory bounded on long calls
|
|
35
|
+
const MAX_TIMING_SAMPLES = 10000;
|
|
36
|
+
function createTimings() {
|
|
37
|
+
const samples = [];
|
|
38
|
+
let frames = 0;
|
|
39
|
+
return {
|
|
40
|
+
add(ms) {
|
|
41
|
+
frames++;
|
|
42
|
+
if (samples.length < MAX_TIMING_SAMPLES) {
|
|
43
|
+
samples.push(ms);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
samples[frames % MAX_TIMING_SAMPLES] = ms;
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
summary() {
|
|
50
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
51
|
+
const at = (p) => sorted.length === 0
|
|
52
|
+
? 0
|
|
53
|
+
: sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))];
|
|
54
|
+
return {
|
|
55
|
+
frames,
|
|
56
|
+
p50Ms: at(0.5),
|
|
57
|
+
p95Ms: at(0.95),
|
|
58
|
+
maxMs: sorted.length === 0 ? 0 : sorted[sorted.length - 1]
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Runs audio frames through filters in order. The chain is fail-open: a
|
|
65
|
+
* filter that throws, rejects, or changes the frame length is disabled for
|
|
66
|
+
* the rest of the stream and its input is passed through, so caller audio is
|
|
67
|
+
* never dropped.
|
|
68
|
+
*/
|
|
69
|
+
function createAudioFilterChain(filters) {
|
|
70
|
+
const slots = filters.map((filter) => ({
|
|
71
|
+
filter,
|
|
72
|
+
timings: createTimings(),
|
|
73
|
+
failures: 0,
|
|
74
|
+
disabled: false,
|
|
75
|
+
error: null
|
|
76
|
+
}));
|
|
77
|
+
const totalTimings = createTimings();
|
|
78
|
+
// Serializes frames so output order matches input order even when callers
|
|
79
|
+
// don't await or a filter is async
|
|
80
|
+
let tail = Promise.resolve();
|
|
81
|
+
const disable = (slot, reason) => {
|
|
82
|
+
slot.failures++;
|
|
83
|
+
slot.disabled = true;
|
|
84
|
+
slot.error = reason;
|
|
85
|
+
logger.warn("audio filter disabled for the rest of the stream", {
|
|
86
|
+
filter: slot.filter.name,
|
|
87
|
+
reason
|
|
88
|
+
});
|
|
89
|
+
};
|
|
90
|
+
const runSlot = (slot, input) => __awaiter(this, void 0, void 0, function* () {
|
|
91
|
+
var _a;
|
|
92
|
+
if (slot.disabled) {
|
|
93
|
+
return input;
|
|
94
|
+
}
|
|
95
|
+
const start = perf_hooks_1.performance.now();
|
|
96
|
+
try {
|
|
97
|
+
const output = yield slot.filter.process(input);
|
|
98
|
+
slot.timings.add(perf_hooks_1.performance.now() - start);
|
|
99
|
+
if (output.length !== input.length) {
|
|
100
|
+
disable(slot, `returned ${output.length} samples, expected ${input.length}`);
|
|
101
|
+
return input;
|
|
102
|
+
}
|
|
103
|
+
return output;
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
slot.timings.add(perf_hooks_1.performance.now() - start);
|
|
107
|
+
disable(slot, (_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : String(err));
|
|
108
|
+
return input;
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
const run = (frame) => __awaiter(this, void 0, void 0, function* () {
|
|
112
|
+
const chainStart = perf_hooks_1.performance.now();
|
|
113
|
+
const output = yield slots.reduce((previous, slot) => previous.then((input) => runSlot(slot, input)), Promise.resolve(frame));
|
|
114
|
+
totalTimings.add(perf_hooks_1.performance.now() - chainStart);
|
|
115
|
+
return output;
|
|
116
|
+
});
|
|
117
|
+
return {
|
|
118
|
+
ready() {
|
|
119
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
120
|
+
yield Promise.all(slots.map((slot) => __awaiter(this, void 0, void 0, function* () {
|
|
121
|
+
var _a, _b, _c;
|
|
122
|
+
try {
|
|
123
|
+
yield ((_b = (_a = slot.filter).ready) === null || _b === void 0 ? void 0 : _b.call(_a));
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
disable(slot, (_c = err === null || err === void 0 ? void 0 : err.message) !== null && _c !== void 0 ? _c : String(err));
|
|
127
|
+
}
|
|
128
|
+
})));
|
|
129
|
+
});
|
|
130
|
+
},
|
|
131
|
+
drain() {
|
|
132
|
+
return tail.then(() => undefined, () => undefined);
|
|
133
|
+
},
|
|
134
|
+
process(frame) {
|
|
135
|
+
if (slots.length === 0) {
|
|
136
|
+
return Promise.resolve(frame);
|
|
137
|
+
}
|
|
138
|
+
const result = tail.then(() => run(frame));
|
|
139
|
+
tail = result.catch(() => undefined);
|
|
140
|
+
return result;
|
|
141
|
+
},
|
|
142
|
+
getStats() {
|
|
143
|
+
return {
|
|
144
|
+
total: Object.assign(Object.assign({}, totalTimings.summary()), { delayMs: slots.reduce((sum, s) => sum + s.filter.delayMs, 0) }),
|
|
145
|
+
filters: slots.map((slot) => {
|
|
146
|
+
var _a, _b, _c;
|
|
147
|
+
return (Object.assign(Object.assign({ name: slot.filter.name, delayMs: slot.filter.delayMs }, slot.timings.summary()), { failures: slot.failures, disabled: slot.disabled, error: slot.error, filter: (_c = (_b = (_a = slot.filter).getStats) === null || _b === void 0 ? void 0 : _b.call(_a)) !== null && _c !== void 0 ? _c : {} }));
|
|
148
|
+
})
|
|
149
|
+
};
|
|
150
|
+
},
|
|
151
|
+
inspect() {
|
|
152
|
+
return Object.fromEntries(slots
|
|
153
|
+
.filter((slot) => slot.filter.inspect)
|
|
154
|
+
.map((slot) => [slot.filter.name, slot.filter.inspect()]));
|
|
155
|
+
},
|
|
156
|
+
close() {
|
|
157
|
+
slots.forEach((slot) => {
|
|
158
|
+
var _a, _b;
|
|
159
|
+
try {
|
|
160
|
+
(_b = (_a = slot.filter).close) === null || _b === void 0 ? void 0 : _b.call(_a);
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
logger.warn("error closing audio filter", {
|
|
164
|
+
filter: slot.filter.name,
|
|
165
|
+
err
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AudioFilterChain, AudioFilterConfig, ChainStats } from "./types";
|
|
2
|
+
type AudioFilterSession = {
|
|
3
|
+
current(): AudioFilterChain | null;
|
|
4
|
+
set(filters: AudioFilterConfig[]): Promise<void>;
|
|
5
|
+
close(): ChainStats | null;
|
|
6
|
+
};
|
|
7
|
+
/**
|
|
8
|
+
* Owns a call's filter chain: building it, replacing it, and making sure a
|
|
9
|
+
* chain is drained before it is closed and that nothing is left running after
|
|
10
|
+
* the call ends.
|
|
11
|
+
*/
|
|
12
|
+
declare function createAudioFilterSession(deps?: {
|
|
13
|
+
createChain: (filters: AudioFilterConfig[]) => AudioFilterChain;
|
|
14
|
+
}): AudioFilterSession;
|
|
15
|
+
export { AudioFilterSession, createAudioFilterSession };
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.createAudioFilterSession = createAudioFilterSession;
|
|
13
|
+
/**
|
|
14
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
15
|
+
* http://github.com/fonoster/fonoster
|
|
16
|
+
*
|
|
17
|
+
* This file is part of Fonoster
|
|
18
|
+
*
|
|
19
|
+
* Licensed under the MIT License (the "License");
|
|
20
|
+
* you may not use this file except in compliance with
|
|
21
|
+
* the License. You may obtain a copy of the License at
|
|
22
|
+
*
|
|
23
|
+
* https://opensource.org/licenses/MIT
|
|
24
|
+
*
|
|
25
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
26
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
27
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
28
|
+
* See the License for the specific language governing permissions and
|
|
29
|
+
* limitations under the License.
|
|
30
|
+
*/
|
|
31
|
+
const logger_1 = require("@fonoster/logger");
|
|
32
|
+
const AudioFilterFactory_1 = require("./AudioFilterFactory");
|
|
33
|
+
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
34
|
+
/**
|
|
35
|
+
* Owns a call's filter chain: building it, replacing it, and making sure a
|
|
36
|
+
* chain is drained before it is closed and that nothing is left running after
|
|
37
|
+
* the call ends.
|
|
38
|
+
*/
|
|
39
|
+
function createAudioFilterSession(deps = {
|
|
40
|
+
createChain: (filters) => AudioFilterFactory_1.AudioFilterFactory.createChainFromConfig(filters)
|
|
41
|
+
}) {
|
|
42
|
+
let chain = null;
|
|
43
|
+
let closed = false;
|
|
44
|
+
// Frames may still be inside the chain, so let it finish before disposing
|
|
45
|
+
// whatever the filters hold
|
|
46
|
+
const retire = (retiring) => {
|
|
47
|
+
if (!retiring) {
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
retiring
|
|
51
|
+
.drain()
|
|
52
|
+
.catch(() => undefined)
|
|
53
|
+
.then(() => retiring.close());
|
|
54
|
+
};
|
|
55
|
+
return {
|
|
56
|
+
current: () => chain,
|
|
57
|
+
set(filters) {
|
|
58
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
59
|
+
if (closed) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
if (filters.length === 0) {
|
|
63
|
+
const previous = chain;
|
|
64
|
+
chain = null;
|
|
65
|
+
retire(previous);
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
// Throws on an unknown filter or invalid options, before anything runs
|
|
69
|
+
const next = deps.createChain(filters);
|
|
70
|
+
try {
|
|
71
|
+
yield next.ready();
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
next.close();
|
|
75
|
+
throw err;
|
|
76
|
+
}
|
|
77
|
+
// ready() resolves even when a filter failed to start, because the
|
|
78
|
+
// chain's job is to keep audio flowing. The application asked for
|
|
79
|
+
// filtering though, so tell it rather than pretending it is on.
|
|
80
|
+
const failed = next
|
|
81
|
+
.getStats()
|
|
82
|
+
.filters.filter((filter) => filter.disabled);
|
|
83
|
+
if (failed.length > 0) {
|
|
84
|
+
next.close();
|
|
85
|
+
throw new Error(`could not start audio filter(s): ${failed
|
|
86
|
+
.map((filter) => `${filter.name}: ${filter.error}`)
|
|
87
|
+
.join("; ")}`);
|
|
88
|
+
}
|
|
89
|
+
// The call can end while a model is loading; don't resurrect it
|
|
90
|
+
if (closed) {
|
|
91
|
+
next.close();
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
const previous = chain;
|
|
95
|
+
chain = next;
|
|
96
|
+
retire(previous);
|
|
97
|
+
});
|
|
98
|
+
},
|
|
99
|
+
close() {
|
|
100
|
+
closed = true;
|
|
101
|
+
const retiring = chain;
|
|
102
|
+
chain = null;
|
|
103
|
+
if (!retiring) {
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
106
|
+
const stats = retiring.getStats();
|
|
107
|
+
retire(retiring);
|
|
108
|
+
logger.verbose("audio filter session closed", {
|
|
109
|
+
frames: stats.total.frames
|
|
110
|
+
});
|
|
111
|
+
return stats;
|
|
112
|
+
}
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
export * from "./AudioFilterFactory";
|
|
20
|
+
export * from "./createAudioFilterChain";
|
|
21
|
+
export * from "./createAudioFilterSession";
|
|
22
|
+
export * from "./aiCoustics/createAiCousticsFilter";
|
|
23
|
+
export * from "./aiCoustics/types";
|
|
24
|
+
export * from "./pcm";
|
|
25
|
+
export * from "./types";
|