@fonoster/apiserver 0.25.2 → 0.26.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/applications/createGetFnUtil.d.ts +1 -1
  2. package/dist/applications/validation/assistantWithoutApiKeySchema.d.ts +18 -0
  3. package/dist/applications/validation/createValidationSchema.d.ts +44 -2
  4. package/dist/voice/VoiceDispatcher.js +1 -0
  5. package/dist/voice/client/AudioSocketHandler.d.ts +3 -0
  6. package/dist/voice/client/AudioSocketHandler.js +3 -1
  7. package/dist/voice/client/VoiceClientImpl.d.ts +8 -0
  8. package/dist/voice/client/VoiceClientImpl.js +36 -1
  9. package/dist/voice/filters/AudioFilterFactory.d.ts +35 -0
  10. package/dist/voice/filters/AudioFilterFactory.js +94 -0
  11. package/dist/voice/filters/aiCoustics/createAiCousticsFilter.d.ts +64 -0
  12. package/dist/voice/filters/aiCoustics/createAiCousticsFilter.js +252 -0
  13. package/dist/voice/filters/aiCoustics/types.d.ts +45 -0
  14. package/dist/voice/filters/aiCoustics/types.js +29 -0
  15. package/dist/voice/filters/createAudioFilterChain.d.ts +9 -0
  16. package/dist/voice/filters/createAudioFilterChain.js +171 -0
  17. package/dist/voice/filters/createAudioFilterSession.d.ts +15 -0
  18. package/dist/voice/filters/createAudioFilterSession.js +114 -0
  19. package/dist/voice/filters/index.d.ts +25 -0
  20. package/dist/voice/filters/index.js +41 -0
  21. package/dist/voice/filters/pcm.d.ts +21 -0
  22. package/dist/voice/filters/pcm.js +37 -0
  23. package/dist/voice/filters/types.d.ts +61 -0
  24. package/dist/voice/filters/types.js +26 -0
  25. package/dist/voice/handlers/createSetAudioFiltersHandler.d.ts +11 -0
  26. package/dist/voice/handlers/createSetAudioFiltersHandler.js +74 -0
  27. package/dist/voice/handlers/index.d.ts +1 -0
  28. package/dist/voice/handlers/index.js +1 -0
  29. package/dist/voice/transcribeOnConnection.d.ts +14 -1
  30. package/dist/voice/transcribeOnConnection.js +67 -2
  31. package/dist/voice/types/voice.d.ts +2 -0
  32. package/package.json +13 -10
@@ -0,0 +1,41 @@
1
+ "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
+ };
16
+ Object.defineProperty(exports, "__esModule", { value: true });
17
+ /**
18
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
19
+ * http://github.com/fonoster/fonoster
20
+ *
21
+ * This file is part of Fonoster
22
+ *
23
+ * Licensed under the MIT License (the "License");
24
+ * you may not use this file except in compliance with
25
+ * the License. You may obtain a copy of the License at
26
+ *
27
+ * https://opensource.org/licenses/MIT
28
+ *
29
+ * Unless required by applicable law or agreed to in writing, software
30
+ * distributed under the License is distributed on an "AS IS" BASIS,
31
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
32
+ * See the License for the specific language governing permissions and
33
+ * limitations under the License.
34
+ */
35
+ __exportStar(require("./AudioFilterFactory"), exports);
36
+ __exportStar(require("./createAudioFilterChain"), exports);
37
+ __exportStar(require("./createAudioFilterSession"), exports);
38
+ __exportStar(require("./aiCoustics/createAiCousticsFilter"), exports);
39
+ __exportStar(require("./aiCoustics/types"), exports);
40
+ __exportStar(require("./pcm"), exports);
41
+ __exportStar(require("./types"), exports);
@@ -0,0 +1,21 @@
1
+ /**
2
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
3
+ * http://github.com/fonoster/fonoster
4
+ *
5
+ * This file is part of Fonoster
6
+ *
7
+ * Licensed under the MIT License (the "License");
8
+ * you may not use this file except in compliance with
9
+ * the License. You may obtain a copy of the License at
10
+ *
11
+ * https://opensource.org/licenses/MIT
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+ declare function bufferToInt16(buffer: Buffer): Int16Array;
20
+ declare function int16ToBuffer(samples: Int16Array): Buffer;
21
+ export { bufferToInt16, int16ToBuffer };
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ /**
3
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
4
+ * http://github.com/fonoster/fonoster
5
+ *
6
+ * This file is part of Fonoster
7
+ *
8
+ * Licensed under the MIT License (the "License");
9
+ * you may not use this file except in compliance with
10
+ * the License. You may obtain a copy of the License at
11
+ *
12
+ * https://opensource.org/licenses/MIT
13
+ *
14
+ * Unless required by applicable law or agreed to in writing, software
15
+ * distributed under the License is distributed on an "AS IS" BASIS,
16
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ * See the License for the specific language governing permissions and
18
+ * limitations under the License.
19
+ */
20
+ Object.defineProperty(exports, "__esModule", { value: true });
21
+ exports.bufferToInt16 = bufferToInt16;
22
+ exports.int16ToBuffer = int16ToBuffer;
23
+ // Reads little-endian signed 16-bit PCM. Node Buffers are often slices of a
24
+ // shared pool at an odd offset, which an Int16Array view can't start at, so
25
+ // those are copied first.
26
+ function bufferToInt16(buffer) {
27
+ const samples = Math.floor(buffer.length / 2);
28
+ if (buffer.byteOffset % 2 === 0) {
29
+ return new Int16Array(buffer.buffer, buffer.byteOffset, samples);
30
+ }
31
+ const copy = new Uint8Array(samples * 2);
32
+ copy.set(buffer.subarray(0, samples * 2));
33
+ return new Int16Array(copy.buffer, 0, samples);
34
+ }
35
+ function int16ToBuffer(samples) {
36
+ return Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
37
+ }
@@ -0,0 +1,61 @@
1
+ /**
2
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
3
+ * http://github.com/fonoster/fonoster
4
+ *
5
+ * This file is part of Fonoster
6
+ *
7
+ * Licensed under the MIT License (the "License");
8
+ * you may not use this file except in compliance with
9
+ * the License. You may obtain a copy of the License at
10
+ *
11
+ * https://opensource.org/licenses/MIT
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+ declare const SAMPLE_RATE = 16000;
20
+ type AudioFilter = {
21
+ readonly name: string;
22
+ readonly delayMs: number;
23
+ ready?(): Promise<void>;
24
+ process(frame: Int16Array): Int16Array | Promise<Int16Array>;
25
+ getStats?(): Record<string, number>;
26
+ inspect?(): Record<string, number>;
27
+ close?(): void;
28
+ };
29
+ type AudioFilterConfig = {
30
+ name: string;
31
+ options?: Record<string, unknown>;
32
+ };
33
+ type FilterTimingStats = {
34
+ frames: number;
35
+ p50Ms: number;
36
+ p95Ms: number;
37
+ maxMs: number;
38
+ };
39
+ type FilterStats = FilterTimingStats & {
40
+ name: string;
41
+ delayMs: number;
42
+ failures: number;
43
+ disabled: boolean;
44
+ error: string | null;
45
+ filter: Record<string, number>;
46
+ };
47
+ type ChainStats = {
48
+ total: FilterTimingStats & {
49
+ delayMs: number;
50
+ };
51
+ filters: FilterStats[];
52
+ };
53
+ type AudioFilterChain = {
54
+ ready(): Promise<void>;
55
+ drain(): Promise<void>;
56
+ process(frame: Int16Array): Promise<Int16Array>;
57
+ getStats(): ChainStats;
58
+ inspect(): Record<string, Record<string, number>>;
59
+ close(): void;
60
+ };
61
+ export { AudioFilter, AudioFilterChain, AudioFilterConfig, ChainStats, FilterStats, FilterTimingStats, SAMPLE_RATE };
@@ -0,0 +1,26 @@
1
+ "use strict";
2
+ /**
3
+ * Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
4
+ * http://github.com/fonoster/fonoster
5
+ *
6
+ * This file is part of Fonoster
7
+ *
8
+ * Licensed under the MIT License (the "License");
9
+ * you may not use this file except in compliance with
10
+ * the License. You may obtain a copy of the License at
11
+ *
12
+ * https://opensource.org/licenses/MIT
13
+ *
14
+ * Unless required by applicable law or agreed to in writing, software
15
+ * distributed under the License is distributed on an "AS IS" BASIS,
16
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17
+ * See the License for the specific language governing permissions and
18
+ * limitations under the License.
19
+ */
20
+ Object.defineProperty(exports, "__esModule", { value: true });
21
+ exports.SAMPLE_RATE = void 0;
22
+ // Caller audio as it arrives from Asterisk: 16 kHz mono signed 16-bit PCM
23
+ // (slin16). Frames are usually 20 ms (320 samples) but filters must accept
24
+ // any length.
25
+ const SAMPLE_RATE = 16000;
26
+ exports.SAMPLE_RATE = SAMPLE_RATE;
@@ -0,0 +1,11 @@
1
+ import { VoiceClient } from "../types";
2
+ /**
3
+ * Sets the audio filters for the session. The filters run on the caller's
4
+ * audio before it reaches speech recognition and the application's media
5
+ * stream.
6
+ *
7
+ * Failure here never fails the call: the application is told what went wrong
8
+ * and the call continues with whatever filtering was already in place, or none.
9
+ */
10
+ declare function createSetAudioFiltersHandler(voiceClient: VoiceClient): (request: import("@fonoster/common").VerbRequest) => Promise<void>;
11
+ export { createSetAudioFiltersHandler };
@@ -0,0 +1,74 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ exports.createSetAudioFiltersHandler = createSetAudioFiltersHandler;
13
+ const logger_1 = require("@fonoster/logger");
14
+ const pb_util_1 = require("pb-util");
15
+ const zod_1 = require("zod");
16
+ const zod_validation_error_1 = require("zod-validation-error");
17
+ const withErrorHandling_1 = require("./utils/withErrorHandling");
18
+ const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
19
+ const requestSchema = zod_1.z.object({
20
+ mediaSessionRef: zod_1.z.string(),
21
+ filters: zod_1.z
22
+ .array(zod_1.z.object({
23
+ name: zod_1.z.string().min(1),
24
+ options: zod_1.z.unknown().optional()
25
+ }))
26
+ .optional()
27
+ });
28
+ /**
29
+ * Sets the audio filters for the session. The filters run on the caller's
30
+ * audio before it reaches speech recognition and the application's media
31
+ * stream.
32
+ *
33
+ * Failure here never fails the call: the application is told what went wrong
34
+ * and the call continues with whatever filtering was already in place, or none.
35
+ */
36
+ function createSetAudioFiltersHandler(voiceClient) {
37
+ return (0, withErrorHandling_1.withErrorHandling)((request) => __awaiter(this, void 0, void 0, function* () {
38
+ var _a, _b;
39
+ const { mediaSessionRef } = request;
40
+ // Never rethrow, and always answer: a rejected handler escapes as an
41
+ // unhandled rejection and takes the process down with every call on it,
42
+ // and a verb left unanswered strands the application's await for the rest
43
+ // of the call. The application is told instead, and the call keeps going
44
+ // with whatever filtering it had.
45
+ try {
46
+ requestSchema.parse(request);
47
+ const config = ((_a = request.filters) !== null && _a !== void 0 ? _a : []).map((filter) => ({
48
+ name: filter.name,
49
+ options: filter.options
50
+ ? pb_util_1.struct.decode(filter.options)
51
+ : {}
52
+ }));
53
+ yield voiceClient.setAudioFilters(config);
54
+ }
55
+ catch (err) {
56
+ const message = err instanceof zod_1.z.ZodError
57
+ ? (0, zod_validation_error_1.fromError)(err, { prefix: null }).toString()
58
+ : ((_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : String(err));
59
+ logger.warn("could not set audio filters; call continues unfiltered", {
60
+ mediaSessionRef,
61
+ error: message
62
+ });
63
+ voiceClient.sendResponse({
64
+ setAudioFiltersResponse: { mediaSessionRef, error: message }
65
+ });
66
+ return;
67
+ }
68
+ voiceClient.sendResponse({
69
+ setAudioFiltersResponse: {
70
+ mediaSessionRef
71
+ }
72
+ });
73
+ }));
74
+ }
@@ -25,6 +25,7 @@ export * from "./createPlayDtmfHandler";
25
25
  export * from "./createPlaybackControlHandler";
26
26
  export * from "./createRecordHandler";
27
27
  export * from "./createSayHandler";
28
+ export * from "./createSetAudioFiltersHandler";
28
29
  export * from "./createStreamGatherHandler";
29
30
  export * from "./createUnmuteHandler";
30
31
  export * from "./dial/createDialHandler";
@@ -41,6 +41,7 @@ __exportStar(require("./createPlayDtmfHandler"), exports);
41
41
  __exportStar(require("./createPlaybackControlHandler"), exports);
42
42
  __exportStar(require("./createRecordHandler"), exports);
43
43
  __exportStar(require("./createSayHandler"), exports);
44
+ __exportStar(require("./createSetAudioFiltersHandler"), exports);
44
45
  __exportStar(require("./createStreamGatherHandler"), exports);
45
46
  __exportStar(require("./createUnmuteHandler"), exports);
46
47
  __exportStar(require("./dial/createDialHandler"), exports);
@@ -18,5 +18,18 @@
18
18
  */
19
19
  import { Stream } from "stream";
20
20
  import { AudioStream } from "@fonoster/streams";
21
- declare function transcribeOnConnection(stream: Stream): (_: any, res: AudioStream) => Promise<void>;
21
+ import { AudioFilterChain } from "./filters";
22
+ /**
23
+ * Fans the caller's audio out to everything that consumes it: speech
24
+ * recognition and the application's media stream.
25
+ *
26
+ * When the session has audio filters, frames run through the chain first, so
27
+ * every consumer sees the same filtered audio. With no filters the buffer is
28
+ * emitted synchronously, exactly as before.
29
+ *
30
+ * Frames are emitted in arrival order even while filters are being replaced
31
+ * or removed: once anything is in flight, later frames queue behind it rather
32
+ * than overtaking it on the unfiltered path.
33
+ */
34
+ declare function transcribeOnConnection(stream: Stream, getFilterChain?: () => AudioFilterChain | null): (_: any, res: AudioStream) => Promise<void>;
22
35
  export { transcribeOnConnection };
@@ -11,13 +11,78 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
11
11
  Object.defineProperty(exports, "__esModule", { value: true });
12
12
  exports.transcribeOnConnection = transcribeOnConnection;
13
13
  const logger_1 = require("@fonoster/logger");
14
+ const filters_1 = require("./filters");
14
15
  const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
15
- function transcribeOnConnection(stream) {
16
+ // About a second of audio. Past this the filter is not keeping up with the
17
+ // call, so frames skip it rather than piling up without bound.
18
+ const MAX_FRAMES_IN_FLIGHT = 50;
19
+ /**
20
+ * Fans the caller's audio out to everything that consumes it: speech
21
+ * recognition and the application's media stream.
22
+ *
23
+ * When the session has audio filters, frames run through the chain first, so
24
+ * every consumer sees the same filtered audio. With no filters the buffer is
25
+ * emitted synchronously, exactly as before.
26
+ *
27
+ * Frames are emitted in arrival order even while filters are being replaced
28
+ * or removed: once anything is in flight, later frames queue behind it rather
29
+ * than overtaking it on the unfiltered path.
30
+ */
31
+ function transcribeOnConnection(stream, getFilterChain = () => null) {
16
32
  return (_, res) => __awaiter(this, void 0, void 0, function* () {
17
33
  logger.verbose("starting instance of audio stream for transcription");
34
+ let queue = Promise.resolve();
35
+ let inFlight = 0;
36
+ let warnedAboutBacklog = false;
37
+ // Listeners run synchronously here, so one that throws must not take the
38
+ // audio path down with it, and must never cause a frame to be emitted
39
+ // twice
40
+ const emit = (data) => {
41
+ try {
42
+ stream.emit("data", data);
43
+ }
44
+ catch (err) {
45
+ logger.warn("a consumer of the audio stream threw", { err });
46
+ }
47
+ };
48
+ const handle = (data) => __awaiter(this, void 0, void 0, function* () {
49
+ const chain = getFilterChain();
50
+ if (!chain) {
51
+ emit(data);
52
+ return;
53
+ }
54
+ if (inFlight > MAX_FRAMES_IN_FLIGHT) {
55
+ if (!warnedAboutBacklog) {
56
+ warnedAboutBacklog = true;
57
+ logger.warn("audio filters are behind real time; frames are skipping them", { framesInFlight: inFlight });
58
+ }
59
+ emit(data);
60
+ return;
61
+ }
62
+ try {
63
+ emit((0, filters_1.int16ToBuffer)(yield chain.process((0, filters_1.bufferToInt16)(data))));
64
+ }
65
+ catch (err) {
66
+ logger.warn("audio filter chain failed; passing audio through", {
67
+ err
68
+ });
69
+ emit(data);
70
+ }
71
+ });
18
72
  res
19
73
  .onData((data) => {
20
- stream.emit("data", data);
74
+ // Nothing pending and no filters: keep the original synchronous path
75
+ if (inFlight === 0 && !getFilterChain()) {
76
+ emit(data);
77
+ return;
78
+ }
79
+ inFlight++;
80
+ queue = queue
81
+ .then(() => handle(data))
82
+ .catch((err) => logger.error("unexpected error in the audio path", { err }))
83
+ .finally(() => {
84
+ inFlight--;
85
+ });
21
86
  })
22
87
  .onError((err) => {
23
88
  logger.error("stream error:", err);
@@ -19,6 +19,7 @@
19
19
  import { Readable, Stream } from "stream";
20
20
  import { SayOptions, StreamContent, VoiceClientConfig, VoiceIn, VoiceSessionStreamClient } from "@fonoster/common";
21
21
  import * as grpc from "@grpc/grpc-js";
22
+ import { AudioFilterConfig } from "../filters";
22
23
  import { SpeechResult, StreamSpeech } from "../stt/types";
23
24
  type VoiceClient = {
24
25
  config: VoiceClientConfig;
@@ -46,6 +47,7 @@ type VoiceClient = {
46
47
  digits: string;
47
48
  }>;
48
49
  getTranscriptionsStream: () => Stream;
50
+ setAudioFilters: (filters: AudioFilterConfig[]) => Promise<void>;
49
51
  stopSynthesis: () => Promise<void>;
50
52
  };
51
53
  type TextToSpeech = {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fonoster/apiserver",
3
- "version": "0.25.2",
3
+ "version": "0.26.0",
4
4
  "description": "APIServer for Fonoster",
5
5
  "author": "Pedro Sanders <psanders@fonoster.com>",
6
6
  "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -22,14 +22,14 @@
22
22
  },
23
23
  "dependencies": {
24
24
  "@deepgram/sdk": "^3.5.1",
25
- "@fonoster/authz": "^0.25.1",
26
- "@fonoster/autopilot": "^0.25.1",
27
- "@fonoster/common": "^0.25.1",
28
- "@fonoster/identity": "^0.25.1",
29
- "@fonoster/logger": "^0.25.0",
30
- "@fonoster/sipnet": "^0.25.1",
31
- "@fonoster/streams": "^0.25.2",
32
- "@fonoster/types": "^0.25.0",
25
+ "@fonoster/authz": "^0.26.0",
26
+ "@fonoster/autopilot": "^0.26.0",
27
+ "@fonoster/common": "^0.26.0",
28
+ "@fonoster/identity": "^0.26.0",
29
+ "@fonoster/logger": "^0.26.0",
30
+ "@fonoster/sipnet": "^0.26.0",
31
+ "@fonoster/streams": "^0.26.0",
32
+ "@fonoster/types": "^0.26.0",
33
33
  "@google-cloud/speech": "^6.6.0",
34
34
  "@google-cloud/text-to-speech": "^5.3.0",
35
35
  "@grpc/grpc-js": "~1.10.11",
@@ -76,5 +76,8 @@
76
76
  "@types/uuid": "^10.0.0",
77
77
  "@types/validator": "^13.15.10"
78
78
  },
79
- "gitHead": "6a1a458326108fd79f670c5b0470f15d957ae867"
79
+ "gitHead": "d14f414d791a33f92e8c4225f7e462abc440a142",
80
+ "optionalDependencies": {
81
+ "@ai-coustics/aic-sdk": "^0.24.0"
82
+ }
80
83
  }