@fonoster/apiserver 0.25.2 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/applications/createGetFnUtil.d.ts +1 -1
- package/dist/applications/validation/assistantWithoutApiKeySchema.d.ts +18 -0
- package/dist/applications/validation/createValidationSchema.d.ts +44 -2
- package/dist/voice/VoiceDispatcher.js +1 -0
- package/dist/voice/client/AudioSocketHandler.d.ts +3 -0
- package/dist/voice/client/AudioSocketHandler.js +3 -1
- package/dist/voice/client/VoiceClientImpl.d.ts +8 -0
- package/dist/voice/client/VoiceClientImpl.js +36 -1
- package/dist/voice/filters/AudioFilterFactory.d.ts +35 -0
- package/dist/voice/filters/AudioFilterFactory.js +94 -0
- package/dist/voice/filters/aiCoustics/createAiCousticsFilter.d.ts +64 -0
- package/dist/voice/filters/aiCoustics/createAiCousticsFilter.js +252 -0
- package/dist/voice/filters/aiCoustics/types.d.ts +45 -0
- package/dist/voice/filters/aiCoustics/types.js +29 -0
- package/dist/voice/filters/createAudioFilterChain.d.ts +9 -0
- package/dist/voice/filters/createAudioFilterChain.js +171 -0
- package/dist/voice/filters/createAudioFilterSession.d.ts +15 -0
- package/dist/voice/filters/createAudioFilterSession.js +114 -0
- package/dist/voice/filters/index.d.ts +25 -0
- package/dist/voice/filters/index.js +41 -0
- package/dist/voice/filters/pcm.d.ts +21 -0
- package/dist/voice/filters/pcm.js +37 -0
- package/dist/voice/filters/types.d.ts +61 -0
- package/dist/voice/filters/types.js +26 -0
- package/dist/voice/handlers/createSetAudioFiltersHandler.d.ts +11 -0
- package/dist/voice/handlers/createSetAudioFiltersHandler.js +74 -0
- package/dist/voice/handlers/index.d.ts +1 -0
- package/dist/voice/handlers/index.js +1 -0
- package/dist/voice/stt/Deepgram.d.ts +26 -2
- package/dist/voice/stt/Deepgram.js +21 -6
- package/dist/voice/stt/types.d.ts +5 -3
- package/dist/voice/stt/types.js +5 -1
- package/dist/voice/transcribeOnConnection.d.ts +14 -1
- package/dist/voice/transcribeOnConnection.js +67 -2
- package/dist/voice/types/voice.d.ts +2 -0
- package/package.json +13 -10
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
/**
|
|
18
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
19
|
+
* http://github.com/fonoster/fonoster
|
|
20
|
+
*
|
|
21
|
+
* This file is part of Fonoster
|
|
22
|
+
*
|
|
23
|
+
* Licensed under the MIT License (the "License");
|
|
24
|
+
* you may not use this file except in compliance with
|
|
25
|
+
* the License. You may obtain a copy of the License at
|
|
26
|
+
*
|
|
27
|
+
* https://opensource.org/licenses/MIT
|
|
28
|
+
*
|
|
29
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
30
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
31
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
32
|
+
* See the License for the specific language governing permissions and
|
|
33
|
+
* limitations under the License.
|
|
34
|
+
*/
|
|
35
|
+
__exportStar(require("./AudioFilterFactory"), exports);
|
|
36
|
+
__exportStar(require("./createAudioFilterChain"), exports);
|
|
37
|
+
__exportStar(require("./createAudioFilterSession"), exports);
|
|
38
|
+
__exportStar(require("./aiCoustics/createAiCousticsFilter"), exports);
|
|
39
|
+
__exportStar(require("./aiCoustics/types"), exports);
|
|
40
|
+
__exportStar(require("./pcm"), exports);
|
|
41
|
+
__exportStar(require("./types"), exports);
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
declare function bufferToInt16(buffer: Buffer): Int16Array;
|
|
20
|
+
declare function int16ToBuffer(samples: Int16Array): Buffer;
|
|
21
|
+
export { bufferToInt16, int16ToBuffer };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
4
|
+
* http://github.com/fonoster/fonoster
|
|
5
|
+
*
|
|
6
|
+
* This file is part of Fonoster
|
|
7
|
+
*
|
|
8
|
+
* Licensed under the MIT License (the "License");
|
|
9
|
+
* you may not use this file except in compliance with
|
|
10
|
+
* the License. You may obtain a copy of the License at
|
|
11
|
+
*
|
|
12
|
+
* https://opensource.org/licenses/MIT
|
|
13
|
+
*
|
|
14
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
15
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
16
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
17
|
+
* See the License for the specific language governing permissions and
|
|
18
|
+
* limitations under the License.
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.bufferToInt16 = bufferToInt16;
|
|
22
|
+
exports.int16ToBuffer = int16ToBuffer;
|
|
23
|
+
// Reads little-endian signed 16-bit PCM. Node Buffers are often slices of a
|
|
24
|
+
// shared pool at an odd offset, which an Int16Array view can't start at, so
|
|
25
|
+
// those are copied first.
|
|
26
|
+
function bufferToInt16(buffer) {
|
|
27
|
+
const samples = Math.floor(buffer.length / 2);
|
|
28
|
+
if (buffer.byteOffset % 2 === 0) {
|
|
29
|
+
return new Int16Array(buffer.buffer, buffer.byteOffset, samples);
|
|
30
|
+
}
|
|
31
|
+
const copy = new Uint8Array(samples * 2);
|
|
32
|
+
copy.set(buffer.subarray(0, samples * 2));
|
|
33
|
+
return new Int16Array(copy.buffer, 0, samples);
|
|
34
|
+
}
|
|
35
|
+
function int16ToBuffer(samples) {
|
|
36
|
+
return Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
37
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
+
* http://github.com/fonoster/fonoster
|
|
4
|
+
*
|
|
5
|
+
* This file is part of Fonoster
|
|
6
|
+
*
|
|
7
|
+
* Licensed under the MIT License (the "License");
|
|
8
|
+
* you may not use this file except in compliance with
|
|
9
|
+
* the License. You may obtain a copy of the License at
|
|
10
|
+
*
|
|
11
|
+
* https://opensource.org/licenses/MIT
|
|
12
|
+
*
|
|
13
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
* See the License for the specific language governing permissions and
|
|
17
|
+
* limitations under the License.
|
|
18
|
+
*/
|
|
19
|
+
declare const SAMPLE_RATE = 16000;
|
|
20
|
+
type AudioFilter = {
|
|
21
|
+
readonly name: string;
|
|
22
|
+
readonly delayMs: number;
|
|
23
|
+
ready?(): Promise<void>;
|
|
24
|
+
process(frame: Int16Array): Int16Array | Promise<Int16Array>;
|
|
25
|
+
getStats?(): Record<string, number>;
|
|
26
|
+
inspect?(): Record<string, number>;
|
|
27
|
+
close?(): void;
|
|
28
|
+
};
|
|
29
|
+
type AudioFilterConfig = {
|
|
30
|
+
name: string;
|
|
31
|
+
options?: Record<string, unknown>;
|
|
32
|
+
};
|
|
33
|
+
type FilterTimingStats = {
|
|
34
|
+
frames: number;
|
|
35
|
+
p50Ms: number;
|
|
36
|
+
p95Ms: number;
|
|
37
|
+
maxMs: number;
|
|
38
|
+
};
|
|
39
|
+
type FilterStats = FilterTimingStats & {
|
|
40
|
+
name: string;
|
|
41
|
+
delayMs: number;
|
|
42
|
+
failures: number;
|
|
43
|
+
disabled: boolean;
|
|
44
|
+
error: string | null;
|
|
45
|
+
filter: Record<string, number>;
|
|
46
|
+
};
|
|
47
|
+
type ChainStats = {
|
|
48
|
+
total: FilterTimingStats & {
|
|
49
|
+
delayMs: number;
|
|
50
|
+
};
|
|
51
|
+
filters: FilterStats[];
|
|
52
|
+
};
|
|
53
|
+
type AudioFilterChain = {
|
|
54
|
+
ready(): Promise<void>;
|
|
55
|
+
drain(): Promise<void>;
|
|
56
|
+
process(frame: Int16Array): Promise<Int16Array>;
|
|
57
|
+
getStats(): ChainStats;
|
|
58
|
+
inspect(): Record<string, Record<string, number>>;
|
|
59
|
+
close(): void;
|
|
60
|
+
};
|
|
61
|
+
export { AudioFilter, AudioFilterChain, AudioFilterConfig, ChainStats, FilterStats, FilterTimingStats, SAMPLE_RATE };
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
4
|
+
* http://github.com/fonoster/fonoster
|
|
5
|
+
*
|
|
6
|
+
* This file is part of Fonoster
|
|
7
|
+
*
|
|
8
|
+
* Licensed under the MIT License (the "License");
|
|
9
|
+
* you may not use this file except in compliance with
|
|
10
|
+
* the License. You may obtain a copy of the License at
|
|
11
|
+
*
|
|
12
|
+
* https://opensource.org/licenses/MIT
|
|
13
|
+
*
|
|
14
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
15
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
16
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
17
|
+
* See the License for the specific language governing permissions and
|
|
18
|
+
* limitations under the License.
|
|
19
|
+
*/
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.SAMPLE_RATE = void 0;
|
|
22
|
+
// Caller audio as it arrives from Asterisk: 16 kHz mono signed 16-bit PCM
|
|
23
|
+
// (slin16). Frames are usually 20 ms (320 samples) but filters must accept
|
|
24
|
+
// any length.
|
|
25
|
+
const SAMPLE_RATE = 16000;
|
|
26
|
+
exports.SAMPLE_RATE = SAMPLE_RATE;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { VoiceClient } from "../types";
|
|
2
|
+
/**
|
|
3
|
+
* Sets the audio filters for the session. The filters run on the caller's
|
|
4
|
+
* audio before it reaches speech recognition and the application's media
|
|
5
|
+
* stream.
|
|
6
|
+
*
|
|
7
|
+
* Failure here never fails the call: the application is told what went wrong
|
|
8
|
+
* and the call continues with whatever filtering was already in place, or none.
|
|
9
|
+
*/
|
|
10
|
+
declare function createSetAudioFiltersHandler(voiceClient: VoiceClient): (request: import("@fonoster/common").VerbRequest) => Promise<void>;
|
|
11
|
+
export { createSetAudioFiltersHandler };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.createSetAudioFiltersHandler = createSetAudioFiltersHandler;
|
|
13
|
+
const logger_1 = require("@fonoster/logger");
|
|
14
|
+
const pb_util_1 = require("pb-util");
|
|
15
|
+
const zod_1 = require("zod");
|
|
16
|
+
const zod_validation_error_1 = require("zod-validation-error");
|
|
17
|
+
const withErrorHandling_1 = require("./utils/withErrorHandling");
|
|
18
|
+
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
19
|
+
const requestSchema = zod_1.z.object({
|
|
20
|
+
mediaSessionRef: zod_1.z.string(),
|
|
21
|
+
filters: zod_1.z
|
|
22
|
+
.array(zod_1.z.object({
|
|
23
|
+
name: zod_1.z.string().min(1),
|
|
24
|
+
options: zod_1.z.unknown().optional()
|
|
25
|
+
}))
|
|
26
|
+
.optional()
|
|
27
|
+
});
|
|
28
|
+
/**
|
|
29
|
+
* Sets the audio filters for the session. The filters run on the caller's
|
|
30
|
+
* audio before it reaches speech recognition and the application's media
|
|
31
|
+
* stream.
|
|
32
|
+
*
|
|
33
|
+
* Failure here never fails the call: the application is told what went wrong
|
|
34
|
+
* and the call continues with whatever filtering was already in place, or none.
|
|
35
|
+
*/
|
|
36
|
+
function createSetAudioFiltersHandler(voiceClient) {
|
|
37
|
+
return (0, withErrorHandling_1.withErrorHandling)((request) => __awaiter(this, void 0, void 0, function* () {
|
|
38
|
+
var _a, _b;
|
|
39
|
+
const { mediaSessionRef } = request;
|
|
40
|
+
// Never rethrow, and always answer: a rejected handler escapes as an
|
|
41
|
+
// unhandled rejection and takes the process down with every call on it,
|
|
42
|
+
// and a verb left unanswered strands the application's await for the rest
|
|
43
|
+
// of the call. The application is told instead, and the call keeps going
|
|
44
|
+
// with whatever filtering it had.
|
|
45
|
+
try {
|
|
46
|
+
requestSchema.parse(request);
|
|
47
|
+
const config = ((_a = request.filters) !== null && _a !== void 0 ? _a : []).map((filter) => ({
|
|
48
|
+
name: filter.name,
|
|
49
|
+
options: filter.options
|
|
50
|
+
? pb_util_1.struct.decode(filter.options)
|
|
51
|
+
: {}
|
|
52
|
+
}));
|
|
53
|
+
yield voiceClient.setAudioFilters(config);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
const message = err instanceof zod_1.z.ZodError
|
|
57
|
+
? (0, zod_validation_error_1.fromError)(err, { prefix: null }).toString()
|
|
58
|
+
: ((_b = err === null || err === void 0 ? void 0 : err.message) !== null && _b !== void 0 ? _b : String(err));
|
|
59
|
+
logger.warn("could not set audio filters; call continues unfiltered", {
|
|
60
|
+
mediaSessionRef,
|
|
61
|
+
error: message
|
|
62
|
+
});
|
|
63
|
+
voiceClient.sendResponse({
|
|
64
|
+
setAudioFiltersResponse: { mediaSessionRef, error: message }
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
voiceClient.sendResponse({
|
|
69
|
+
setAudioFiltersResponse: {
|
|
70
|
+
mediaSessionRef
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
@@ -25,6 +25,7 @@ export * from "./createPlayDtmfHandler";
|
|
|
25
25
|
export * from "./createPlaybackControlHandler";
|
|
26
26
|
export * from "./createRecordHandler";
|
|
27
27
|
export * from "./createSayHandler";
|
|
28
|
+
export * from "./createSetAudioFiltersHandler";
|
|
28
29
|
export * from "./createStreamGatherHandler";
|
|
29
30
|
export * from "./createUnmuteHandler";
|
|
30
31
|
export * from "./dial/createDialHandler";
|
|
@@ -41,6 +41,7 @@ __exportStar(require("./createPlayDtmfHandler"), exports);
|
|
|
41
41
|
__exportStar(require("./createPlaybackControlHandler"), exports);
|
|
42
42
|
__exportStar(require("./createRecordHandler"), exports);
|
|
43
43
|
__exportStar(require("./createSayHandler"), exports);
|
|
44
|
+
__exportStar(require("./createSetAudioFiltersHandler"), exports);
|
|
44
45
|
__exportStar(require("./createStreamGatherHandler"), exports);
|
|
45
46
|
__exportStar(require("./createUnmuteHandler"), exports);
|
|
46
47
|
__exportStar(require("./dial/createDialHandler"), exports);
|
|
@@ -20,7 +20,7 @@ import { Stream } from "stream";
|
|
|
20
20
|
import * as z from "zod";
|
|
21
21
|
import { SpeechToText } from "../types";
|
|
22
22
|
import { AbstractSpeechToText } from "./AbstractSpeechToText";
|
|
23
|
-
import { DeepgramSttConfig, SpeechResult, StreamSpeech } from "./types";
|
|
23
|
+
import { DeepgramLanguage, DeepgramModel, DeepgramSttConfig, SpeechResult, StreamSpeech } from "./types";
|
|
24
24
|
declare const DeepgramClient: any;
|
|
25
25
|
declare const ENGINE_NAME = "stt.deepgram";
|
|
26
26
|
declare class Deepgram extends AbstractSpeechToText<typeof ENGINE_NAME> implements SpeechToText {
|
|
@@ -33,4 +33,28 @@ declare class Deepgram extends AbstractSpeechToText<typeof ENGINE_NAME> implemen
|
|
|
33
33
|
static getConfigValidationSchema(): z.Schema;
|
|
34
34
|
static getCredentialsValidationSchema(): z.Schema;
|
|
35
35
|
}
|
|
36
|
-
|
|
36
|
+
declare function buildTranscribeConfig(config: {
|
|
37
|
+
model: DeepgramModel;
|
|
38
|
+
languageCode: DeepgramLanguage;
|
|
39
|
+
smartFormat?: boolean;
|
|
40
|
+
noDelay?: boolean;
|
|
41
|
+
interimResults?: boolean;
|
|
42
|
+
utteranceEndMs?: number;
|
|
43
|
+
}): {
|
|
44
|
+
endpointing: number;
|
|
45
|
+
model: DeepgramModel;
|
|
46
|
+
language: DeepgramLanguage;
|
|
47
|
+
encoding: string;
|
|
48
|
+
sample_rate: number;
|
|
49
|
+
channels: number;
|
|
50
|
+
smart_format: boolean;
|
|
51
|
+
no_delay: boolean;
|
|
52
|
+
interim_results: boolean;
|
|
53
|
+
utterance_end_ms: number;
|
|
54
|
+
languageCode: DeepgramLanguage;
|
|
55
|
+
smartFormat?: boolean;
|
|
56
|
+
noDelay?: boolean;
|
|
57
|
+
interimResults?: boolean;
|
|
58
|
+
utteranceEndMs?: number;
|
|
59
|
+
};
|
|
60
|
+
export { buildTranscribeConfig, Deepgram, ENGINE_NAME };
|
|
@@ -43,6 +43,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.ENGINE_NAME = exports.Deepgram = void 0;
|
|
46
|
+
exports.buildTranscribeConfig = buildTranscribeConfig;
|
|
46
47
|
/**
|
|
47
48
|
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
48
49
|
* http://github.com/fonoster/fonoster
|
|
@@ -72,6 +73,11 @@ const { DeepgramClient, LiveTranscriptionEvents, createClient
|
|
|
72
73
|
} = require("@deepgram/sdk"); // Why Deepgram :(
|
|
73
74
|
const ENGINE_NAME = "stt.deepgram";
|
|
74
75
|
exports.ENGINE_NAME = ENGINE_NAME;
|
|
76
|
+
// Nova-3 detects English, Spanish and eight other languages; Nova-2 only
|
|
77
|
+
// English + Spanish. The phonecall/conversationalai variants are English-only.
|
|
78
|
+
const MULTILINGUAL_MODELS = [types_1.DeepgramModel.NOVA_3, types_1.DeepgramModel.NOVA_2];
|
|
79
|
+
// Deepgram's recommended endpointing for code-switching (its default is 10ms)
|
|
80
|
+
const MULTILINGUAL_ENDPOINTING_MS = 100;
|
|
75
81
|
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
76
82
|
class Deepgram extends AbstractSpeechToText_1.AbstractSpeechToText {
|
|
77
83
|
constructor(config) {
|
|
@@ -269,19 +275,24 @@ class Deepgram extends AbstractSpeechToText_1.AbstractSpeechToText {
|
|
|
269
275
|
});
|
|
270
276
|
}
|
|
271
277
|
static getConfigValidationSchema() {
|
|
272
|
-
return z
|
|
278
|
+
return z
|
|
279
|
+
.object({
|
|
273
280
|
smartFormat: z.boolean().optional(),
|
|
274
281
|
noDelay: z.boolean().optional(),
|
|
275
282
|
languageCode: z
|
|
276
|
-
.nativeEnum(common_1.VoiceLanguage, {
|
|
277
|
-
message: common_1.Messages.VALID_LANGUAGE_CODE
|
|
278
|
-
})
|
|
283
|
+
.union([z.nativeEnum(common_1.VoiceLanguage), z.literal(types_1.DEEPGRAM_MULTILINGUAL)], { errorMap: () => ({ message: common_1.Messages.VALID_LANGUAGE_CODE }) })
|
|
279
284
|
.optional(),
|
|
280
285
|
model: z
|
|
281
286
|
.nativeEnum(types_1.DeepgramModel, { message: "Invalid Deepgram model" })
|
|
282
287
|
.optional(),
|
|
283
288
|
interimResults: z.boolean().optional(),
|
|
284
289
|
utteranceEndMs: z.number().int().min(1000).max(5000).optional()
|
|
290
|
+
})
|
|
291
|
+
.refine((config) => config.languageCode !== types_1.DEEPGRAM_MULTILINGUAL ||
|
|
292
|
+
!config.model ||
|
|
293
|
+
MULTILINGUAL_MODELS.includes(config.model), {
|
|
294
|
+
message: `Language "${types_1.DEEPGRAM_MULTILINGUAL}" requires one of these models: ${MULTILINGUAL_MODELS.join(", ")}`,
|
|
295
|
+
path: ["model"]
|
|
285
296
|
});
|
|
286
297
|
}
|
|
287
298
|
static getCredentialsValidationSchema() {
|
|
@@ -298,11 +309,15 @@ function buildTranscribeConfig(config) {
|
|
|
298
309
|
// Default utterance_end_ms to 1000ms (minimum required value)
|
|
299
310
|
// This enables UtteranceEnd events as a fallback when speech_final never becomes true
|
|
300
311
|
const utteranceEndMs = config.utteranceEndMs || 1000;
|
|
301
|
-
|
|
312
|
+
const isMultilingual = config.languageCode === types_1.DEEPGRAM_MULTILINGUAL;
|
|
313
|
+
return Object.assign(Object.assign(Object.assign({}, config), {
|
|
314
|
+
// The phonecall default is English-only, so multilingual needs its own
|
|
315
|
+
model: config.model ||
|
|
316
|
+
(isMultilingual ? types_1.DeepgramModel.NOVA_3 : types_1.DeepgramModel.NOVA_2_PHONECALL), language: config.languageCode || common_1.VoiceLanguage.EN_US, encoding: "linear16", sample_rate: 16000, channels: 1, smart_format: config.smartFormat !== false,
|
|
302
317
|
// This needs to be set to true to avoid delays while using smart_format
|
|
303
318
|
no_delay: config.noDelay !== false,
|
|
304
319
|
// REQUIRED for UtteranceEnd: interim_results must be true
|
|
305
320
|
interim_results: interimResults,
|
|
306
321
|
// REQUIRED for UtteranceEnd: utterance_end_ms parameter
|
|
307
|
-
utterance_end_ms: utteranceEndMs });
|
|
322
|
+
utterance_end_ms: utteranceEndMs }), (isMultilingual && { endpointing: MULTILINGUAL_ENDPOINTING_MS }));
|
|
308
323
|
}
|
|
@@ -27,7 +27,7 @@ type StreamSpeech = {
|
|
|
27
27
|
};
|
|
28
28
|
type SttConfig = {
|
|
29
29
|
config: {
|
|
30
|
-
languageCode:
|
|
30
|
+
languageCode: string;
|
|
31
31
|
};
|
|
32
32
|
};
|
|
33
33
|
type GoogleSttConfig = {
|
|
@@ -45,9 +45,11 @@ declare enum DeepgramModel {
|
|
|
45
45
|
NOVA_2_PHONECALL = "nova-2-phonecall",
|
|
46
46
|
NOVA_2_CONVERSATIONALAI = "nova-2-conversationalai"
|
|
47
47
|
}
|
|
48
|
+
declare const DEEPGRAM_MULTILINGUAL = "multi";
|
|
49
|
+
type DeepgramLanguage = VoiceLanguage | typeof DEEPGRAM_MULTILINGUAL;
|
|
48
50
|
type DeepgramSttConfig = {
|
|
49
51
|
config: {
|
|
50
|
-
languageCode:
|
|
52
|
+
languageCode: DeepgramLanguage;
|
|
51
53
|
model: DeepgramModel;
|
|
52
54
|
smartFormat: boolean;
|
|
53
55
|
noDelay: boolean;
|
|
@@ -58,4 +60,4 @@ type DeepgramSttConfig = {
|
|
|
58
60
|
apiKey: string;
|
|
59
61
|
};
|
|
60
62
|
};
|
|
61
|
-
export { DeepgramModel, DeepgramSttConfig, GoogleSttConfig, SpeechResult, StreamSpeech, SttConfig };
|
|
63
|
+
export { DEEPGRAM_MULTILINGUAL, DeepgramLanguage, DeepgramModel, DeepgramSttConfig, GoogleSttConfig, SpeechResult, StreamSpeech, SttConfig };
|
package/dist/voice/stt/types.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.DeepgramModel = void 0;
|
|
3
|
+
exports.DeepgramModel = exports.DEEPGRAM_MULTILINGUAL = void 0;
|
|
4
4
|
var DeepgramModel;
|
|
5
5
|
(function (DeepgramModel) {
|
|
6
6
|
DeepgramModel["NOVA_3"] = "nova-3";
|
|
@@ -8,3 +8,7 @@ var DeepgramModel;
|
|
|
8
8
|
DeepgramModel["NOVA_2_PHONECALL"] = "nova-2-phonecall";
|
|
9
9
|
DeepgramModel["NOVA_2_CONVERSATIONALAI"] = "nova-2-conversationalai";
|
|
10
10
|
})(DeepgramModel || (exports.DeepgramModel = DeepgramModel = {}));
|
|
11
|
+
// Deepgram's code-switching mode: transcribes speech that mixes languages
|
|
12
|
+
// (e.g. a caller who answers "Hello" on a Spanish call). Nova-3 and Nova-2.
|
|
13
|
+
const DEEPGRAM_MULTILINGUAL = "multi";
|
|
14
|
+
exports.DEEPGRAM_MULTILINGUAL = DEEPGRAM_MULTILINGUAL;
|
|
@@ -18,5 +18,18 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import { Stream } from "stream";
|
|
20
20
|
import { AudioStream } from "@fonoster/streams";
|
|
21
|
-
|
|
21
|
+
import { AudioFilterChain } from "./filters";
|
|
22
|
+
/**
|
|
23
|
+
* Fans the caller's audio out to everything that consumes it: speech
|
|
24
|
+
* recognition and the application's media stream.
|
|
25
|
+
*
|
|
26
|
+
* When the session has audio filters, frames run through the chain first, so
|
|
27
|
+
* every consumer sees the same filtered audio. With no filters the buffer is
|
|
28
|
+
* emitted synchronously, exactly as before.
|
|
29
|
+
*
|
|
30
|
+
* Frames are emitted in arrival order even while filters are being replaced
|
|
31
|
+
* or removed: once anything is in flight, later frames queue behind it rather
|
|
32
|
+
* than overtaking it on the unfiltered path.
|
|
33
|
+
*/
|
|
34
|
+
declare function transcribeOnConnection(stream: Stream, getFilterChain?: () => AudioFilterChain | null): (_: any, res: AudioStream) => Promise<void>;
|
|
22
35
|
export { transcribeOnConnection };
|
|
@@ -11,13 +11,78 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.transcribeOnConnection = transcribeOnConnection;
|
|
13
13
|
const logger_1 = require("@fonoster/logger");
|
|
14
|
+
const filters_1 = require("./filters");
|
|
14
15
|
const logger = (0, logger_1.getLogger)({ service: "apiserver", filePath: __filename });
|
|
15
|
-
|
|
16
|
+
// About a second of audio. Past this the filter is not keeping up with the
|
|
17
|
+
// call, so frames skip it rather than piling up without bound.
|
|
18
|
+
const MAX_FRAMES_IN_FLIGHT = 50;
|
|
19
|
+
/**
|
|
20
|
+
* Fans the caller's audio out to everything that consumes it: speech
|
|
21
|
+
* recognition and the application's media stream.
|
|
22
|
+
*
|
|
23
|
+
* When the session has audio filters, frames run through the chain first, so
|
|
24
|
+
* every consumer sees the same filtered audio. With no filters the buffer is
|
|
25
|
+
* emitted synchronously, exactly as before.
|
|
26
|
+
*
|
|
27
|
+
* Frames are emitted in arrival order even while filters are being replaced
|
|
28
|
+
* or removed: once anything is in flight, later frames queue behind it rather
|
|
29
|
+
* than overtaking it on the unfiltered path.
|
|
30
|
+
*/
|
|
31
|
+
function transcribeOnConnection(stream, getFilterChain = () => null) {
|
|
16
32
|
return (_, res) => __awaiter(this, void 0, void 0, function* () {
|
|
17
33
|
logger.verbose("starting instance of audio stream for transcription");
|
|
34
|
+
let queue = Promise.resolve();
|
|
35
|
+
let inFlight = 0;
|
|
36
|
+
let warnedAboutBacklog = false;
|
|
37
|
+
// Listeners run synchronously here, so one that throws must not take the
|
|
38
|
+
// audio path down with it, and must never cause a frame to be emitted
|
|
39
|
+
// twice
|
|
40
|
+
const emit = (data) => {
|
|
41
|
+
try {
|
|
42
|
+
stream.emit("data", data);
|
|
43
|
+
}
|
|
44
|
+
catch (err) {
|
|
45
|
+
logger.warn("a consumer of the audio stream threw", { err });
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
const handle = (data) => __awaiter(this, void 0, void 0, function* () {
|
|
49
|
+
const chain = getFilterChain();
|
|
50
|
+
if (!chain) {
|
|
51
|
+
emit(data);
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
if (inFlight > MAX_FRAMES_IN_FLIGHT) {
|
|
55
|
+
if (!warnedAboutBacklog) {
|
|
56
|
+
warnedAboutBacklog = true;
|
|
57
|
+
logger.warn("audio filters are behind real time; frames are skipping them", { framesInFlight: inFlight });
|
|
58
|
+
}
|
|
59
|
+
emit(data);
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
try {
|
|
63
|
+
emit((0, filters_1.int16ToBuffer)(yield chain.process((0, filters_1.bufferToInt16)(data))));
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
logger.warn("audio filter chain failed; passing audio through", {
|
|
67
|
+
err
|
|
68
|
+
});
|
|
69
|
+
emit(data);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
18
72
|
res
|
|
19
73
|
.onData((data) => {
|
|
20
|
-
|
|
74
|
+
// Nothing pending and no filters: keep the original synchronous path
|
|
75
|
+
if (inFlight === 0 && !getFilterChain()) {
|
|
76
|
+
emit(data);
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
inFlight++;
|
|
80
|
+
queue = queue
|
|
81
|
+
.then(() => handle(data))
|
|
82
|
+
.catch((err) => logger.error("unexpected error in the audio path", { err }))
|
|
83
|
+
.finally(() => {
|
|
84
|
+
inFlight--;
|
|
85
|
+
});
|
|
21
86
|
})
|
|
22
87
|
.onError((err) => {
|
|
23
88
|
logger.error("stream error:", err);
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import { Readable, Stream } from "stream";
|
|
20
20
|
import { SayOptions, StreamContent, VoiceClientConfig, VoiceIn, VoiceSessionStreamClient } from "@fonoster/common";
|
|
21
21
|
import * as grpc from "@grpc/grpc-js";
|
|
22
|
+
import { AudioFilterConfig } from "../filters";
|
|
22
23
|
import { SpeechResult, StreamSpeech } from "../stt/types";
|
|
23
24
|
type VoiceClient = {
|
|
24
25
|
config: VoiceClientConfig;
|
|
@@ -46,6 +47,7 @@ type VoiceClient = {
|
|
|
46
47
|
digits: string;
|
|
47
48
|
}>;
|
|
48
49
|
getTranscriptionsStream: () => Stream;
|
|
50
|
+
setAudioFilters: (filters: AudioFilterConfig[]) => Promise<void>;
|
|
49
51
|
stopSynthesis: () => Promise<void>;
|
|
50
52
|
};
|
|
51
53
|
type TextToSpeech = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fonoster/apiserver",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.27.0",
|
|
4
4
|
"description": "APIServer for Fonoster",
|
|
5
5
|
"author": "Pedro Sanders <psanders@fonoster.com>",
|
|
6
6
|
"homepage": "https://github.com/fonoster/fonoster#readme",
|
|
@@ -22,14 +22,14 @@
|
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
24
|
"@deepgram/sdk": "^3.5.1",
|
|
25
|
-
"@fonoster/authz": "^0.
|
|
26
|
-
"@fonoster/autopilot": "^0.
|
|
27
|
-
"@fonoster/common": "^0.
|
|
28
|
-
"@fonoster/identity": "^0.
|
|
29
|
-
"@fonoster/logger": "^0.
|
|
30
|
-
"@fonoster/sipnet": "^0.
|
|
31
|
-
"@fonoster/streams": "^0.
|
|
32
|
-
"@fonoster/types": "^0.
|
|
25
|
+
"@fonoster/authz": "^0.27.0",
|
|
26
|
+
"@fonoster/autopilot": "^0.27.0",
|
|
27
|
+
"@fonoster/common": "^0.27.0",
|
|
28
|
+
"@fonoster/identity": "^0.27.0",
|
|
29
|
+
"@fonoster/logger": "^0.27.0",
|
|
30
|
+
"@fonoster/sipnet": "^0.27.0",
|
|
31
|
+
"@fonoster/streams": "^0.27.0",
|
|
32
|
+
"@fonoster/types": "^0.27.0",
|
|
33
33
|
"@google-cloud/speech": "^6.6.0",
|
|
34
34
|
"@google-cloud/text-to-speech": "^5.3.0",
|
|
35
35
|
"@grpc/grpc-js": "~1.10.11",
|
|
@@ -76,5 +76,8 @@
|
|
|
76
76
|
"@types/uuid": "^10.0.0",
|
|
77
77
|
"@types/validator": "^13.15.10"
|
|
78
78
|
},
|
|
79
|
-
"gitHead": "
|
|
79
|
+
"gitHead": "1191b34677b51a36000177887ff82303ef2693e1",
|
|
80
|
+
"optionalDependencies": {
|
|
81
|
+
"@ai-coustics/aic-sdk": "^0.24.0"
|
|
82
|
+
}
|
|
80
83
|
}
|