@fonoster/autopilot 0.27.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -51
- package/dist/index.d.ts +0 -5
- package/dist/index.js +0 -5
- package/dist/knowledge/AbstractKnowledgeBase.d.ts +3 -2
- package/dist/knowledge/AbstractKnowledgeBase.js +2 -7
- package/dist/knowledge/FilesKnowledgeBase.d.ts +1 -1
- package/dist/knowledge/types.d.ts +1 -1
- package/dist/models/LanguageModelFactory.d.ts +5 -24
- package/dist/models/LanguageModelFactory.js +0 -1
- package/dist/models/createLanguageModel.d.ts +2 -2
- package/dist/models/evaluations/createEvalEffectiveConfig.d.ts +10 -106
- package/dist/models/evaluations/createEvalEffectiveConfig.js +47 -7
- package/dist/models/evaluations/createEvaluateIntelligence.d.ts +10 -20
- package/dist/models/evaluations/createEvaluateIntelligence.js +32 -23
- package/dist/models/evaluations/evalTestCases.d.ts +2 -2
- package/dist/models/evaluations/extractAiResponse.d.ts +2 -3
- package/dist/models/evaluations/index.d.ts +1 -0
- package/dist/models/evaluations/index.js +1 -0
- package/dist/models/evaluations/runEval.d.ts +2 -20
- package/dist/models/evaluations/session/EvalSession.d.ts +73 -0
- package/dist/models/evaluations/session/EvalSession.js +134 -0
- package/dist/models/evaluations/session/evaluateSessionScenario.d.ts +58 -0
- package/dist/models/evaluations/session/evaluateSessionScenario.js +182 -0
- package/dist/{vad → models/evaluations/session}/index.d.ts +3 -2
- package/dist/{vad → models/evaluations/session}/index.js +3 -2
- package/dist/models/evaluations/session/runSessionEval.d.ts +17 -0
- package/dist/models/evaluations/session/runSessionEval.js +59 -0
- package/dist/models/evaluations/types.d.ts +7 -6
- package/dist/models/types.d.ts +19 -1
- package/dist/session/Agent.d.ts +138 -0
- package/dist/session/Agent.js +474 -0
- package/dist/{loadAssistantFromAPI.d.ts → session/AsyncQueue.d.ts} +12 -7
- package/dist/session/AsyncQueue.js +67 -0
- package/dist/{machine/guards/index.d.ts → session/config.d.ts} +2 -2
- package/dist/session/config.js +27 -0
- package/dist/session/convertLegacyAssistant.d.ts +77 -0
- package/dist/session/convertLegacyAssistant.js +118 -0
- package/dist/session/createAutopilot.d.ts +47 -0
- package/dist/session/createAutopilot.js +119 -0
- package/dist/session/hosted.d.ts +456 -0
- package/dist/session/hosted.js +506 -0
- package/dist/{assistants → session}/index.d.ts +11 -2
- package/dist/{machine/actors → session}/index.js +11 -3
- package/dist/{machine/actions/appendSpeech.js → session/knowledge.d.ts} +14 -17
- package/dist/session/knowledge.js +10 -0
- package/dist/session/llm.d.ts +86 -0
- package/dist/session/llm.js +83 -0
- package/dist/session/machine.d.ts +206 -0
- package/dist/session/machine.js +126 -0
- package/dist/session/main.js +44 -0
- package/dist/session/sentences.d.ts +5 -0
- package/dist/session/sentences.js +53 -0
- package/dist/{machine/actors/index.d.ts → session/think.d.ts} +2 -3
- package/dist/{machine/actions/announceSystemError.js → session/think.js} +8 -10
- package/dist/session/tools.d.ts +22 -0
- package/dist/session/tools.js +83 -0
- package/dist/tools/builtin/transferToolDefinition.js +1 -1
- package/dist/voice/index.d.ts +0 -1
- package/dist/voice/index.js +0 -1
- package/dist/voice/types.d.ts +5 -18
- package/package.json +13 -14
- package/dist/Autopilot.d.ts +0 -14
- package/dist/Autopilot.js +0 -133
- package/dist/assistants/index.js +0 -36
- package/dist/assistants/loadAndValidateAssistant.d.ts +0 -3
- package/dist/assistants/loadAndValidateAssistant.js +0 -49
- package/dist/assistants/types.d.ts +0 -24
- package/dist/assistants/types.js +0 -2
- package/dist/envs.d.ts +0 -15
- package/dist/envs.js +0 -78
- package/dist/handleVoiceRequest.d.ts +0 -3
- package/dist/handleVoiceRequest.js +0 -167
- package/dist/loadAssistantConfigFromFile.d.ts +0 -3
- package/dist/loadAssistantConfigFromFile.js +0 -68
- package/dist/loadAssistantFromAPI.js +0 -98
- package/dist/machine/actions/announceSystemError.d.ts +0 -4
- package/dist/machine/actions/appendSpeech.d.ts +0 -1
- package/dist/machine/actions/cleanSpeech.d.ts +0 -1
- package/dist/machine/actions/cleanSpeech.js +0 -23
- package/dist/machine/actions/goodbye.d.ts +0 -4
- package/dist/machine/actions/goodbye.js +0 -31
- package/dist/machine/actions/increaseIdleTimeoutCount.d.ts +0 -1
- package/dist/machine/actions/increaseIdleTimeoutCount.js +0 -31
- package/dist/machine/actions/index.d.ts +0 -26
- package/dist/machine/actions/index.js +0 -37
- package/dist/machine/actions/interruptPlayback.d.ts +0 -4
- package/dist/machine/actions/interruptPlayback.js +0 -43
- package/dist/machine/actions/resetIdleTimeoutCount.d.ts +0 -1
- package/dist/machine/actions/resetIdleTimeoutCount.js +0 -31
- package/dist/machine/actions/resetState.d.ts +0 -1
- package/dist/machine/actions/resetState.js +0 -32
- package/dist/machine/actors/doAnnounceIdleTimeout.d.ts +0 -4
- package/dist/machine/actors/doAnnounceIdleTimeout.js +0 -41
- package/dist/machine/actors/doGreetUser.d.ts +0 -4
- package/dist/machine/actors/doGreetUser.js +0 -45
- package/dist/machine/actors/doProcessUserRequest.d.ts +0 -4
- package/dist/machine/actors/doProcessUserRequest.js +0 -75
- package/dist/machine/context.d.ts +0 -52
- package/dist/machine/context.js +0 -28
- package/dist/machine/delays.d.ts +0 -30
- package/dist/machine/delays.js +0 -37
- package/dist/machine/guards/hasSpeechResult.d.ts +0 -4
- package/dist/machine/guards/hasSpeechResult.js +0 -30
- package/dist/machine/guards/idleTimeoutCountExceedsMax.d.ts +0 -4
- package/dist/machine/guards/idleTimeoutCountExceedsMax.js +0 -31
- package/dist/machine/guards/index.js +0 -25
- package/dist/machine/index.d.ts +0 -19
- package/dist/machine/index.js +0 -23
- package/dist/machine/machine.d.ts +0 -408
- package/dist/machine/machine.js +0 -206
- package/dist/machine/setup.d.ts +0 -328
- package/dist/machine/setup.js +0 -116
- package/dist/machine/types.d.ts +0 -54
- package/dist/machine/types.js +0 -2
- package/dist/sendConversationEndedEvent.d.ts +0 -32
- package/dist/sendConversationEndedEvent.js +0 -66
- package/dist/sendConversationStartedEvent.d.ts +0 -6
- package/dist/sendConversationStartedEvent.js +0 -63
- package/dist/server.d.ts +0 -2
- package/dist/server.js +0 -25
- package/dist/types.d.ts +0 -40
- package/dist/types.js +0 -8
- package/dist/vad/SileroVad.d.ts +0 -18
- package/dist/vad/SileroVad.js +0 -40
- package/dist/vad/SileroVadModel.d.ts +0 -14
- package/dist/vad/SileroVadModel.js +0 -83
- package/dist/vad/chunkToFloat32Array.d.ts +0 -20
- package/dist/vad/chunkToFloat32Array.js +0 -34
- package/dist/vad/createVad.d.ts +0 -3
- package/dist/vad/createVad.js +0 -126
- package/dist/vad/types.d.ts +0 -60
- package/dist/vad/types.js +0 -2
- package/dist/vadWorker.js +0 -36
- package/dist/voice/Voice.d.ts +0 -61
- package/dist/voice/Voice.js +0 -96
- package/dist/voiceServerSetup.d.ts +0 -2
- package/dist/voiceServerSetup.js +0 -29
- /package/dist/{vadWorker.d.ts → session/main.d.ts} +0 -0
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.sendConversationStartedEvent = sendConversationStartedEvent;
|
|
4
|
-
/**
|
|
5
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
6
|
-
* http://github.com/fonoster/fonoster
|
|
7
|
-
*
|
|
8
|
-
* This file is part of Fonoster
|
|
9
|
-
*
|
|
10
|
-
* Licensed under the MIT License (the "License");
|
|
11
|
-
* you may not use this file except in compliance with
|
|
12
|
-
* the License. You may obtain a copy of the License at
|
|
13
|
-
*
|
|
14
|
-
* https://opensource.org/licenses/MIT
|
|
15
|
-
*
|
|
16
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
-
* See the License for the specific language governing permissions and
|
|
20
|
-
* limitations under the License.
|
|
21
|
-
*/
|
|
22
|
-
const common_1 = require("@fonoster/common");
|
|
23
|
-
const logger_1 = require("@fonoster/logger");
|
|
24
|
-
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
25
|
-
async function sendConversationStartedEvent(eventsHook, data) {
|
|
26
|
-
const { phone, appRef, callRef } = data;
|
|
27
|
-
if (!eventsHook?.events.includes(common_1.EventsHookAllowedEvents.CONVERSATION_STARTED) &&
|
|
28
|
-
!eventsHook?.events.includes(common_1.EventsHookAllowedEvents.ALL)) {
|
|
29
|
-
return;
|
|
30
|
-
}
|
|
31
|
-
const parsedEventsHook = common_1.eventsHookSchema.parse(eventsHook);
|
|
32
|
-
const params = {
|
|
33
|
-
eventType: common_1.EventsHookAllowedEvents.CONVERSATION_STARTED,
|
|
34
|
-
appRef,
|
|
35
|
-
callRef,
|
|
36
|
-
phone
|
|
37
|
-
};
|
|
38
|
-
logger.verbose("dispatching conversation.started webhook", {
|
|
39
|
-
url: parsedEventsHook.url,
|
|
40
|
-
eventType: params.eventType,
|
|
41
|
-
appRef,
|
|
42
|
-
callRef
|
|
43
|
-
});
|
|
44
|
-
try {
|
|
45
|
-
await (0, common_1.sendHttpRequest)({
|
|
46
|
-
url: parsedEventsHook.url,
|
|
47
|
-
method: common_1.AllowedHttpMethod.POST,
|
|
48
|
-
headers: parsedEventsHook.headers,
|
|
49
|
-
waitForResponse: false,
|
|
50
|
-
params
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
catch (e) {
|
|
54
|
-
logger.error("failed to send conversation.started webhook", {
|
|
55
|
-
url: parsedEventsHook.url,
|
|
56
|
-
method: common_1.AllowedHttpMethod.POST,
|
|
57
|
-
waitForResponse: false,
|
|
58
|
-
appRef,
|
|
59
|
-
callRef,
|
|
60
|
-
error: e instanceof Error ? e.message : e
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
}
|
package/dist/server.d.ts
DELETED
package/dist/server.js
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
"use strict";
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
/**
|
|
5
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
6
|
-
* http://github.com/fonoster/fonoster
|
|
7
|
-
*
|
|
8
|
-
* This file is part of Fonoster
|
|
9
|
-
*
|
|
10
|
-
* Licensed under the MIT License (the "License");
|
|
11
|
-
* you may not use this file except in compliance with
|
|
12
|
-
* the License. You may obtain a copy of the License at
|
|
13
|
-
*
|
|
14
|
-
* https://opensource.org/licenses/MIT
|
|
15
|
-
*
|
|
16
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
-
* See the License for the specific language governing permissions and
|
|
20
|
-
* limitations under the License.
|
|
21
|
-
*/
|
|
22
|
-
const envs_1 = require("./envs");
|
|
23
|
-
const voiceServerSetup_1 = require("./voiceServerSetup");
|
|
24
|
-
const skipIdentity = envs_1.NODE_ENV === "development" || envs_1.SKIP_IDENTITY;
|
|
25
|
-
(0, voiceServerSetup_1.startVoiceServer)(skipIdentity);
|
package/dist/types.d.ts
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
-
* http://github.com/fonoster/fonoster
|
|
4
|
-
*
|
|
5
|
-
* This file is part of Fonoster
|
|
6
|
-
*
|
|
7
|
-
* Licensed under the MIT License (the "License");
|
|
8
|
-
* you may not use this file except in compliance with
|
|
9
|
-
* the License. You may obtain a copy of the License at
|
|
10
|
-
*
|
|
11
|
-
* https://opensource.org/licenses/MIT
|
|
12
|
-
*
|
|
13
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
* See the License for the specific language governing permissions and
|
|
17
|
-
* limitations under the License.
|
|
18
|
-
*/
|
|
19
|
-
import { AudioFilterConfig } from "@fonoster/common";
|
|
20
|
-
import { Application } from "@fonoster/types";
|
|
21
|
-
import { AssistantConfig, ConversationSettings } from "./assistants";
|
|
22
|
-
import { LanguageModel } from "./models";
|
|
23
|
-
import { Voice } from "./voice";
|
|
24
|
-
declare enum ConversationProvider {
|
|
25
|
-
FILE = "file",
|
|
26
|
-
API = "api"
|
|
27
|
-
}
|
|
28
|
-
type AutopilotParams = {
|
|
29
|
-
voice: Voice;
|
|
30
|
-
conversationSettings: ConversationSettings;
|
|
31
|
-
languageModel: LanguageModel;
|
|
32
|
-
audioFilters?: AudioFilterConfig[];
|
|
33
|
-
};
|
|
34
|
-
type AutopilotApplication = Application & {
|
|
35
|
-
intelligence: {
|
|
36
|
-
productRef: string;
|
|
37
|
-
config: AssistantConfig;
|
|
38
|
-
};
|
|
39
|
-
};
|
|
40
|
-
export { AutopilotParams, ConversationProvider, AutopilotApplication };
|
package/dist/types.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.ConversationProvider = void 0;
|
|
4
|
-
var ConversationProvider;
|
|
5
|
-
(function (ConversationProvider) {
|
|
6
|
-
ConversationProvider["FILE"] = "file";
|
|
7
|
-
ConversationProvider["API"] = "api";
|
|
8
|
-
})(ConversationProvider || (exports.ConversationProvider = ConversationProvider = {}));
|
package/dist/vad/SileroVad.d.ts
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
import { Vad } from "./types";
|
|
2
|
-
declare class SileroVad implements Vad {
|
|
3
|
-
private vad;
|
|
4
|
-
private readonly params;
|
|
5
|
-
constructor(params: {
|
|
6
|
-
pathToModel: string;
|
|
7
|
-
activationThreshold: number;
|
|
8
|
-
deactivationThreshold: number;
|
|
9
|
-
debounceFrames: number;
|
|
10
|
-
});
|
|
11
|
-
pathToModel: string;
|
|
12
|
-
activationThreshold: number;
|
|
13
|
-
deactivationThreshold: number;
|
|
14
|
-
debounceFrames: number;
|
|
15
|
-
init(): Promise<void>;
|
|
16
|
-
processChunk(data: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void): void;
|
|
17
|
-
}
|
|
18
|
-
export { SileroVad };
|
package/dist/vad/SileroVad.js
DELETED
|
@@ -1,40 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.SileroVad = void 0;
|
|
4
|
-
/**
|
|
5
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
6
|
-
* http://github.com/fonoster/fonoster
|
|
7
|
-
*
|
|
8
|
-
* This file is part of Fonoster
|
|
9
|
-
*
|
|
10
|
-
* Licensed under the MIT License (the "License");
|
|
11
|
-
* you may not use this file except in compliance with
|
|
12
|
-
* the License. You may obtain a copy of the License at
|
|
13
|
-
*
|
|
14
|
-
* https://opensource.org/licenses/MIT
|
|
15
|
-
*
|
|
16
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
-
* See the License for the specific language governing permissions and
|
|
20
|
-
* limitations under the License.
|
|
21
|
-
*/
|
|
22
|
-
const logger_1 = require("@fonoster/logger");
|
|
23
|
-
const createVad_1 = require("./createVad");
|
|
24
|
-
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
25
|
-
class SileroVad {
|
|
26
|
-
constructor(params) {
|
|
27
|
-
logger.verbose("starting instance of silero vad v5", { ...params });
|
|
28
|
-
this.params = params;
|
|
29
|
-
}
|
|
30
|
-
async init() {
|
|
31
|
-
this.vad = await (0, createVad_1.createVad)(this.params);
|
|
32
|
-
}
|
|
33
|
-
processChunk(data, callback) {
|
|
34
|
-
if (!this.vad) {
|
|
35
|
-
throw new Error("VAD not initialized");
|
|
36
|
-
}
|
|
37
|
-
this.vad(data, callback);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
exports.SileroVad = SileroVad;
|
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import { ONNXRuntimeAPI, SpeechProbabilities } from "./types";
|
|
2
|
-
declare class SileroVadModel {
|
|
3
|
-
private readonly ort;
|
|
4
|
-
private readonly pathToModel;
|
|
5
|
-
private _session;
|
|
6
|
-
private _state;
|
|
7
|
-
private _sr;
|
|
8
|
-
constructor(ort: ONNXRuntimeAPI, pathToModel: string);
|
|
9
|
-
static readonly new: (ort: ONNXRuntimeAPI, pathToModel: string) => Promise<SileroVadModel>;
|
|
10
|
-
init(): Promise<void>;
|
|
11
|
-
resetState: () => void;
|
|
12
|
-
process(audioFrame: Float32Array): Promise<SpeechProbabilities>;
|
|
13
|
-
}
|
|
14
|
-
export { SileroVadModel };
|
|
@@ -1,83 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var _a;
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.SileroVadModel = void 0;
|
|
5
|
-
/**
|
|
6
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
7
|
-
* http://github.com/fonoster/fonoster
|
|
8
|
-
*
|
|
9
|
-
* This file is part of Fonoster
|
|
10
|
-
*
|
|
11
|
-
* Licensed under the MIT License (the "License");
|
|
12
|
-
* you may not use this file except in compliance with
|
|
13
|
-
* the License. You may obtain a copy of the License at
|
|
14
|
-
*
|
|
15
|
-
* https://opensource.org/licenses/MIT
|
|
16
|
-
*
|
|
17
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
18
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
19
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
20
|
-
* See the License for the specific language governing permissions and
|
|
21
|
-
* limitations under the License.
|
|
22
|
-
*/
|
|
23
|
-
const fs_1 = require("fs");
|
|
24
|
-
const SAMPLE_RATE = 16000;
|
|
25
|
-
function getNewState(ortInstance) {
|
|
26
|
-
return new ortInstance.Tensor("float32", new Float32Array(2 * 1 * 128), // Use Float32Array for consistency
|
|
27
|
-
[2, 1, 128]);
|
|
28
|
-
}
|
|
29
|
-
class SileroVadModel {
|
|
30
|
-
constructor(ort, pathToModel) {
|
|
31
|
-
this.ort = ort;
|
|
32
|
-
this.pathToModel = pathToModel;
|
|
33
|
-
this.resetState = () => {
|
|
34
|
-
this._state = getNewState(this.ort);
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
async init() {
|
|
38
|
-
const modelArrayBuffer = (0, fs_1.readFileSync)(this.pathToModel).buffer;
|
|
39
|
-
const sessionOption = {
|
|
40
|
-
interOpNumThreads: 1,
|
|
41
|
-
intraOpNumThreads: 1,
|
|
42
|
-
enableCpuMemArena: false
|
|
43
|
-
};
|
|
44
|
-
this._session = await this.ort.InferenceSession.create(modelArrayBuffer, sessionOption);
|
|
45
|
-
// Validate model inputs/outputs
|
|
46
|
-
const requiredInputs = ["input", "state", "sr"];
|
|
47
|
-
for (const name of requiredInputs) {
|
|
48
|
-
if (!this._session.inputNames.includes(name)) {
|
|
49
|
-
throw new Error(`Model is missing expected input "${name}"`);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
if (!this._session.outputNames.includes("output") ||
|
|
53
|
-
!this._session.outputNames.includes("stateN")) {
|
|
54
|
-
throw new Error("Model is missing expected outputs");
|
|
55
|
-
}
|
|
56
|
-
// Use BigInt for sample rate tensor
|
|
57
|
-
this._sr = new this.ort.Tensor("int64", [BigInt(SAMPLE_RATE)], []);
|
|
58
|
-
this._state = getNewState(this.ort);
|
|
59
|
-
}
|
|
60
|
-
async process(audioFrame) {
|
|
61
|
-
const inputTensor = new this.ort.Tensor("float32", audioFrame, [
|
|
62
|
-
1,
|
|
63
|
-
audioFrame.length
|
|
64
|
-
]);
|
|
65
|
-
const feeds = {
|
|
66
|
-
input: inputTensor,
|
|
67
|
-
state: this._state,
|
|
68
|
-
sr: this._sr
|
|
69
|
-
};
|
|
70
|
-
const out = await this._session.run(feeds);
|
|
71
|
-
this._state = out.stateN;
|
|
72
|
-
const [isSpeech] = out.output.data;
|
|
73
|
-
const notSpeech = 1 - isSpeech;
|
|
74
|
-
return { notSpeech, isSpeech };
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
exports.SileroVadModel = SileroVadModel;
|
|
78
|
-
_a = SileroVadModel;
|
|
79
|
-
SileroVadModel.new = async (ort, pathToModel) => {
|
|
80
|
-
const model = new _a(ort, pathToModel);
|
|
81
|
-
await model.init();
|
|
82
|
-
return model;
|
|
83
|
-
};
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
-
* http://github.com/fonoster/fonoster
|
|
4
|
-
*
|
|
5
|
-
* This file is part of Fonoster
|
|
6
|
-
*
|
|
7
|
-
* Licensed under the MIT License (the "License");
|
|
8
|
-
* you may not use this file except in compliance with
|
|
9
|
-
* the License. You may obtain a copy of the License at
|
|
10
|
-
*
|
|
11
|
-
* https://opensource.org/licenses/MIT
|
|
12
|
-
*
|
|
13
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
* See the License for the specific language governing permissions and
|
|
17
|
-
* limitations under the License.
|
|
18
|
-
*/
|
|
19
|
-
declare function chunkToFloat32Array(chunk: Uint8Array): Float32Array;
|
|
20
|
-
export { chunkToFloat32Array };
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.chunkToFloat32Array = chunkToFloat32Array;
|
|
4
|
-
/**
|
|
5
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
6
|
-
* http://github.com/fonoster/fonoster
|
|
7
|
-
*
|
|
8
|
-
* This file is part of Fonoster
|
|
9
|
-
*
|
|
10
|
-
* Licensed under the MIT License (the "License");
|
|
11
|
-
* you may not use this file except in compliance with
|
|
12
|
-
* the License. You may obtain a copy of the License at
|
|
13
|
-
*
|
|
14
|
-
* https://opensource.org/licenses/MIT
|
|
15
|
-
*
|
|
16
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
17
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
18
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
19
|
-
* See the License for the specific language governing permissions and
|
|
20
|
-
* limitations under the License.
|
|
21
|
-
*/
|
|
22
|
-
function chunkToFloat32Array(chunk) {
|
|
23
|
-
// Create a DataView to handle endianness explicitly
|
|
24
|
-
const dataView = new DataView(chunk.buffer, chunk.byteOffset, chunk.byteLength);
|
|
25
|
-
const floatArray = new Float32Array(chunk.byteLength / 2);
|
|
26
|
-
// Convert each 16-bit sample to float32, explicitly handling little-endian
|
|
27
|
-
for (let i = 0; i < floatArray.length; i++) {
|
|
28
|
-
// Read 16-bit value with explicit little-endian
|
|
29
|
-
const int16Value = dataView.getInt16(i * 2, true); // true = little-endian
|
|
30
|
-
// Normalize to [-1, 1]
|
|
31
|
-
floatArray[i] = int16Value / 32768.0;
|
|
32
|
-
}
|
|
33
|
-
return floatArray;
|
|
34
|
-
}
|
package/dist/vad/createVad.d.ts
DELETED
package/dist/vad/createVad.js
DELETED
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.createVad = createVad;
|
|
37
|
-
/**
|
|
38
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
39
|
-
* http://github.com/fonoster/fonoster
|
|
40
|
-
*
|
|
41
|
-
* This file is part of Fonoster
|
|
42
|
-
*
|
|
43
|
-
* Licensed under the MIT License (the "License");
|
|
44
|
-
* you may not use this file except in compliance with
|
|
45
|
-
* the License. You may obtain a copy of the License at
|
|
46
|
-
*
|
|
47
|
-
* https://opensource.org/licenses/MIT
|
|
48
|
-
*
|
|
49
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
50
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
51
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
52
|
-
* See the License for the specific language governing permissions and
|
|
53
|
-
* limitations under the License.
|
|
54
|
-
*/
|
|
55
|
-
const path_1 = require("path");
|
|
56
|
-
const logger_1 = require("@fonoster/logger");
|
|
57
|
-
const ort = __importStar(require("onnxruntime-node"));
|
|
58
|
-
const chunkToFloat32Array_1 = require("./chunkToFloat32Array");
|
|
59
|
-
const SileroVadModel_1 = require("./SileroVadModel");
|
|
60
|
-
const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
|
|
61
|
-
const FULL_FRAME_SIZE = 1024; // 64ms @ 16kHz
|
|
62
|
-
const BUFFER_SIZE = 512; // 32ms @ 16kHz
|
|
63
|
-
async function createVad(params) {
|
|
64
|
-
const { pathToModel, activationThreshold, deactivationThreshold, debounceFrames } = params;
|
|
65
|
-
const effectivePath = pathToModel || (0, path_1.join)(__dirname, "..", "..", "silero_vad_v5.onnx");
|
|
66
|
-
const ortAdapter = {
|
|
67
|
-
InferenceSession: {
|
|
68
|
-
create: ort.InferenceSession.create.bind(ort.InferenceSession)
|
|
69
|
-
},
|
|
70
|
-
Tensor: ort.Tensor
|
|
71
|
-
};
|
|
72
|
-
const silero = await SileroVadModel_1.SileroVadModel.new(ortAdapter, effectivePath);
|
|
73
|
-
let sampleBuffer = [];
|
|
74
|
-
let isSpeechActive = false;
|
|
75
|
-
let framesSinceStateChange = 0;
|
|
76
|
-
// Reset internal state after a state change.
|
|
77
|
-
const resetState = () => {
|
|
78
|
-
isSpeechActive = false;
|
|
79
|
-
framesSinceStateChange = 0;
|
|
80
|
-
// Clear any pending audio samples to avoid using outdated values.
|
|
81
|
-
sampleBuffer = [];
|
|
82
|
-
silero.resetState();
|
|
83
|
-
logger.silly("State reset -- sampleBuffer cleared");
|
|
84
|
-
};
|
|
85
|
-
return async function process(chunk, callback) {
|
|
86
|
-
// Convert the incoming chunk to normalized Float32 samples (using chunkToFloat32Array)
|
|
87
|
-
const float32Array = (0, chunkToFloat32Array_1.chunkToFloat32Array)(chunk);
|
|
88
|
-
sampleBuffer.push(...float32Array);
|
|
89
|
-
// Wait until we've collected a full frame worth of samples.
|
|
90
|
-
while (sampleBuffer.length >= FULL_FRAME_SIZE) {
|
|
91
|
-
const fullFrame = sampleBuffer.slice(0, FULL_FRAME_SIZE);
|
|
92
|
-
sampleBuffer = sampleBuffer.slice(FULL_FRAME_SIZE);
|
|
93
|
-
// Use the last BUFFER_SIZE samples from the full frame.
|
|
94
|
-
const frame = fullFrame.slice(fullFrame.length - BUFFER_SIZE);
|
|
95
|
-
const result = await silero.process(new Float32Array(frame));
|
|
96
|
-
const rawScore = result.isSpeech;
|
|
97
|
-
logger.silly("Frame processing", {
|
|
98
|
-
rawScore,
|
|
99
|
-
isSpeechActive,
|
|
100
|
-
framesSinceStateChange,
|
|
101
|
-
pendingSamples: sampleBuffer.length
|
|
102
|
-
});
|
|
103
|
-
framesSinceStateChange++;
|
|
104
|
-
if (isSpeechActive) {
|
|
105
|
-
// If already in speech, check if the score has dropped below deactivationThreshold
|
|
106
|
-
if (rawScore < deactivationThreshold &&
|
|
107
|
-
framesSinceStateChange >= debounceFrames) {
|
|
108
|
-
callback("SPEECH_END");
|
|
109
|
-
resetState();
|
|
110
|
-
logger.silly("Speech end detected", { rawScore });
|
|
111
|
-
continue;
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
else {
|
|
115
|
-
// If currently not speaking, check if the score is above activationThreshold
|
|
116
|
-
if (rawScore > activationThreshold &&
|
|
117
|
-
framesSinceStateChange >= debounceFrames) {
|
|
118
|
-
isSpeechActive = true;
|
|
119
|
-
framesSinceStateChange = 0;
|
|
120
|
-
callback("SPEECH_START");
|
|
121
|
-
logger.silly("Speech start detected", { rawScore });
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
};
|
|
126
|
-
}
|
package/dist/vad/types.d.ts
DELETED
|
@@ -1,60 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
-
* http://github.com/fonoster/fonoster
|
|
4
|
-
*
|
|
5
|
-
* This file is part of Fonoster
|
|
6
|
-
*
|
|
7
|
-
* Licensed under the MIT License (the "License");
|
|
8
|
-
* you may not use this file except in compliance with
|
|
9
|
-
* the License. You may obtain a copy of the License at
|
|
10
|
-
*
|
|
11
|
-
* https://opensource.org/licenses/MIT
|
|
12
|
-
*
|
|
13
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
* See the License for the specific language governing permissions and
|
|
17
|
-
* limitations under the License.
|
|
18
|
-
*/
|
|
19
|
-
type VadEvent = "SPEECH_START" | "SPEECH_END";
|
|
20
|
-
type Vad = {
|
|
21
|
-
processChunk: (chunk: Uint8Array, callback: (event: VadEvent) => void) => void;
|
|
22
|
-
};
|
|
23
|
-
type VadParams = {
|
|
24
|
-
pathToModel?: string;
|
|
25
|
-
activationThreshold: number;
|
|
26
|
-
deactivationThreshold: number;
|
|
27
|
-
debounceFrames: number;
|
|
28
|
-
};
|
|
29
|
-
export interface SpeechProbabilities {
|
|
30
|
-
notSpeech: number;
|
|
31
|
-
isSpeech: number;
|
|
32
|
-
}
|
|
33
|
-
export interface ONNXRuntimeAPI {
|
|
34
|
-
InferenceSession: {
|
|
35
|
-
create: (modelPath: ArrayBuffer | string, options?: {
|
|
36
|
-
interOpNumThreads: number;
|
|
37
|
-
intraOpNumThreads: number;
|
|
38
|
-
enableCpuMemArena: boolean;
|
|
39
|
-
}) => Promise<ONNXSession>;
|
|
40
|
-
};
|
|
41
|
-
Tensor: new (type: string, data: Float32Array | bigint[], dims: number[]) => ONNXTensor;
|
|
42
|
-
}
|
|
43
|
-
export interface ONNXSession {
|
|
44
|
-
run: (feeds: {
|
|
45
|
-
[key: string]: ONNXTensor;
|
|
46
|
-
}) => Promise<{
|
|
47
|
-
output: {
|
|
48
|
-
data: Float32Array;
|
|
49
|
-
};
|
|
50
|
-
stateN: ONNXTensor;
|
|
51
|
-
}>;
|
|
52
|
-
inputNames: string[];
|
|
53
|
-
outputNames: string[];
|
|
54
|
-
}
|
|
55
|
-
export interface ONNXTensor {
|
|
56
|
-
data: Float32Array | bigint[];
|
|
57
|
-
dims: number[];
|
|
58
|
-
type: string;
|
|
59
|
-
}
|
|
60
|
-
export { Vad, VadEvent, VadParams };
|
package/dist/vad/types.js
DELETED
package/dist/vadWorker.js
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/**
|
|
4
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
5
|
-
* http://github.com/fonoster/fonoster
|
|
6
|
-
*
|
|
7
|
-
* This file is part of Fonoster
|
|
8
|
-
*
|
|
9
|
-
* Licensed under the MIT License (the "License");
|
|
10
|
-
* you may not use this file except in compliance with
|
|
11
|
-
* the License. You may obtain a copy of the License at
|
|
12
|
-
*
|
|
13
|
-
* https://opensource.org/licenses/MIT
|
|
14
|
-
*
|
|
15
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
16
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
17
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
18
|
-
* See the License for the specific language governing permissions and
|
|
19
|
-
* limitations under the License.
|
|
20
|
-
*/
|
|
21
|
-
const path_1 = require("path");
|
|
22
|
-
const worker_threads_1 = require("worker_threads");
|
|
23
|
-
const SileroVad_1 = require("./vad/SileroVad");
|
|
24
|
-
const vad = new SileroVad_1.SileroVad({
|
|
25
|
-
...worker_threads_1.workerData,
|
|
26
|
-
pathToModel: (0, path_1.join)(__dirname, "..", "silero_vad_v5.onnx")
|
|
27
|
-
});
|
|
28
|
-
vad.init().then(() => {
|
|
29
|
-
// Send ready message to parent
|
|
30
|
-
worker_threads_1.parentPort?.postMessage("VAD_READY");
|
|
31
|
-
worker_threads_1.parentPort?.on("message", (chunk) => {
|
|
32
|
-
vad.processChunk(chunk, (voiceActivity) => {
|
|
33
|
-
worker_threads_1.parentPort?.postMessage(voiceActivity);
|
|
34
|
-
});
|
|
35
|
-
});
|
|
36
|
-
});
|
package/dist/voice/Voice.d.ts
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Copyright (C) 2025 by Fonoster Inc (https://fonoster.com)
|
|
3
|
-
* http://github.com/fonoster/fonoster
|
|
4
|
-
*
|
|
5
|
-
* This file is part of Fonoster
|
|
6
|
-
*
|
|
7
|
-
* Licensed under the MIT License (the "License");
|
|
8
|
-
* you may not use this file except in compliance with
|
|
9
|
-
* the License. You may obtain a copy of the License at
|
|
10
|
-
*
|
|
11
|
-
* https://opensource.org/licenses/MIT
|
|
12
|
-
*
|
|
13
|
-
* Unless required by applicable law or agreed to in writing, software
|
|
14
|
-
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
-
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
-
* See the License for the specific language governing permissions and
|
|
17
|
-
* limitations under the License.
|
|
18
|
-
*/
|
|
19
|
-
import { AudioFilterConfig } from "@fonoster/common";
|
|
20
|
-
import { VoiceResponse } from "@fonoster/voice";
|
|
21
|
-
import { Voice } from "./types";
|
|
22
|
-
declare class VoiceImpl implements Voice {
|
|
23
|
-
private readonly voice;
|
|
24
|
-
private readonly playbackRef;
|
|
25
|
-
mediaSessionRef: string;
|
|
26
|
-
sgatherStream: {
|
|
27
|
-
stop: () => Promise<void>;
|
|
28
|
-
onData: (cb: (payload: {
|
|
29
|
-
speech: string;
|
|
30
|
-
responseTime: number;
|
|
31
|
-
}) => void) => void;
|
|
32
|
-
};
|
|
33
|
-
vadStream: {
|
|
34
|
-
stop: () => Promise<void>;
|
|
35
|
-
onData: (cb: (chunk: Uint8Array) => void) => void;
|
|
36
|
-
};
|
|
37
|
-
constructor(mediaSessionRef: string, voice: VoiceResponse);
|
|
38
|
-
answer(): Promise<void>;
|
|
39
|
-
setAudioFilters(filters: AudioFilterConfig[]): Promise<void>;
|
|
40
|
-
hangup(): Promise<void>;
|
|
41
|
-
say(text: string): Promise<void>;
|
|
42
|
-
playDtmf(dtmf: string): Promise<void>;
|
|
43
|
-
sgather(): Promise<{
|
|
44
|
-
stop: () => Promise<void>;
|
|
45
|
-
onData: (cb: (payload: {
|
|
46
|
-
speech: string;
|
|
47
|
-
responseTime: number;
|
|
48
|
-
}) => void) => void;
|
|
49
|
-
}>;
|
|
50
|
-
stream(): Promise<{
|
|
51
|
-
stop: () => Promise<void>;
|
|
52
|
-
onData: (cb: (chunk: Uint8Array) => void) => void;
|
|
53
|
-
}>;
|
|
54
|
-
transfer(to: string, options: {
|
|
55
|
-
record: boolean;
|
|
56
|
-
timeout: number;
|
|
57
|
-
}): Promise<void>;
|
|
58
|
-
stopSpeech(): Promise<void>;
|
|
59
|
-
stopStreams(): Promise<void>;
|
|
60
|
-
}
|
|
61
|
-
export { VoiceImpl };
|