@fonoster/autopilot 0.7.3 → 0.7.5

package/dist/tools/sendRequest.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sendRequest = sendRequest;
+/*
+ * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const zod_1 = require("zod");
+const ToolSchema_1 = require("./ToolSchema");
+const responseSchema = zod_1.z.object({
+    result: zod_1.z.string()
+});
+async function sendRequest(input) {
+    const { url, method, body, headers, waitForResponse } = input;
+    const options = {
+        method,
+        headers,
+        body: method === ToolSchema_1.AllowedOperations.POST ? JSON.stringify(body) : undefined
+    };
+    if (waitForResponse && method === ToolSchema_1.AllowedOperations.POST) {
+        setTimeout(() => fetch(url, options), 0);
+        return { result: "request sent" };
+    }
+    else {
+        const response = await fetch(url, options);
+        const data = await response.json();
+        return responseSchema.parse(data);
+    }
+}

package/dist/tools/type.d.ts

@@ -0,0 +1,8 @@
+import { z } from "zod";
+import { toolSchema } from "./ToolSchema";
+type Tool = z.infer<typeof toolSchema>;
+type OpenAITool = {
+    type: "function";
+    function: Omit<Tool, "operation">;
+};
+export { OpenAITool, Tool };

package/dist/tools/type.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types.d.ts

@@ -1,7 +1,15 @@
-import { VoiceResponse } from "@fonoster/voice";
-import { AssistantConfig } from "./assistants/types";
-type AutopilotConfig = {
-    voice: VoiceResponse;
-    assistantConfig: AssistantConfig;
+import { ConversationSettings } from "./assistants";
+import { LanguageModel } from "./models";
+import { Vad } from "./vad";
+import { Voice } from "./voice";
+declare enum LANGUAGE_MODEL_PROVIDER {
+    OPENAI = "openai",
+    GROQ = "groq"
+}
+type AutopilotParams = {
+    voice: Voice;
+    vad: Vad;
+    conversationSettings: ConversationSettings;
+    languageModel: LanguageModel;
 };
-export { AutopilotConfig };
+export { AutopilotParams, LANGUAGE_MODEL_PROVIDER };

package/dist/types.js

@@ -1,2 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.LANGUAGE_MODEL_PROVIDER = void 0;
+var LANGUAGE_MODEL_PROVIDER;
+(function (LANGUAGE_MODEL_PROVIDER) {
+    LANGUAGE_MODEL_PROVIDER["OPENAI"] = "openai";
+    LANGUAGE_MODEL_PROVIDER["GROQ"] = "groq";
+})(LANGUAGE_MODEL_PROVIDER || (exports.LANGUAGE_MODEL_PROVIDER = LANGUAGE_MODEL_PROVIDER = {}));

package/dist/vad/SileroVad.d.ts

@@ -0,0 +1,8 @@
+import { Vad } from "./types";
+declare class SileroVad implements Vad {
+    private vad;
+    constructor();
+    private init;
+    processChunk(data: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void): void;
+}
+export { SileroVad };

package/dist/vad/SileroVad.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SileroVad = void 0;
+/* eslint-disable no-loops/no-loops */
+/*
+ * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const makeVad_1 = require("./makeVad");
+class SileroVad {
+    constructor() {
+        this.init();
+    }
+    async init() {
+        // FIXME: It feels not to do this in the constructor
+        this.vad = await (0, makeVad_1.makeVad)();
+    }
+    processChunk(data, callback) {
+        if (!this.vad) {
+            throw new Error("VAD not initialized)");
+        }
+        this.vad(data, callback);
+    }
+}
+exports.SileroVad = SileroVad;

package/dist/vad/chunkToFloat32Array.d.ts

@@ -1,2 +1,2 @@
-declare function chunkToFloat32Array(chunk: Float32Array): Float32Array;
+declare function chunkToFloat32Array(chunk: Uint8Array): Float32Array;
 export { chunkToFloat32Array };

package/dist/vad/chunkToFloat32Array.js

@@ -1,6 +1,4 @@
 "use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.chunkToFloat32Array = chunkToFloat32Array;
 /*
  * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
  * http://github.com/fonoster/fonoster
@@ -19,7 +17,23 @@ exports.chunkToFloat32Array = chunkToFloat32Array;
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.chunkToFloat32Array = chunkToFloat32Array;
+// This version of the chunkToFloat32Array accounts for the case where
+// the byteOffset is misaligned.
+//
+// Q. Would it be the same if we just created a new Uint8Array from the chunk?
 function chunkToFloat32Array(chunk) {
-    const int16Array = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.byteLength / Int16Array.BYTES_PER_ELEMENT);
+    // Check if byteOffset is not aligned
+    const alignedByteOffset = chunk.byteOffset % Int16Array.BYTES_PER_ELEMENT === 0;
+    let int16Array;
+    if (alignedByteOffset) {
+        int16Array = new Int16Array(chunk.buffer, chunk.byteOffset, chunk.byteLength / Int16Array.BYTES_PER_ELEMENT);
+    }
+    else {
+        // Create a new aligned Uint8Array and then an Int16Array from it
+        const alignedChunk = new Uint8Array(chunk);
+        int16Array = new Int16Array(alignedChunk.buffer, alignedChunk.byteOffset, alignedChunk.byteLength / Int16Array.BYTES_PER_ELEMENT);
+    }
     return new Float32Array(Array.from(int16Array, (sample) => sample / 32768.0));
 }

package/dist/vad/index.d.ts

@@ -1,3 +1,2 @@
-export * from "./SileroVadModel";
+export * from "./SileroVad";
 export * from "./types";
-export * from "./makeVad";

package/dist/vad/index.js

@@ -33,6 +33,5 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-__exportStar(require("./SileroVadModel"), exports);
+__exportStar(require("./SileroVad"), exports);
 __exportStar(require("./types"), exports);
-__exportStar(require("./makeVad"), exports);

package/dist/vad/makeVad.d.ts

@@ -1,2 +1,2 @@
-declare function makeVad(pathToModel?: string): Promise<(chunk: Float32Array, callback: (event: "SPEECH_START" | "SPEECH_END", data: Record<string, unknown>) => void) => Promise<void>>;
+declare function makeVad(pathToModel?: string): Promise<(chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => Promise<void>>;
 export { makeVad };

package/dist/vad/makeVad.js

@@ -65,16 +65,14 @@ async function makeVad(pathToModel) {
             if (result.isSpeech > 0.5) {
                 if (!isSpeechActive) {
                     isSpeechActive = true;
-                    callback("SPEECH_START", {});
+                    callback("SPEECH_START");
                     return processBuffer(remainingBuffer);
                 }
             }
-            else {
-                if (isSpeechActive) {
-                    isSpeechActive = false;
-                    callback("SPEECH_END", {});
-                    return processBuffer(remainingBuffer);
-                }
+            else if (isSpeechActive) {
+                isSpeechActive = false;
+                callback("SPEECH_END");
+                return processBuffer(remainingBuffer);
             }
             return processBuffer(remainingBuffer);
         };

package/dist/vad/types.d.ts

@@ -1,4 +1,6 @@
-type Vad = (chunk: Float32Array, cb: (event: string) => void) => Promise<void>;
+type Vad = {
+    processChunk: (chunk: Uint8Array, callback: (event: "SPEECH_START" | "SPEECH_END") => void) => void;
+};
 type SpeechProbabilities = {
     notSpeech: number;
     isSpeech: number;
@@ -14,4 +16,4 @@ type ONNXRuntimeAPI = {
         new (type: "float32", data: Float32Array, dims: [1, number]): unknown;
     };
 };
-export { SpeechProbabilities, ONNXRuntimeAPI, Vad };
+export { ONNXRuntimeAPI, SpeechProbabilities, Vad };

package/dist/voice/Voice.d.ts

@@ -0,0 +1,25 @@
+import { VoiceResponse } from "@fonoster/voice";
+import { Voice } from "./types";
+declare class VoiceImpl implements Voice {
+    private voice;
+    private playbackRef;
+    sessionRef: string;
+    constructor(sessionRef: string, voice: VoiceResponse);
+    answer(): Promise<void>;
+    hangup(): Promise<void>;
+    say(text: string): Promise<void>;
+    sgather(): Promise<{
+        stop: () => Promise<void>;
+        onData: (cb: (speech: string) => void) => void;
+    }>;
+    stream(): Promise<{
+        stop: () => Promise<void>;
+        onData: (cb: (chunk: Uint8Array) => void) => void;
+    }>;
+    transfer(to: string, options: {
+        record: boolean;
+        timeout: number;
+    }): Promise<void>;
+    stopSpeech(): Promise<void>;
+}
+export { VoiceImpl };

package/dist/voice/Voice.js

@@ -0,0 +1,81 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VoiceImpl = void 0;
+/*
+ * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const common_1 = require("@fonoster/common");
+const uuid_1 = require("uuid");
+class VoiceImpl {
+    constructor(sessionRef, voice) {
+        this.voice = voice;
+        this.sessionRef = sessionRef;
+        this.playbackRef = (0, uuid_1.v4)();
+    }
+    async answer() {
+        await this.voice.answer();
+    }
+    async hangup() {
+        await this.voice.hangup();
+    }
+    async say(text) {
+        await this.voice.say(text, { playbackRef: this.playbackRef });
+    }
+    async sgather() {
+        const stream = await this.voice.sgather({
+            source: common_1.StreamGatherSource.SPEECH
+        });
+        return {
+            stop: async () => {
+                stream.close();
+                stream.cleanup(() => { });
+            },
+            onData: (cb) => {
+                stream.onPayload((payload) => {
+                    cb(payload.speech);
+                });
+            }
+        };
+    }
+    async stream() {
+        const stream = await this.voice.stream();
+        return {
+            stop: async () => {
+                stream.close();
+                stream.cleanup(() => { });
+            },
+            onData: (cb) => {
+                stream.onPayload((payload) => {
+                    cb(payload.data);
+                });
+            }
+        };
+    }
+    async transfer(to, options) {
+        const { record, timeout } = options;
+        const effectiveOptions = {
+            recordDirection: record ? common_1.DialRecordDirection.BOTH : undefined,
+            timeout
+        };
+        await this.voice.dial(to, effectiveOptions);
+    }
+    async stopSpeech() {
+        await this.voice.playbackControl(this.playbackRef, common_1.PlaybackControlAction.STOP);
+    }
+}
+exports.VoiceImpl = VoiceImpl;

package/dist/voice/index.d.ts

@@ -0,0 +1,2 @@
+export * from "./Voice";
+export * from "./types";

package/dist/voice/index.js

@@ -0,0 +1,36 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./Voice"), exports);
+/*
+ * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
+ * http://github.com/fonoster/fonoster
+ *
+ * This file is part of Fonoster
+ *
+ * Licensed under the MIT License (the "License");
+ * you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    https://opensource.org/licenses/MIT
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+__exportStar(require("./types"), exports);

package/dist/voice/types.d.ts

@@ -0,0 +1,23 @@
+type GatherStream = {
+    stop: () => Promise<void>;
+    onData: (cb: (speech: string) => void) => void;
+};
+type Stream = {
+    stop: () => Promise<void>;
+    onData: (cb: (chunk: Uint8Array) => void) => void;
+};
+type TransferOptions = {
+    timeout?: number;
+    record?: boolean;
+};
+type Voice = {
+    sessionRef: string;
+    answer: () => Promise<void>;
+    hangup: () => Promise<void>;
+    say: (text: string) => Promise<void>;
+    sgather: () => Promise<GatherStream>;
+    transfer: (destination: string, options?: TransferOptions) => Promise<void>;
+    stream: () => Promise<Stream>;
+    stopSpeech: () => Promise<void>;
+};
+export { TransferOptions, Voice };

package/dist/voice/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/voiceServerSetup.d.ts

@@ -0,0 +1,2 @@
+declare function startVoiceServer(skipIdentity: boolean): void;
+export { startVoiceServer };

package/dist/{runner.js → voiceServerSetup.js}

@@ -1,9 +1,9 @@
-#!/usr/bin/env node
 "use strict";
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.startVoiceServer = startVoiceServer;
 /*
  * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
  * http://github.com/fonoster/fonoster
@@ -22,25 +22,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const logger_1 = require("@fonoster/logger");
 const voice_1 = __importDefault(require("@fonoster/voice"));
-const assistants_1 = require("./assistants");
-const Autopilot_1 = require("./Autopilot");
-const envs_1 = require("./envs");
-const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
-const skipIdentity = process.env.NODE_ENV === "dev";
-new voice_1.default({ skipIdentity }).listen(async (req, voice) => {
-    const { ingressNumber, sessionRef, appRef } = req;
-    logger.verbose("voice request", { ingressNumber, sessionRef, appRef });
-    const assistantPath = process.argv[2];
-    const assistant = (0, assistants_1.loadAndValidateAssistant)(assistantPath);
-    logger.verbose("interacting with assistant", { name: assistant.name });
-    const autopilot = new Autopilot_1.Autopilot({
-        voice,
-        assistantConfig: {
-            ...assistant,
-            apiKey: envs_1.OPENAI_API_KEY
-        }
-    });
-    autopilot.start();
-});
+const handleVoiceRequest_1 = require("./handleVoiceRequest");
+function startVoiceServer(skipIdentity) {
+    new voice_1.default({ skipIdentity }).listen(handleVoiceRequest_1.handleVoiceRequest);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fonoster/autopilot",
-  "version": "0.7.3",
+  "version": "0.7.5",
   "description": "Voice AI for the Fonoster platform",
   "author": "Pedro Sanders <psanders@fonoster.com>",
   "homepage": "https://github.com/fonoster/fonoster#readme",
@@ -14,7 +14,8 @@
   "scripts": {
     "prebuild": "rimraf ./dist tsconfig.tsbuildinfo",
     "build": "tsc -b tsconfig.json",
-    "clean": "rimraf ./dist node_modules tsconfig.tsbuildinfo"
+    "clean": "rimraf ./dist node_modules tsconfig.tsbuildinfo",
+    "generate:readme": "node ../../.scripts/gen-readme.js"
   },
   "bin": {
     "autopilot": "./dist/runner.js"
@@ -34,17 +35,24 @@
     "url": "https://github.com/fonoster/fonoster/issues"
   },
   "dependencies": {
-    "@fonoster/logger": "^0.7.2",
-    "@fonoster/voice": "^0.7.3",
-    "@langchain/openai": "^0.2.7",
+    "@fonoster/common": "^0.7.5",
+    "@fonoster/logger": "^0.7.5",
+    "@fonoster/voice": "^0.7.5",
+    "@langchain/community": "^0.2.31",
+    "@langchain/core": "^0.2.31",
+    "@langchain/groq": "^0.0.17",
+    "@langchain/openai": "^0.2.8",
+    "cheerio": "^1.0.0",
     "dotenv": "^16.4.5",
+    "langchain": "^0.2.17",
     "onnxruntime-node": "^1.19.0",
+    "pdf-parse": "^1.1.1",
+    "uuid": "^10.0.0",
     "xstate": "^5.17.3",
     "zod": "^3.23.8"
   },
   "devDependencies": {
-    "node-record-lpcm16": "^1.0.1",
     "typescript": "^5.5.4"
   },
-  "gitHead": "97aa6649691819fe8a704b96bc62b1b142162393"
+  "gitHead": "cf28444503e4e698ccc001e061c7c918a4204757"
 }

package/dist/assistants/assistants.d.ts

@@ -1,4 +0,0 @@
-import { AssistantConfig } from "./types";
-type Assistant = ReturnType<typeof makeAssistant>;
-declare function makeAssistant(config: AssistantConfig): import("@langchain/core/runnables").Runnable<any, string, import("@langchain/core/runnables").RunnableConfig>;
-export { makeAssistant, Assistant };

package/dist/machine/types.d.ts

@@ -1,29 +0,0 @@
-import { VoiceResponse } from "@fonoster/voice";
-import { Assistant } from "../assistants/assistants";
-declare const types: {
-    context: {
-        firstMessage: string;
-        voice: VoiceResponse;
-        assistant: Assistant;
-        playbackRef: string;
-        speechBuffer: string;
-        speechResponseStartTime: number;
-        speechResponseTime: number;
-    };
-    input: {
-        firstMessage: string;
-        voice: VoiceResponse;
-        assistant: Assistant;
-    };
-    events: {
-        type: "SPEECH_START";
-    } | {
-        type: "SPEECH_END";
-    } | {
-        type: "SESSION_END";
-    } | {
-        type: "HUMAN_PROMPT";
-        speech: string;
-    };
-};
-export { types };

package/dist/machine/types.js

@@ -1,9 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.types = void 0;
-const types = {
-    context: {},
-    input: {},
-    events: {}
-};
-exports.types = types;

package/dist/runner.d.ts

@@ -1,2 +0,0 @@
-#!/usr/bin/env node
-export {};

package/dist/vad/micVadTest.js

@@ -1,48 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-/*
- * Copyright (C) 2024 by Fonoster Inc (https://fonoster.com)
- * http://github.com/fonoster/fonoster
- *
- * This file is part of Fonoster
- *
- * Licensed under the MIT License (the "License");
- * you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- *    https://opensource.org/licenses/MIT
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-const logger_1 = require("@fonoster/logger");
-const node_record_lpcm16_1 = __importDefault(require("node-record-lpcm16"));
-const makeVad_1 = require("./makeVad");
-const logger = (0, logger_1.getLogger)({ service: "autopilot", filePath: __filename });
-async function main() {
-    const vad = await (0, makeVad_1.makeVad)();
-    // Start recording from the default microphone
-    const mic = node_record_lpcm16_1.default
-        .record({
-        sampleRate: 16000, // 16 kHz sample rate
-        channels: 1,
-        threshold: 0.5
-    })
-        .stream();
-    mic.on("data", async (data) => {
-        const chunk = new Float32Array(data.buffer);
-        await vad(chunk, (event, _data) => {
-            logger.info("vad event:", { event, data: _data });
-        });
-    });
-    mic.on("error", (err) => {
-        logger.error("an error occurred:", { err });
-    });
-}
-main().catch(logger.error);