@cogitator-ai/voice 0.4.7 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -1
- package/dist/tools.d.ts +6 -0
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +32 -2
- package/dist/tools.js.map +1 -1
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -528,7 +528,19 @@ const agent = new Agent({
|
|
|
528
528
|
});
|
|
529
529
|
```
|
|
530
530
|
|
|
531
|
-
`VoiceTool` objects carry Zod parameter schemas, so they can be passed to `tool()` as-is. `transcribeTool(stt)` and `speakTool(tts)` create the tools one at a time. `transcribe_audio` takes `audioBase64` (+ optional `language`)
|
|
531
|
+
`VoiceTool` objects carry Zod parameter schemas, so they can be passed to `tool()` as-is. `transcribeTool(stt)` and `speakTool(tts)` create the tools one at a time. `transcribe_audio` takes `audioBase64` (+ optional `language`), `speak_text` takes `text`, `voice`, `format` and returns a `ToolContentResult` (see `toolContent` in `@cogitator-ai/core`): a line of text for the model and a `file` part with the base64 audio (`mediaType` such as `audio/mpeg`, `filename` such as `speech.mp3`). The model sees only the description, so the audio never fills its context, and your application reads it from the result:
|
|
532
|
+
|
|
533
|
+
```typescript
|
|
534
|
+
await cogitator.run(agent, {
|
|
535
|
+
input: 'Say good morning',
|
|
536
|
+
onToolResult: ({ result }) => {
|
|
537
|
+
const audio = isToolContentResult(result)
|
|
538
|
+
? result.content.find((part) => part.type === 'file')
|
|
539
|
+
: undefined;
|
|
540
|
+
if (audio) play(Buffer.from(audio.data, 'base64'), audio.mediaType);
|
|
541
|
+
},
|
|
542
|
+
});
|
|
543
|
+
```
|
|
532
544
|
|
|
533
545
|
---
|
|
534
546
|
|
package/dist/tools.d.ts
CHANGED
|
@@ -25,6 +25,12 @@ declare const SpeakParamsSchema: z.ZodObject<{
|
|
|
25
25
|
}, z.core.$strip>;
|
|
26
26
|
type SpeakParams = z.infer<typeof SpeakParamsSchema>;
|
|
27
27
|
export declare function transcribeTool(stt: STTProvider): VoiceTool<TranscribeParams>;
|
|
28
|
+
/**
|
|
29
|
+
* `speak_text`: synthesizes speech and returns it as a tool result with a `file` part (see
|
|
30
|
+
* `ToolContentResult`). The application gets the audio from the result, in `onToolResult` or
|
|
31
|
+
* `RunResult.toolCalls`, while the model sees only a one-line description instead of the base64
|
|
32
|
+
* audio, which would fill its context.
|
|
33
|
+
*/
|
|
28
34
|
export declare function speakTool(tts: TTSProvider): VoiceTool<SpeakParams>;
|
|
29
35
|
export declare function voiceTools(config: {
|
|
30
36
|
stt: STTProvider;
|
package/dist/tools.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE3D,MAAM,WAAW,SAAS,CAAC,OAAO,GAAG,OAAO;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC/B,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAChD;AAID,QAAA,MAAM,sBAAsB;;;iBAM1B,CAAC;AAEH,KAAK,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAE/D,QAAA,MAAM,iBAAiB;;;;;;;;;;;iBAOrB,CAAC;AAEH,KAAK,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAErD,wBAAgB,cAAc,CAAC,GAAG,EAAE,WAAW,GAAG,SAAS,CAAC,gBAAgB,CAAC,CAW5E;AAaD;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,CA0BlE;AAED,wBAAgB,UAAU,CAAC,MAAM,EAAE;IACjC,GAAG,EAAE,WAAW,CAAC;IACjB,GAAG,EAAE,WAAW,CAAC;CAClB,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC,CAExD"}
|
package/dist/tools.js
CHANGED
|
@@ -27,14 +27,44 @@ export function transcribeTool(stt) {
|
|
|
27
27
|
},
|
|
28
28
|
};
|
|
29
29
|
}
|
|
30
|
+
const SPEECH_MEDIA_TYPES = {
|
|
31
|
+
mp3: 'audio/mpeg',
|
|
32
|
+
opus: 'audio/ogg',
|
|
33
|
+
aac: 'audio/aac',
|
|
34
|
+
flac: 'audio/flac',
|
|
35
|
+
wav: 'audio/wav',
|
|
36
|
+
pcm16: 'audio/L16',
|
|
37
|
+
};
|
|
38
|
+
/**
|
|
39
|
+
* `speak_text`: synthesizes speech and returns it as a tool result with a `file` part (see
|
|
40
|
+
* `ToolContentResult`). The application gets the audio from the result, in `onToolResult` or
|
|
41
|
+
* `RunResult.toolCalls`, while the model sees only a one-line description instead of the base64
|
|
42
|
+
* audio, which would fill its context.
|
|
43
|
+
*/
|
|
30
44
|
export function speakTool(tts) {
|
|
31
45
|
return {
|
|
32
46
|
name: 'speak_text',
|
|
33
|
-
description: 'Convert text to speech audio',
|
|
47
|
+
description: 'Convert text to speech audio, which is delivered to the application',
|
|
34
48
|
parameters: SpeakParamsSchema,
|
|
35
49
|
execute: async ({ text, voice, format }) => {
|
|
36
50
|
const audio = await tts.synthesize(text, { voice, format });
|
|
37
|
-
|
|
51
|
+
const speechFormat = format ?? 'mp3';
|
|
52
|
+
const result = {
|
|
53
|
+
type: 'tool-content',
|
|
54
|
+
content: [
|
|
55
|
+
{
|
|
56
|
+
type: 'text',
|
|
57
|
+
text: `Synthesized speech: ${audio.length} bytes of ${speechFormat} audio, delivered to the application.`,
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
type: 'file',
|
|
61
|
+
data: audio.toString('base64'),
|
|
62
|
+
mediaType: SPEECH_MEDIA_TYPES[speechFormat],
|
|
63
|
+
filename: `speech.${speechFormat === 'pcm16' ? 'pcm' : speechFormat}`,
|
|
64
|
+
},
|
|
65
|
+
],
|
|
66
|
+
};
|
|
67
|
+
return result;
|
|
38
68
|
},
|
|
39
69
|
};
|
|
40
70
|
}
|
package/dist/tools.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;
|
|
1
|
+
{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAWxB,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAExD,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IACtC,WAAW,EAAE,CAAC;SACX,MAAM,EAAE;SACR,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;SACvC,QAAQ,CAAC,2BAA2B,CAAC;IACxC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,kCAAkC,CAAC;CAC7E,CAAC,CAAC;AAIH,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACtD,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;IACrD,MAAM,EAAE,CAAC;SACN,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;SACpD,QAAQ,EAAE;SACV,QAAQ,CAAC,qBAAqB,CAAC;CACnC,CAAC,CAAC;AAIH,MAAM,UAAU,cAAc,CAAC,GAAgB;IAC7C,OAAO;QACL,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE,yDAAyD;QACtE,UAAU,EAAE,sBAAsB;QAClC,OAAO,EAAE,KAAK,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;YACzD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrF,CAAC;KACF,CAAC;AACJ,CAAC;AAID,MAAM,kBAAkB,GAAiC;IACvD,GAAG,EAAE,YAAY;IACjB,IAAI,EAAE,WAAW;IACjB,GAAG,EAAE,WAAW;IAChB,IAAI,EAAE,YAAY;IAClB,GAAG,EAAE,WAAW;IAChB,KAAK,EAAE,WAAW;CACnB,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,GAAgB;IACxC,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,qEAAqE;QAClF,UAAU,EAAE,iBAAiB;QAC7B,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YACzC,MAAM,KAAK,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5D,MAAM,YAAY,GAAG,MAAM,IAAI,KAAK,CAAC;YACrC,MAAM,MAAM,GAAsB;gBAChC,IAAI,EAAE,cAAc;gBACpB,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,uBAAuB,KAAK,CAAC,MAAM,aAAa,YAAY,uCAAuC;qBAC1G;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC9B,SAAS,EAAE,kBAAkB,CAAC,YAAY,CAAC;wBAC3C,QAAQ,EAAE,UAAU,YAAY,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,EAAE;qBACtE;iBACF;aACF,CAAC;YACF,OAAO,MAAM,CAAC;QAChB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,MAG1B;IACC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7D,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cogitator-ai/voice",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Voice/Realtime agent capabilities for Cogitator",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cogitator",
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
"exports": {
|
|
19
19
|
".": {
|
|
20
20
|
"types": "./dist/index.d.ts",
|
|
21
|
-
"import": "./dist/index.js"
|
|
21
|
+
"import": "./dist/index.js",
|
|
22
|
+
"default": "./dist/index.js"
|
|
22
23
|
}
|
|
23
24
|
},
|
|
24
25
|
"files": [
|
|
@@ -28,13 +29,13 @@
|
|
|
28
29
|
"node": ">=22.12.0"
|
|
29
30
|
},
|
|
30
31
|
"dependencies": {
|
|
31
|
-
"@cogitator-ai/types": "0.
|
|
32
|
+
"@cogitator-ai/types": "0.36.0",
|
|
32
33
|
"nanoid": "^6.0.1",
|
|
33
34
|
"ws": "^8.22.0",
|
|
34
35
|
"zod": "^4.6.5"
|
|
35
36
|
},
|
|
36
37
|
"peerDependencies": {
|
|
37
|
-
"@cogitator-ai/core": ">=0.
|
|
38
|
+
"@cogitator-ai/core": ">=0.34.0",
|
|
38
39
|
"openai": "^6.0.0 || ^7.0.0"
|
|
39
40
|
},
|
|
40
41
|
"peerDependenciesMeta": {
|