@pyai/sdk 0.7.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -7
- package/dist/cli-omni-template.d.ts +2 -2
- package/dist/cli-omni-template.js +2 -2
- package/dist/index.d.ts +68 -10
- package/dist/index.js +321 -43
- package/dist/speech-metadata.d.ts +5 -0
- package/dist/speech-metadata.js +37 -0
- package/dist/speech-text.d.ts +18 -0
- package/dist/speech-text.js +75 -0
- package/package.json +1 -1
- package/src/cli-omni-template.ts +2 -2
- package/src/index.ts +335 -41
- package/src/speech-metadata.ts +33 -0
- package/src/speech-text.ts +71 -0
package/README.md
CHANGED
|
@@ -54,6 +54,31 @@ const job = await pyai.transcriptionJobs.create(
|
|
|
54
54
|
const done = await pyai.transcriptionJobs.get(job.job_id);
|
|
55
55
|
```
|
|
56
56
|
|
|
57
|
+
## Explicit pronunciation of English fields
|
|
58
|
+
|
|
59
|
+
`formatSpeechText` prepares a plain string for Speak `input` or a Cast line's
|
|
60
|
+
`text`. Mark each field explicitly; ordinary text stays unchanged.
|
|
61
|
+
|
|
62
|
+
```ts
|
|
63
|
+
import { formatSpeechText } from "@pyai/sdk";
|
|
64
|
+
|
|
65
|
+
const input = formatSpeechText([
|
|
66
|
+
"Your total is ", { type: "currency", value: "12.50", currency: "USD" },
|
|
67
|
+
". Your appointment is ", { type: "date", value: "2026-10-12", locale: "en-US" },
|
|
68
|
+
". Your reference is ", { type: "digits", value: "0042" }, ".",
|
|
69
|
+
]);
|
|
70
|
+
const audio = await pyai.audio.speech({ input, voice: "stock_amelia_en_gb" });
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
Supported fields are USD/GBP/EUR amounts (decimal strings, up to nine whole
|
|
74
|
+
digits and two fractional digits), ISO calendar dates with `en-US` or `en-GB`
|
|
75
|
+
ordering, phone numbers spoken digit by digit (3–15 digits, optional leading
|
|
76
|
+
`+`, spaces, parentheses, dots or hyphens), and digit strings (1–64 digits).
|
|
77
|
+
Phone formatting preserves leading zeros; extensions must be separate literal
|
|
78
|
+
text and a `digits` field. Invalid or ambiguous fields throw before a request.
|
|
79
|
+
Include spaces and punctuation in the literal parts. This helper supports English
|
|
80
|
+
only; listen to the selected voice before using the output in your application.
|
|
81
|
+
|
|
57
82
|
## Use with MCP (AI coding agents)
|
|
58
83
|
|
|
59
84
|
Building this SDK *with* an AI coding agent (Cursor, Claude Code, Codex)? Add the
|
|
@@ -81,12 +106,12 @@ the instant the socket opens and routes server frames to typed callbacks, so you
|
|
|
81
106
|
is acked but silently dropped, giving you a connected session with zero turns):
|
|
82
107
|
|
|
83
108
|
<!-- omni-starter:start -->
|
|
84
|
-
Requires SDK 0.
|
|
109
|
+
Requires SDK 0.9.1 for `--template omni`. The runnable starter uses ESM,
|
|
85
110
|
Node 20.19+ with an explicit `ws` transport, or Node 22+. MCP 0.5.0
|
|
86
111
|
requires Node 22+. The SDK's REST client continues to support Node 18+.
|
|
87
112
|
|
|
88
113
|
```sh
|
|
89
|
-
npm install @pyai/sdk@0.
|
|
114
|
+
npm install @pyai/sdk@0.9.1
|
|
90
115
|
npx pyai init voice-demo --template omni
|
|
91
116
|
cd voice-demo
|
|
92
117
|
npm install
|
|
@@ -114,8 +139,8 @@ Running it consumes Omni and Hear usage under the injected key. It saves a
|
|
|
114
139
|
private report and WAV files. The report separates received audio from an
|
|
115
140
|
answer recovered by Hear from captured bytes; synthesis text alone cannot pass.
|
|
116
141
|
The playback sink is simulated. Interruption clears that queue; physical speaker
|
|
117
|
-
playback remains a separate test.
|
|
118
|
-
|
|
142
|
+
playback remains a separate test. This starter does not negotiate playback
|
|
143
|
+
receipts, so capture uses two seconds of quiet after the queue drains.
|
|
119
144
|
|
|
120
145
|
This section and the CLI project are generated from the same tested example.
|
|
121
146
|
Release readiness: https://pyai.com/agents/bot-release.json
|
|
@@ -124,8 +149,12 @@ Release readiness: https://pyai.com/agents/bot-release.json
|
|
|
124
149
|
Live `0x02` transcript bodies are plain UTF-8 caller-text deltas, not JSON.
|
|
125
150
|
`onTranscript` receives the normalized
|
|
126
151
|
`{ event:"transcript", role:"user", text, final:false, mode:"delta" }` shape.
|
|
127
|
-
Coalesce successive deltas for the current caller turn.
|
|
128
|
-
|
|
152
|
+
Coalesce successive deltas for the current caller turn. Braces and JSON-shaped
|
|
153
|
+
speech remain literal caller text; they cannot change the speaker or completion
|
|
154
|
+
state. Only for a known older JSON transport, pass
|
|
155
|
+
`transcriptFormat: "legacy-json"` to `omni.connect()`, or `"legacy-json"` as the
|
|
156
|
+
second argument to `normalizeOmniTranscriptBody()`. This explicit mode accepts
|
|
157
|
+
bounded JSON objects and rejects plain-text bodies.
|
|
129
158
|
|
|
130
159
|
Since version 0.5.1, the engine's four-field assistant synthesis advisory on
|
|
131
160
|
`0x03` also reaches `onTranscript` with `role: "assistant"`, `final: true`,
|
|
@@ -135,6 +164,12 @@ Caller transcripts continue to require the `0x02` carrier.
|
|
|
135
164
|
|
|
136
165
|
`rate` configures caller input. A `rate: 16000` session still receives 24 kHz
|
|
137
166
|
agent audio; `rate: 8000` receives 8 kHz. Read `hello.audio_out` before playback.
|
|
167
|
+
The connection requires the native `hello` and validates PCM16 mono declarations
|
|
168
|
+
before delivering audio. A conflicting input/output rate, format, channel count,
|
|
169
|
+
segment rate, or odd PCM byte count clears pending receipts, invokes `onBargeIn`
|
|
170
|
+
with a `flush` event, closes with code `1002`, and reports `onError`. Clear your
|
|
171
|
+
player queue in `onBargeIn`. Older servers may omit playback acknowledgements;
|
|
172
|
+
their native audio declaration is still required.
|
|
138
173
|
Omni has no commit frame—keep streaming silence during caller pauses.
|
|
139
174
|
Use one paced input stream: send caller PCM when available, otherwise silence.
|
|
140
175
|
Pause any separate silence timer while microphone or fixture frames are being
|
|
@@ -412,7 +447,7 @@ Common codes: `unauthorized`, `forbidden`, `credit_exhausted`,
|
|
|
412
447
|
## CLI (`pyai`)
|
|
413
448
|
|
|
414
449
|
The package provides a `pyai` executable for engineers, CI, and coding agents.
|
|
415
|
-
Install version 0.
|
|
450
|
+
Install version 0.9.1 with `npm install -g @pyai/sdk@0.9.1`, then use
|
|
416
451
|
`pyai login` for browser sign-in. Environment API keys work for unattended
|
|
417
452
|
automation.
|
|
418
453
|
|
|
@@ -443,3 +478,65 @@ npm install
|
|
|
443
478
|
npm test # node --test, fetch injected (no network)
|
|
444
479
|
npm run build # emits dist/ (incl. the pyai CLI bin)
|
|
445
480
|
```
|
|
481
|
+
|
|
482
|
+
## Network recovery
|
|
483
|
+
|
|
484
|
+
REST automatically retries GET/HEAD only: transient connection failures with a
|
|
485
|
+
recognized cause, and HTTP 429/500/502/503/504. `maxRetries` defaults to 2;
|
|
486
|
+
`retryWindowMs` defaults to 10000. Backoff uses jitter and respects both numeric
|
|
487
|
+
and HTTP-date `Retry-After`. A delay outside the remaining window returns the
|
|
488
|
+
failure instead of retrying early. Original transport exceptions retain their
|
|
489
|
+
cause. An unclassified browser `TypeError` is surfaced without retry.
|
|
490
|
+
|
|
491
|
+
The window bounds admission of retries, **not** an in-flight request or response
|
|
492
|
+
body. Pass a caller-owned `signal` (for example `AbortSignal.timeout(10000)` where
|
|
493
|
+
supported) to bound the REST operation and cancel backoff. A signal belongs to
|
|
494
|
+
its deadline: create a new client/signal for a later independent operation.
|
|
495
|
+
Custom fetch implementations must honor the signal. Response-body failures after
|
|
496
|
+
successful headers are surfaced; the SDK does not replay a partially read body.
|
|
497
|
+
|
|
498
|
+
Writes, including speech and job creation, are no longer automatically retried
|
|
499
|
+
on server errors. The server may already have acted even when no response audio
|
|
500
|
+
arrived. Where the endpoint documents idempotency, applications can explicitly
|
|
501
|
+
retry the identical operation with the same supported idempotency key. An
|
|
502
|
+
arbitrary header does not make every endpoint safe to replay.
|
|
503
|
+
|
|
504
|
+
For an explicit Omni restoration attempt, use `omni.connect({ resumeToken,
|
|
505
|
+
resumeTimeoutMs: 5000, ... })` with the single-use token from the previous
|
|
506
|
+
`session_started`. The SDK waits for `session_started.resumed === true` before
|
|
507
|
+
sending automatic configuration or delivering media/tools. Sending audio or
|
|
508
|
+
controls earlier throws. Rejection or a missing acknowledgement closes with
|
|
509
|
+
`resume_failed`; it cannot silently become a new conversation. `onOpen` only
|
|
510
|
+
means the socket opened. After successful restoration, follow the normal
|
|
511
|
+
`configured` readiness rules before sending microphone audio. Fresh connections
|
|
512
|
+
are unchanged. The same guard applies to `query.resume_token`.
|
|
513
|
+
|
|
514
|
+
This is an explicit, bounded restoration attempt, not automatic reconnection.
|
|
515
|
+
Do not loop a consumed token, replay unacknowledged audio, or replay tool actions.
|
|
516
|
+
Store rotated tokens privately; do not log connection URLs containing them.
|
|
517
|
+
Clear queued playback after a disconnect and tell the caller when speech was
|
|
518
|
+
lost. Conversation-state restoration does not guarantee recovery of audio sent
|
|
519
|
+
immediately before a disconnect. Python currently provides URL helpers rather
|
|
520
|
+
than this WebSocket lifecycle guard.
|
|
521
|
+
|
|
522
|
+
## Playback-aware Omni history
|
|
523
|
+
|
|
524
|
+
Set `configure.playback_ack: true` on the initial connection and check that
|
|
525
|
+
`onConfigured` receives `playback_ack: true`. Older serving versions can omit
|
|
526
|
+
this acknowledgment; in that case the client continues without player receipts.
|
|
527
|
+
|
|
528
|
+
`onAudio(chunk, playback)` supplies optional reply and segment identifiers,
|
|
529
|
+
output sample rate, and a `playback.played()` callback. Call it after that chunk
|
|
530
|
+
finishes playing naturally. The SDK advances receipts only through contiguous
|
|
531
|
+
completed chunks. Receipt or scheduling time alone does not count as playback.
|
|
532
|
+
|
|
533
|
+
Use `onBargeIn` to stop the current audio and clear the queued audio. The callback
|
|
534
|
+
now also receives `reply_interrupted` and `reply_failed`. Pending playback
|
|
535
|
+
callbacks are invalidated before this handler runs, so forced player stops do
|
|
536
|
+
not acknowledge discarded audio.
|
|
537
|
+
|
|
538
|
+
`onReplyState` reports `reply_generation_done`, `reply_playback_done`,
|
|
539
|
+
`reply_interrupted`, and `reply_failed`. Generation can finish while audio is
|
|
540
|
+
still queued. History retains fully confirmed segments; an interrupted partial
|
|
541
|
+
segment is omitted. Successful tool actions remain completed even if their
|
|
542
|
+
spoken confirmation was interrupted. See the [protocol contract](https://api.pyai.com/omni-frames.json).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare const omniTemplate: Readonly<{
|
|
2
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
3
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
2
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.9.1\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n";
|
|
3
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.9.1\",\n \"ws\": \"^8.18.3\"\n }\n}\n";
|
|
4
4
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n";
|
|
5
5
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n";
|
|
6
6
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.9.1\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.9.1\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/dist/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
export { formatSpeechText, type SpeechTextPart } from "./speech-text.ts";
|
|
1
2
|
/**
|
|
2
3
|
* @pyai/sdk, official TypeScript/JavaScript client for the PyAI API.
|
|
3
4
|
*
|
|
@@ -13,8 +14,12 @@ export interface PyAIOptions {
|
|
|
13
14
|
baseURL?: string;
|
|
14
15
|
/** Injectable for tests / custom transports. Defaults to global fetch. */
|
|
15
16
|
fetch?: typeof fetch;
|
|
16
|
-
/** Retries on
|
|
17
|
+
/** Retries safe reads on transient failures. Writes are never replayed. Default 2. */
|
|
17
18
|
maxRetries?: number;
|
|
19
|
+
/** Retry admission window, in ms. Does not time out an in-flight request/body. Default 10000. */
|
|
20
|
+
retryWindowMs?: number;
|
|
21
|
+
/** Optional caller-owned cancellation/deadline signal for REST requests and backoff. */
|
|
22
|
+
signal?: AbortSignal;
|
|
18
23
|
}
|
|
19
24
|
/** Stable, machine-readable error. Branch on `code`, not `message`. */
|
|
20
25
|
export declare class PyAIError extends Error {
|
|
@@ -41,6 +46,8 @@ export interface Voice {
|
|
|
41
46
|
available_on?: Array<"speak" | "omni">;
|
|
42
47
|
/** Accepted Speak delivery modes. Empty means the voice has no Speak surface. */
|
|
43
48
|
synthesis_modes?: Array<"streaming" | "async">;
|
|
49
|
+
/** Qualified Speak directions. Empty or absent means no directed emotion. */
|
|
50
|
+
speak_emotions?: Array<"neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised">;
|
|
44
51
|
/** Voice-specific amount on top of the selected product's base rate. */
|
|
45
52
|
pricing?: {
|
|
46
53
|
included_in_base_price: boolean;
|
|
@@ -126,6 +133,8 @@ export interface SpeechParams {
|
|
|
126
133
|
sample_rate?: SpeechSampleRate;
|
|
127
134
|
/** Reserved; currently returns `400 unsupported_parameter` when provided. */
|
|
128
135
|
speed?: number;
|
|
136
|
+
/** Named delivery; inspect speak_emotions on the selected catalog voice. */
|
|
137
|
+
emotion?: "neutral" | "happy" | "sad" | "angry" | "fearful" | "surprised";
|
|
129
138
|
/**
|
|
130
139
|
* Reserved; currently returns `400 unsupported_parameter` when provided.
|
|
131
140
|
*/
|
|
@@ -184,12 +193,16 @@ export interface HearVocabulary {
|
|
|
184
193
|
/** Unix milliseconds, or null before the first save. */
|
|
185
194
|
updated_at: number | null;
|
|
186
195
|
}
|
|
196
|
+
/** Canonical Omni session language identifiers accepted by the public API. */
|
|
197
|
+
export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh";
|
|
198
|
+
/** Internal candidate identifiers; the public control plane rejects these until qualified. */
|
|
199
|
+
export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
|
|
187
200
|
export interface AgentConfig {
|
|
188
201
|
name?: string;
|
|
189
202
|
persona_system_prompt?: string | null;
|
|
190
203
|
greeting?: string | null;
|
|
191
204
|
voice_id?: string | null;
|
|
192
|
-
language?:
|
|
205
|
+
language?: OmniLanguage | null;
|
|
193
206
|
/**
|
|
194
207
|
* Opt-in speech-recognition vocabulary. PyAI sanitizes and keeps at most
|
|
195
208
|
* five terms. An empty list or null turns it off.
|
|
@@ -411,7 +424,7 @@ export interface HearStreamOptions {
|
|
|
411
424
|
/** Streaming STT model. Server default `pyai-hear`. */
|
|
412
425
|
model?: string;
|
|
413
426
|
/** Omit or use auto for automatic detection; an explicit code pins recognition. */
|
|
414
|
-
language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
|
|
427
|
+
language?: "auto" | "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
|
|
415
428
|
/** Input PCM sample rate in Hz. Default 16000 server-side. */
|
|
416
429
|
sampleRate?: number;
|
|
417
430
|
/** Audio frame encoding. Default "pcm16". */
|
|
@@ -587,8 +600,10 @@ export interface OmniTranscriptFrame extends OmniServerFrame {
|
|
|
587
600
|
/** Optional ordering hint on legacy JSON frames. */
|
|
588
601
|
sequence?: number;
|
|
589
602
|
}
|
|
590
|
-
/**
|
|
591
|
-
export
|
|
603
|
+
/** Transcript encoding is selected by the transport, never inferred from text. */
|
|
604
|
+
export type OmniTranscriptFormat = "text" | "legacy-json";
|
|
605
|
+
/** Normalize native caller text, or explicitly selected bounded legacy JSON. */
|
|
606
|
+
export declare function normalizeOmniTranscriptBody(bytes: Uint8Array, format?: OmniTranscriptFormat): OmniTranscriptFrame | null;
|
|
592
607
|
/** A binary agent-audio chunk delivered to {@link OmniConnectOptions.onAudio}. */
|
|
593
608
|
export type OmniAudioChunk = ArrayBuffer | ArrayBufferView | Blob;
|
|
594
609
|
export interface OmniToolDef {
|
|
@@ -613,7 +628,19 @@ export interface OmniToolCallFrame {
|
|
|
613
628
|
* you, which is the whole point: a hand-rolled `{"event":"configure"}` is
|
|
614
629
|
* acked but silently dropped, leaving the agent with no brain and zero turns.
|
|
615
630
|
*/
|
|
631
|
+
/** Audio identity and progress in output-rate PCM samples, never wall time. */
|
|
632
|
+
export interface OmniPlaybackChunk {
|
|
633
|
+
replyId: string;
|
|
634
|
+
segmentId: string;
|
|
635
|
+
sampleRate: number;
|
|
636
|
+
endSample: number;
|
|
637
|
+
/** Call only after this chunk actually finishes playing, never on receipt. */
|
|
638
|
+
played: () => void;
|
|
639
|
+
}
|
|
616
640
|
export interface OmniConfigure {
|
|
641
|
+
/** Require player receipts before assistant speech enters conversation memory.
|
|
642
|
+
* Confirm support through configured.playback_ack before relying on it. */
|
|
643
|
+
playback_ack?: boolean;
|
|
617
644
|
/** Voice to speak with (stock / clone / designed id). */
|
|
618
645
|
voice_id?: string;
|
|
619
646
|
/** System prompt / role + instructions for the agent. */
|
|
@@ -632,7 +659,7 @@ export interface OmniConfigure {
|
|
|
632
659
|
* proceeds and bills as what was served. Per-language availability is
|
|
633
660
|
* staged, see the Language support reference.
|
|
634
661
|
*/
|
|
635
|
-
language?:
|
|
662
|
+
language?: OmniLanguage;
|
|
636
663
|
/** Function calling: hosted catalog names, client-loop schemas, or names of
|
|
637
664
|
* server tools already registered with POST /v1/tools. An inline `endpoint`
|
|
638
665
|
* is rejected with `unsupported_tool_transport`. */
|
|
@@ -641,8 +668,14 @@ export interface OmniConfigure {
|
|
|
641
668
|
[k: string]: unknown;
|
|
642
669
|
}
|
|
643
670
|
export interface OmniConnectOptions {
|
|
671
|
+
/** Native `0x02` caller text by default. Select legacy-json only for a known older JSON transport. */
|
|
672
|
+
transcriptFormat?: OmniTranscriptFormat;
|
|
644
673
|
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
645
674
|
sessionLabel?: string;
|
|
675
|
+
/** Single-use token from session_started. Failed restoration closes instead of starting fresh. */
|
|
676
|
+
resumeToken?: string;
|
|
677
|
+
/** Maximum wait for resume acknowledgement, including socket opening. Default 5000 ms. */
|
|
678
|
+
resumeTimeoutMs?: number;
|
|
646
679
|
/**
|
|
647
680
|
* Ephemeral browser token from `omni.createSession`. When set it is used as
|
|
648
681
|
* the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
|
|
@@ -667,7 +700,7 @@ export interface OmniConnectOptions {
|
|
|
667
700
|
/** Fired once the socket opens (after the optional auto-configure). */
|
|
668
701
|
onOpen?: () => void;
|
|
669
702
|
/** Fired for each binary agent-audio chunk, play it out as it arrives. */
|
|
670
|
-
onAudio?: (chunk: OmniAudioChunk) => void;
|
|
703
|
+
onAudio?: (chunk: OmniAudioChunk, playback?: OmniPlaybackChunk) => void;
|
|
671
704
|
/** Fired on the `hello` handshake frame. */
|
|
672
705
|
onHello?: (frame: OmniServerFrame) => void;
|
|
673
706
|
/** Fired on the `configured` ack. */
|
|
@@ -678,8 +711,10 @@ export interface OmniConnectOptions {
|
|
|
678
711
|
onTurn?: (frame: OmniServerFrame) => void;
|
|
679
712
|
/** Fired for each normalized caller-transcript delta (`0x02` plain UTF-8 live). */
|
|
680
713
|
onTranscript?: (frame: OmniServerFrame) => void;
|
|
681
|
-
/**
|
|
714
|
+
/** Clear queued audio on `barge_in`, `flush`, `reply_interrupted` or `reply_failed`. */
|
|
682
715
|
onBargeIn?: (frame: OmniServerFrame) => void;
|
|
716
|
+
/** Generation completion is distinct from acknowledged playback completion. */
|
|
717
|
+
onReplyState?: (frame: OmniServerFrame) => void;
|
|
683
718
|
/** Fired when the engine requests a client-loop tool invocation. */
|
|
684
719
|
onToolCall?: (frame: OmniToolCallFrame) => void;
|
|
685
720
|
/** Fired on `session_end`. */
|
|
@@ -713,11 +748,25 @@ export declare class OmniConnection {
|
|
|
713
748
|
private readonly ws;
|
|
714
749
|
private readonly opts;
|
|
715
750
|
private closed;
|
|
751
|
+
private playbackEnabled;
|
|
752
|
+
private audioReady;
|
|
753
|
+
private readonly inputRate;
|
|
754
|
+
private readonly outputRate;
|
|
755
|
+
private playbackEpoch;
|
|
756
|
+
private playbackSegment?;
|
|
757
|
+
private awaitingResume;
|
|
758
|
+
private resumeTimer?;
|
|
716
759
|
/** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
|
|
717
760
|
private blobTail;
|
|
761
|
+
private outboundQueued;
|
|
718
762
|
/** Serializes inbound Blob decoding so browser frames stay ordered. */
|
|
719
763
|
private inboundTail;
|
|
764
|
+
private inboundQueued;
|
|
720
765
|
constructor(url: string, subprotocol: string, opts: OmniConnectOptions);
|
|
766
|
+
private rejectResume;
|
|
767
|
+
private assertSendable;
|
|
768
|
+
private failAudioContract;
|
|
769
|
+
private validateAudioControl;
|
|
721
770
|
private handleMessage;
|
|
722
771
|
private dispatchFrame;
|
|
723
772
|
/**
|
|
@@ -732,7 +781,14 @@ export declare class OmniConnection {
|
|
|
732
781
|
* default branch, so an untagged chunk is dropped with no error.
|
|
733
782
|
*/
|
|
734
783
|
sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void;
|
|
735
|
-
|
|
784
|
+
private queueOutbound;
|
|
785
|
+
private sendOrdered;
|
|
786
|
+
/** Report samples actually rendered by the player in one reply segment.
|
|
787
|
+
* Receiving or scheduling bytes is not playback. Stale receipts are ignored
|
|
788
|
+
* server-side; this never changes completed tool actions. */
|
|
789
|
+
playbackProgress(replyId: string, segmentId: string, playedSamples: number): void;
|
|
790
|
+
/** Send a nonempty DTMF sequence as ordered single-key control frames.
|
|
791
|
+
* Valid keys are 0–9, *, # and A–D; lowercase a–d are normalized. */
|
|
736
792
|
sendDtmf(digits: string): void;
|
|
737
793
|
/** Reply to a client-loop {@link OmniEvent.ToolCall}. */
|
|
738
794
|
toolResult(callId: string, payload: {
|
|
@@ -1146,6 +1202,8 @@ export declare class PyAI {
|
|
|
1146
1202
|
private readonly baseURL;
|
|
1147
1203
|
private readonly fetchImpl;
|
|
1148
1204
|
private readonly maxRetries;
|
|
1205
|
+
private readonly retryWindowMs;
|
|
1206
|
+
private readonly signal?;
|
|
1149
1207
|
constructor(opts: PyAIOptions);
|
|
1150
1208
|
private authHeaders;
|
|
1151
1209
|
private request;
|
|
@@ -1209,7 +1267,7 @@ export declare class PyAI {
|
|
|
1209
1267
|
filename?: string;
|
|
1210
1268
|
model?: string;
|
|
1211
1269
|
/** Optional language hint. Omission enables automatic detection. */
|
|
1212
|
-
language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl";
|
|
1270
|
+
language?: "en" | "es" | "fr" | "de" | "hi" | "it" | "pt" | "nl" | "zh" | "yue" | "ar" | "id" | "ko" | "th" | "vi" | "ja" | "tr" | "ms" | "sv" | "da" | "fi" | "pl" | "cs" | "fil" | "fa" | "el" | "hu" | "mk" | "ro";
|
|
1213
1271
|
numerals?: boolean;
|
|
1214
1272
|
smart_format?: boolean;
|
|
1215
1273
|
dictation?: boolean;
|