@pyai/sdk 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_GUIDE.md +4 -4
- package/CLI.md +5 -5
- package/README.md +3 -3
- package/dist/cli-dx.js +3 -1
- package/dist/cli-init.js +4 -3
- package/dist/cli-omni-template.d.ts +2 -2
- package/dist/cli-omni-template.js +2 -2
- package/dist/index.d.ts +44 -7
- package/dist/index.js +12 -1
- package/package.json +1 -1
- package/src/cli-dx.ts +5 -1
- package/src/cli-init.ts +5 -3
- package/src/cli-omni-template.ts +2 -2
- package/src/index.ts +45 -7
package/AGENT_GUIDE.md
CHANGED
|
@@ -10,8 +10,8 @@ Markdown and does not require the marketing website renderer.
|
|
|
10
10
|
|
|
11
11
|
## Availability and sources of truth
|
|
12
12
|
|
|
13
|
-
The CLI is included in `@pyai/sdk` 0.
|
|
14
|
-
`npm install -g @pyai/sdk@0.
|
|
13
|
+
The CLI is included in `@pyai/sdk` 0.19.0. Install it with
|
|
14
|
+
`npm install -g @pyai/sdk@0.19.0`. Browser login supports existing PyAI accounts;
|
|
15
15
|
unattended agents should use an authorized environment key. Inspect the
|
|
16
16
|
installed command schema before acting.
|
|
17
17
|
|
|
@@ -35,7 +35,7 @@ do not invent an endpoint or present the feature as deployed.
|
|
|
35
35
|
Install the published CLI with Node.js 22 or newer recommended:
|
|
36
36
|
|
|
37
37
|
```bash
|
|
38
|
-
npm install -g @pyai/sdk@0.
|
|
38
|
+
npm install -g @pyai/sdk@0.19.0
|
|
39
39
|
pyai --version
|
|
40
40
|
pyai schema --json
|
|
41
41
|
```
|
|
@@ -51,7 +51,7 @@ npm install -g .
|
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
Alternatively install a supplied build with
|
|
54
|
-
`npm install -g /path/to/pyai-sdk-0.
|
|
54
|
+
`npm install -g /path/to/pyai-sdk-0.19.0.tgz`. The compiled executable supports
|
|
55
55
|
Node.js 18 or newer. Use `node dist/cli.js` explicitly when another package's
|
|
56
56
|
`pyai` shadows this executable.
|
|
57
57
|
|
package/CLI.md
CHANGED
|
@@ -6,8 +6,8 @@ scripts and in coding agents with shell access, including Cursor, Claude Code,
|
|
|
6
6
|
and Codex. `--json` provides structured results; `--help` and `schema` describe
|
|
7
7
|
the installed command surface.
|
|
8
8
|
|
|
9
|
-
**Version:** the expanded CLI is included in `@pyai/sdk` 0.
|
|
10
|
-
npm with `npm install -g @pyai/sdk@0.
|
|
9
|
+
**Version:** the expanded CLI is included in `@pyai/sdk` 0.19.0. Install it from
|
|
10
|
+
npm with `npm install -g @pyai/sdk@0.19.0`, then run `pyai login` for browser
|
|
11
11
|
sign-in or provide `PYAI_API_KEY` for unattended automation.
|
|
12
12
|
|
|
13
13
|
## Contents
|
|
@@ -31,7 +31,7 @@ sign-in or provide `PYAI_API_KEY` for unattended automation.
|
|
|
31
31
|
Install the published package (Node.js 22 or newer recommended):
|
|
32
32
|
|
|
33
33
|
```bash
|
|
34
|
-
npm install -g @pyai/sdk@0.
|
|
34
|
+
npm install -g @pyai/sdk@0.19.0
|
|
35
35
|
pyai --version
|
|
36
36
|
pyai login
|
|
37
37
|
pyai speak "Hello from PyAI." -o hello.wav
|
|
@@ -56,7 +56,7 @@ also be packaged and installed on another machine:
|
|
|
56
56
|
# In sdk/typescript after building
|
|
57
57
|
npm pack --ignore-scripts
|
|
58
58
|
# On the destination machine, using the actual tarball path
|
|
59
|
-
npm install -g /path/to/pyai-sdk-0.
|
|
59
|
+
npm install -g /path/to/pyai-sdk-0.19.0.tgz
|
|
60
60
|
pyai schema --json
|
|
61
61
|
```
|
|
62
62
|
|
|
@@ -641,7 +641,7 @@ artifacts. A completed call does not guarantee a recording or summary.
|
|
|
641
641
|
Open PyAI Phone in incoming-only mode:
|
|
642
642
|
|
|
643
643
|
```sh
|
|
644
|
-
npx @pyai/sdk@0.
|
|
644
|
+
npx @pyai/sdk@0.19.0 dialer --receive
|
|
645
645
|
```
|
|
646
646
|
|
|
647
647
|
Copy the one-use desktop enrollment JSON from your calling workspace into
|
package/README.md
CHANGED
|
@@ -106,12 +106,12 @@ the instant the socket opens and routes server frames to typed callbacks, so you
|
|
|
106
106
|
is acked but silently dropped, giving you a connected session with zero turns):
|
|
107
107
|
|
|
108
108
|
<!-- omni-starter:start -->
|
|
109
|
-
Requires SDK 0.
|
|
110
|
-
Node 20.19+ with an explicit `ws` transport, or Node 22+. MCP 0.5.
|
|
109
|
+
Requires SDK 0.19.0 for `--template omni`. The runnable starter uses ESM,
|
|
110
|
+
Node 20.19+ with an explicit `ws` transport, or Node 22+. MCP 0.5.22
|
|
111
111
|
requires Node 22+. The SDK's REST client continues to support Node 18+.
|
|
112
112
|
|
|
113
113
|
```sh
|
|
114
|
-
npm install @pyai/sdk@0.
|
|
114
|
+
npm install @pyai/sdk@0.19.0
|
|
115
115
|
npx pyai init voice-demo --template omni
|
|
116
116
|
cd voice-demo
|
|
117
117
|
npm install
|
package/dist/cli-dx.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
const sdkVersion = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
1
3
|
/** Offline discovery and copyable workflows; recipes never execute commands. */
|
|
2
4
|
export const aliases = Object.assign(Object.create(null), {
|
|
3
5
|
login: "auth login", logout: "auth logout", whoami: "auth status", use: "profiles use",
|
|
@@ -34,5 +36,5 @@ export const recipes = [
|
|
|
34
36
|
notes: ["Local schema and recipes work without credentials. Live OpenAPI needs a network connection but no API key.", "The CLI manages Agent profiles; use the SDK or WebSocket API for a live Omni audio session."] },
|
|
35
37
|
{ name: "ci", title: "Use PyAI in a build", description: "Check access and produce an audio artifact with machine-readable output.",
|
|
36
38
|
commands: ["pyai whoami --json", 'pyai speak "Your build is ready" -o build.wav --json', "pyai doctor --json"],
|
|
37
|
-
notes: ["Set PYAI_API_KEY through the CI secret store; do not put it in source or shell arguments.", "doctor synthesizes and transcribes a sample, which consumes usage. Use whoami for an authentication-only check.",
|
|
39
|
+
notes: ["Set PYAI_API_KEY through the CI secret store; do not put it in source or shell arguments.", "doctor synthesizes and transcribes a sample, which consumes usage. Use whoami for an authentication-only check.", `Install the CLI with npm install -g @pyai/sdk@${sdkVersion} and pin that version in repeatable builds.`] },
|
|
38
40
|
].map(recipe => ({ ...recipe, docs_url: "https://pyai.com/agents/speech-calling.md" }));
|
package/dist/cli-init.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { constants } from "node:fs";
|
|
1
|
+
import { constants, readFileSync } from "node:fs";
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import { basename, dirname, isAbsolute, join, parse, resolve } from "node:path";
|
|
4
4
|
import { omniTemplate } from "./cli-omni-template.js";
|
|
@@ -10,6 +10,7 @@ export class CliInitError extends Error {
|
|
|
10
10
|
this.code = code;
|
|
11
11
|
}
|
|
12
12
|
}
|
|
13
|
+
const sdkVersion = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
13
14
|
const context = `# Build with PyAI
|
|
14
15
|
|
|
15
16
|
This file is a project-local API integration guide for engineers and coding agents.
|
|
@@ -128,7 +129,7 @@ const install = `## Install the CLI
|
|
|
128
129
|
Install the published CLI with Node.js 22 or newer recommended:
|
|
129
130
|
|
|
130
131
|
\`\`\`bash
|
|
131
|
-
npm install -g @pyai/sdk
|
|
132
|
+
npm install -g @pyai/sdk@${sdkVersion}
|
|
132
133
|
pyai --version
|
|
133
134
|
\`\`\`
|
|
134
135
|
|
|
@@ -221,7 +222,7 @@ and product calls below are separate explicit steps. Read \`PYAI.md\` first.
|
|
|
221
222
|
Install the official SDK:
|
|
222
223
|
|
|
223
224
|
\`\`\`bash
|
|
224
|
-
npm install @pyai/sdk
|
|
225
|
+
npm install @pyai/sdk@${sdkVersion}
|
|
225
226
|
cp .env.example .env
|
|
226
227
|
\`\`\`
|
|
227
228
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare const omniTemplate: Readonly<{
|
|
2
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
3
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
2
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.19.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n";
|
|
3
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.19.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n";
|
|
4
4
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n";
|
|
5
5
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n";
|
|
6
6
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.19.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.19.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/dist/index.d.ts
CHANGED
|
@@ -102,6 +102,8 @@ export type SpeechSampleRate = (typeof SPEECH_SAMPLE_RATES)[number];
|
|
|
102
102
|
export type SpeakModel = "pyai-speak" | "tts-1" | "tts-1-hd";
|
|
103
103
|
export interface SpeechParams {
|
|
104
104
|
input: string;
|
|
105
|
+
/** Reuse one unique call ID across utterances for enabled call-stable voice recovery. */
|
|
106
|
+
speech_session_id?: string;
|
|
105
107
|
voice?: string;
|
|
106
108
|
model?: SpeakModel;
|
|
107
109
|
/**
|
|
@@ -284,6 +286,8 @@ export declare const HearFrameType: {
|
|
|
284
286
|
readonly SpeechFinal: "speech_final";
|
|
285
287
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
286
288
|
readonly Final: "final";
|
|
289
|
+
/** Protected speech was withheld for this utterance. */
|
|
290
|
+
readonly TranscriptGap: "transcript_gap";
|
|
287
291
|
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
288
292
|
readonly Usage: "usage";
|
|
289
293
|
/** Server-side fault frame. */
|
|
@@ -304,6 +308,8 @@ export declare const WSCloseCode: {
|
|
|
304
308
|
readonly PolicyViolation: 1008;
|
|
305
309
|
/** Engine/internal error. */
|
|
306
310
|
readonly InternalError: 1011;
|
|
311
|
+
/** Hear admission is busy after the socket upgrade. Retry with backoff. */
|
|
312
|
+
readonly TryAgainLater: 1013;
|
|
307
313
|
/** Over the concurrency cap (PyAI-specific; mirrors HTTP 429). */
|
|
308
314
|
readonly OverCapacity: 4429;
|
|
309
315
|
};
|
|
@@ -360,7 +366,7 @@ export interface HearPartialFrame {
|
|
|
360
366
|
/** The still-changing tail. */
|
|
361
367
|
active_text?: string;
|
|
362
368
|
utterance_id: string;
|
|
363
|
-
/**
|
|
369
|
+
/** Elapsed server time when the event is created, ms; not a word timestamp. */
|
|
364
370
|
t_ms: number;
|
|
365
371
|
}
|
|
366
372
|
/** Finalized-utterance frame (`speech_final` / `final`). */
|
|
@@ -368,7 +374,25 @@ export type HearEndpointReason = "peak_te_early" | "silence_backstop" | "commit"
|
|
|
368
374
|
export interface HearFinalFrame {
|
|
369
375
|
type: "speech_final" | "final";
|
|
370
376
|
text: string;
|
|
377
|
+
/** Final words in seconds from stream start, including silence, when timing is available. */
|
|
378
|
+
words?: Array<{
|
|
379
|
+
word: string;
|
|
380
|
+
start: number;
|
|
381
|
+
end: number;
|
|
382
|
+
timing_status?: undefined;
|
|
383
|
+
} | {
|
|
384
|
+
word: string;
|
|
385
|
+
start: null;
|
|
386
|
+
end: null;
|
|
387
|
+
timing_status: "unavailable";
|
|
388
|
+
}>;
|
|
389
|
+
/** Some or all word offsets are unavailable. The utterance bounds still locate the sentence. */
|
|
390
|
+
word_timing_status?: "partial" | "unavailable";
|
|
391
|
+
/** Bounds of the utterance's audio, seconds from the beginning of this stream. */
|
|
392
|
+
start?: number;
|
|
393
|
+
end?: number;
|
|
371
394
|
utterance_id: string;
|
|
395
|
+
/** Elapsed server time when the event is created, ms; use words for speaker ordering. */
|
|
372
396
|
t_ms: number;
|
|
373
397
|
/** Active-speech length of the utterance (the billed signal), ms. */
|
|
374
398
|
audio_ms: number;
|
|
@@ -377,10 +401,7 @@ export interface HearFinalFrame {
|
|
|
377
401
|
/** Present only with Cue grounding enabled (top KB passages). */
|
|
378
402
|
grounding?: HearGroundingPassage[];
|
|
379
403
|
}
|
|
380
|
-
/**
|
|
381
|
-
* so realtime spend can be reconciled in-band (a realtime WS carries no
|
|
382
|
-
* `x-pyai-units` response header). Best-effort: absent if there was no billable
|
|
383
|
-
* audio or the close was abnormal. */
|
|
404
|
+
/** Terminal billed-usage summary (`usage`), when available. */
|
|
384
405
|
export interface HearUsageFrame {
|
|
385
406
|
type: "usage";
|
|
386
407
|
/** `hear` for plain streaming, `cue` when grounding was enabled. */
|
|
@@ -389,8 +410,20 @@ export interface HearUsageFrame {
|
|
|
389
410
|
meter: string;
|
|
390
411
|
/** Summed active-speech audio billed for the session, in seconds. */
|
|
391
412
|
audio_seconds: number;
|
|
392
|
-
/**
|
|
413
|
+
/** Billed meter units from the wire. Hear measures usage in minutes. */
|
|
414
|
+
units?: number;
|
|
415
|
+
/** Billed minutes, also provided for clients using the original SDK field. */
|
|
393
416
|
minutes: number;
|
|
417
|
+
/** Coverage reported when a timed stream finishes under the best-effort policy. */
|
|
418
|
+
word_timing_status?: "complete" | "partial";
|
|
419
|
+
/** Number of utterances with one or more unavailable word offsets on this socket. */
|
|
420
|
+
utterances_without_word_times?: number;
|
|
421
|
+
}
|
|
422
|
+
/** One utterance could not be protected. Later verified speech may follow. */
|
|
423
|
+
export interface HearTranscriptGapFrame {
|
|
424
|
+
type: "transcript_gap";
|
|
425
|
+
code: "redaction_unavailable";
|
|
426
|
+
utterance_id: string;
|
|
394
427
|
}
|
|
395
428
|
/** Server fault frame (`error`). */
|
|
396
429
|
export interface HearErrorFrame {
|
|
@@ -398,7 +431,7 @@ export interface HearErrorFrame {
|
|
|
398
431
|
code?: string;
|
|
399
432
|
message: string;
|
|
400
433
|
}
|
|
401
|
-
export type HearFrame = HearConfigAckFrame | HearPartialFrame | HearFinalFrame | HearUsageFrame | HearErrorFrame;
|
|
434
|
+
export type HearFrame = HearConfigAckFrame | HearPartialFrame | HearFinalFrame | HearTranscriptGapFrame | HearUsageFrame | HearErrorFrame;
|
|
402
435
|
/**
|
|
403
436
|
* Minimal structural WebSocket, matches both the browser/Node global
|
|
404
437
|
* `WebSocket` and the `ws` package, and lets tests inject a mock.
|
|
@@ -441,6 +474,8 @@ export interface HearStreamOptions {
|
|
|
441
474
|
sampleRate?: number;
|
|
442
475
|
/** Audio frame encoding. Default "pcm16". */
|
|
443
476
|
encoding?: "pcm16";
|
|
477
|
+
/** Override the account default when word spans are missing on a timed stream. */
|
|
478
|
+
wordTimingPolicy?: "strict" | "best_effort";
|
|
444
479
|
/** Emit eager partial hypotheses. Default true server-side. */
|
|
445
480
|
interimResults?: boolean;
|
|
446
481
|
/**
|
|
@@ -499,6 +534,8 @@ export interface HearStreamOptions {
|
|
|
499
534
|
onPartial?: (frame: HearPartialFrame) => void;
|
|
500
535
|
/** Fired on `speech_final` / `final`. */
|
|
501
536
|
onFinal?: (frame: HearFinalFrame) => void;
|
|
537
|
+
/** Discard any prior preview for this utterance and continue reading later finals. */
|
|
538
|
+
onTranscriptGap?: (frame: HearTranscriptGapFrame) => void;
|
|
502
539
|
/** Fired on the final `usage` summary (in-band realtime reconciliation). */
|
|
503
540
|
onUsage?: (frame: HearUsageFrame) => void;
|
|
504
541
|
/** Fired on an `error` frame or a transport-level error. */
|
package/dist/index.js
CHANGED
|
@@ -52,6 +52,8 @@ export const HearFrameType = {
|
|
|
52
52
|
SpeechFinal: "speech_final",
|
|
53
53
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
54
54
|
Final: "final",
|
|
55
|
+
/** Protected speech was withheld for this utterance. */
|
|
56
|
+
TranscriptGap: "transcript_gap",
|
|
55
57
|
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
56
58
|
Usage: "usage",
|
|
57
59
|
/** Server-side fault frame. */
|
|
@@ -71,6 +73,8 @@ export const WSCloseCode = {
|
|
|
71
73
|
PolicyViolation: 1008,
|
|
72
74
|
/** Engine/internal error. */
|
|
73
75
|
InternalError: 1011,
|
|
76
|
+
/** Hear admission is busy after the socket upgrade. Retry with backoff. */
|
|
77
|
+
TryAgainLater: 1013,
|
|
74
78
|
/** Over the concurrency cap (PyAI-specific; mirrors HTTP 429). */
|
|
75
79
|
OverCapacity: 4429,
|
|
76
80
|
};
|
|
@@ -165,8 +169,13 @@ export class HearStream {
|
|
|
165
169
|
case HearFrameType.Final:
|
|
166
170
|
this.opts.onFinal?.(frame);
|
|
167
171
|
break;
|
|
172
|
+
case HearFrameType.TranscriptGap:
|
|
173
|
+
this.opts.onTranscriptGap?.(frame);
|
|
174
|
+
break;
|
|
168
175
|
case HearFrameType.Usage:
|
|
169
|
-
this.opts.onUsage?.(frame
|
|
176
|
+
this.opts.onUsage?.(frame.minutes === undefined && typeof frame.units === "number"
|
|
177
|
+
? { ...frame, minutes: frame.units }
|
|
178
|
+
: frame);
|
|
170
179
|
break;
|
|
171
180
|
case HearFrameType.Error:
|
|
172
181
|
this.opts.onError?.(frame);
|
|
@@ -1538,6 +1547,8 @@ export class PyAI {
|
|
|
1538
1547
|
q.set("sample_rate", String(opts.sampleRate));
|
|
1539
1548
|
if (opts.encoding)
|
|
1540
1549
|
q.set("encoding", opts.encoding);
|
|
1550
|
+
if (opts.wordTimingPolicy !== undefined)
|
|
1551
|
+
q.set("word_timing_policy", opts.wordTimingPolicy);
|
|
1541
1552
|
if (opts.interimResults !== undefined)
|
|
1542
1553
|
q.set("interim_results", String(opts.interimResults));
|
|
1543
1554
|
if (opts.numerals !== undefined)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pyai/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Official TypeScript/JavaScript SDK for PyAI, speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
package/src/cli-dx.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
|
|
3
|
+
const sdkVersion: string = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
4
|
+
|
|
1
5
|
/** Offline discovery and copyable workflows; recipes never execute commands. */
|
|
2
6
|
export const aliases: Record<string, string> = Object.assign(Object.create(null), {
|
|
3
7
|
login: "auth login", logout: "auth logout", whoami: "auth status", use: "profiles use",
|
|
@@ -35,5 +39,5 @@ export const recipes: Recipe[] = [
|
|
|
35
39
|
notes: ["Local schema and recipes work without credentials. Live OpenAPI needs a network connection but no API key.", "The CLI manages Agent profiles; use the SDK or WebSocket API for a live Omni audio session."] },
|
|
36
40
|
{ name: "ci", title: "Use PyAI in a build", description: "Check access and produce an audio artifact with machine-readable output.",
|
|
37
41
|
commands: ["pyai whoami --json", 'pyai speak "Your build is ready" -o build.wav --json', "pyai doctor --json"],
|
|
38
|
-
notes: ["Set PYAI_API_KEY through the CI secret store; do not put it in source or shell arguments.", "doctor synthesizes and transcribes a sample, which consumes usage. Use whoami for an authentication-only check.",
|
|
42
|
+
notes: ["Set PYAI_API_KEY through the CI secret store; do not put it in source or shell arguments.", "doctor synthesizes and transcribes a sample, which consumes usage. Use whoami for an authentication-only check.", `Install the CLI with npm install -g @pyai/sdk@${sdkVersion} and pin that version in repeatable builds.`] },
|
|
39
43
|
].map(recipe => ({ ...recipe, docs_url: "https://pyai.com/agents/speech-calling.md" }));
|
package/src/cli-init.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { constants, type Stats } from "node:fs";
|
|
1
|
+
import { constants, readFileSync, type Stats } from "node:fs";
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import { basename, dirname, isAbsolute, join, parse, resolve } from "node:path";
|
|
4
4
|
|
|
@@ -32,6 +32,8 @@ export class CliInitError extends Error {
|
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
const sdkVersion: string = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
36
|
+
|
|
35
37
|
const context = `# Build with PyAI
|
|
36
38
|
|
|
37
39
|
This file is a project-local API integration guide for engineers and coding agents.
|
|
@@ -151,7 +153,7 @@ const install = `## Install the CLI
|
|
|
151
153
|
Install the published CLI with Node.js 22 or newer recommended:
|
|
152
154
|
|
|
153
155
|
\`\`\`bash
|
|
154
|
-
npm install -g @pyai/sdk
|
|
156
|
+
npm install -g @pyai/sdk@${sdkVersion}
|
|
155
157
|
pyai --version
|
|
156
158
|
\`\`\`
|
|
157
159
|
|
|
@@ -244,7 +246,7 @@ and product calls below are separate explicit steps. Read \`PYAI.md\` first.
|
|
|
244
246
|
Install the official SDK:
|
|
245
247
|
|
|
246
248
|
\`\`\`bash
|
|
247
|
-
npm install @pyai/sdk
|
|
249
|
+
npm install @pyai/sdk@${sdkVersion}
|
|
248
250
|
cp .env.example .env
|
|
249
251
|
\`\`\`
|
|
250
252
|
|
package/src/cli-omni-template.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.19.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.19.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/src/index.ts
CHANGED
|
@@ -126,6 +126,8 @@ export type SpeakModel = "pyai-speak" | "tts-1" | "tts-1-hd";
|
|
|
126
126
|
|
|
127
127
|
export interface SpeechParams {
|
|
128
128
|
input: string;
|
|
129
|
+
/** Reuse one unique call ID across utterances for enabled call-stable voice recovery. */
|
|
130
|
+
speech_session_id?: string;
|
|
129
131
|
voice?: string;
|
|
130
132
|
model?: SpeakModel;
|
|
131
133
|
/**
|
|
@@ -330,6 +332,8 @@ export const HearFrameType = {
|
|
|
330
332
|
SpeechFinal: "speech_final",
|
|
331
333
|
/** Corrected, full-context transcript following `speech_final`. */
|
|
332
334
|
Final: "final",
|
|
335
|
+
/** Protected speech was withheld for this utterance. */
|
|
336
|
+
TranscriptGap: "transcript_gap",
|
|
333
337
|
/** Final billed-usage summary, emitted just before a graceful close. */
|
|
334
338
|
Usage: "usage",
|
|
335
339
|
/** Server-side fault frame. */
|
|
@@ -351,6 +355,8 @@ export const WSCloseCode = {
|
|
|
351
355
|
PolicyViolation: 1008,
|
|
352
356
|
/** Engine/internal error. */
|
|
353
357
|
InternalError: 1011,
|
|
358
|
+
/** Hear admission is busy after the socket upgrade. Retry with backoff. */
|
|
359
|
+
TryAgainLater: 1013,
|
|
354
360
|
/** Over the concurrency cap (PyAI-specific; mirrors HTTP 429). */
|
|
355
361
|
OverCapacity: 4429,
|
|
356
362
|
} as const;
|
|
@@ -416,7 +422,7 @@ export interface HearPartialFrame {
|
|
|
416
422
|
/** The still-changing tail. */
|
|
417
423
|
active_text?: string;
|
|
418
424
|
utterance_id: string;
|
|
419
|
-
/**
|
|
425
|
+
/** Elapsed server time when the event is created, ms; not a word timestamp. */
|
|
420
426
|
t_ms: number;
|
|
421
427
|
}
|
|
422
428
|
|
|
@@ -430,7 +436,18 @@ export type HearEndpointReason =
|
|
|
430
436
|
export interface HearFinalFrame {
|
|
431
437
|
type: "speech_final" | "final";
|
|
432
438
|
text: string;
|
|
439
|
+
/** Final words in seconds from stream start, including silence, when timing is available. */
|
|
440
|
+
words?: Array<
|
|
441
|
+
| { word: string; start: number; end: number; timing_status?: undefined }
|
|
442
|
+
| { word: string; start: null; end: null; timing_status: "unavailable" }
|
|
443
|
+
>;
|
|
444
|
+
/** Some or all word offsets are unavailable. The utterance bounds still locate the sentence. */
|
|
445
|
+
word_timing_status?: "partial" | "unavailable";
|
|
446
|
+
/** Bounds of the utterance's audio, seconds from the beginning of this stream. */
|
|
447
|
+
start?: number;
|
|
448
|
+
end?: number;
|
|
433
449
|
utterance_id: string;
|
|
450
|
+
/** Elapsed server time when the event is created, ms; use words for speaker ordering. */
|
|
434
451
|
t_ms: number;
|
|
435
452
|
/** Active-speech length of the utterance (the billed signal), ms. */
|
|
436
453
|
audio_ms: number;
|
|
@@ -440,10 +457,7 @@ export interface HearFinalFrame {
|
|
|
440
457
|
grounding?: HearGroundingPassage[];
|
|
441
458
|
}
|
|
442
459
|
|
|
443
|
-
/**
|
|
444
|
-
* so realtime spend can be reconciled in-band (a realtime WS carries no
|
|
445
|
-
* `x-pyai-units` response header). Best-effort: absent if there was no billable
|
|
446
|
-
* audio or the close was abnormal. */
|
|
460
|
+
/** Terminal billed-usage summary (`usage`), when available. */
|
|
447
461
|
export interface HearUsageFrame {
|
|
448
462
|
type: "usage";
|
|
449
463
|
/** `hear` for plain streaming, `cue` when grounding was enabled. */
|
|
@@ -452,8 +466,21 @@ export interface HearUsageFrame {
|
|
|
452
466
|
meter: string;
|
|
453
467
|
/** Summed active-speech audio billed for the session, in seconds. */
|
|
454
468
|
audio_seconds: number;
|
|
455
|
-
/**
|
|
469
|
+
/** Billed meter units from the wire. Hear measures usage in minutes. */
|
|
470
|
+
units?: number;
|
|
471
|
+
/** Billed minutes, also provided for clients using the original SDK field. */
|
|
456
472
|
minutes: number;
|
|
473
|
+
/** Coverage reported when a timed stream finishes under the best-effort policy. */
|
|
474
|
+
word_timing_status?: "complete" | "partial";
|
|
475
|
+
/** Number of utterances with one or more unavailable word offsets on this socket. */
|
|
476
|
+
utterances_without_word_times?: number;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** One utterance could not be protected. Later verified speech may follow. */
|
|
480
|
+
export interface HearTranscriptGapFrame {
|
|
481
|
+
type: "transcript_gap";
|
|
482
|
+
code: "redaction_unavailable";
|
|
483
|
+
utterance_id: string;
|
|
457
484
|
}
|
|
458
485
|
|
|
459
486
|
/** Server fault frame (`error`). */
|
|
@@ -467,6 +494,7 @@ export type HearFrame =
|
|
|
467
494
|
| HearConfigAckFrame
|
|
468
495
|
| HearPartialFrame
|
|
469
496
|
| HearFinalFrame
|
|
497
|
+
| HearTranscriptGapFrame
|
|
470
498
|
| HearUsageFrame
|
|
471
499
|
| HearErrorFrame;
|
|
472
500
|
|
|
@@ -510,6 +538,8 @@ export interface HearStreamOptions {
|
|
|
510
538
|
sampleRate?: number;
|
|
511
539
|
/** Audio frame encoding. Default "pcm16". */
|
|
512
540
|
encoding?: "pcm16";
|
|
541
|
+
/** Override the account default when word spans are missing on a timed stream. */
|
|
542
|
+
wordTimingPolicy?: "strict" | "best_effort";
|
|
513
543
|
/** Emit eager partial hypotheses. Default true server-side. */
|
|
514
544
|
interimResults?: boolean;
|
|
515
545
|
/**
|
|
@@ -568,6 +598,8 @@ export interface HearStreamOptions {
|
|
|
568
598
|
onPartial?: (frame: HearPartialFrame) => void;
|
|
569
599
|
/** Fired on `speech_final` / `final`. */
|
|
570
600
|
onFinal?: (frame: HearFinalFrame) => void;
|
|
601
|
+
/** Discard any prior preview for this utterance and continue reading later finals. */
|
|
602
|
+
onTranscriptGap?: (frame: HearTranscriptGapFrame) => void;
|
|
571
603
|
/** Fired on the final `usage` summary (in-band realtime reconciliation). */
|
|
572
604
|
onUsage?: (frame: HearUsageFrame) => void;
|
|
573
605
|
/** Fired on an `error` frame or a transport-level error. */
|
|
@@ -638,8 +670,13 @@ export class HearStream {
|
|
|
638
670
|
case HearFrameType.Final:
|
|
639
671
|
this.opts.onFinal?.(frame);
|
|
640
672
|
break;
|
|
673
|
+
case HearFrameType.TranscriptGap:
|
|
674
|
+
this.opts.onTranscriptGap?.(frame);
|
|
675
|
+
break;
|
|
641
676
|
case HearFrameType.Usage:
|
|
642
|
-
this.opts.onUsage?.(frame
|
|
677
|
+
this.opts.onUsage?.(frame.minutes === undefined && typeof frame.units === "number"
|
|
678
|
+
? { ...frame, minutes: frame.units }
|
|
679
|
+
: frame);
|
|
643
680
|
break;
|
|
644
681
|
case HearFrameType.Error:
|
|
645
682
|
this.opts.onError?.(frame);
|
|
@@ -2878,6 +2915,7 @@ export class PyAI {
|
|
|
2878
2915
|
if (opts.language) q.set("language", opts.language);
|
|
2879
2916
|
if (opts.sampleRate !== undefined) q.set("sample_rate", String(opts.sampleRate));
|
|
2880
2917
|
if (opts.encoding) q.set("encoding", opts.encoding);
|
|
2918
|
+
if (opts.wordTimingPolicy !== undefined) q.set("word_timing_policy", opts.wordTimingPolicy);
|
|
2881
2919
|
if (opts.interimResults !== undefined) q.set("interim_results", String(opts.interimResults));
|
|
2882
2920
|
if (opts.numerals !== undefined) q.set("numerals", String(opts.numerals));
|
|
2883
2921
|
if (opts.smartFormat !== undefined) q.set("smart_format", String(opts.smartFormat));
|