@pyai/sdk 0.7.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -2
- package/dist/cli-omni-template.d.ts +2 -2
- package/dist/cli-omni-template.js +2 -2
- package/dist/index.d.ts +21 -3
- package/dist/index.js +148 -13
- package/package.json +1 -1
- package/src/cli-omni-template.ts +2 -2
- package/src/index.ts +144 -15
package/README.md
CHANGED
|
@@ -81,12 +81,12 @@ the instant the socket opens and routes server frames to typed callbacks, so you
|
|
|
81
81
|
is acked but silently dropped, giving you a connected session with zero turns):
|
|
82
82
|
|
|
83
83
|
<!-- omni-starter:start -->
|
|
84
|
-
Requires SDK 0.
|
|
84
|
+
Requires SDK 0.8.0 for `--template omni`. The runnable starter uses ESM,
|
|
85
85
|
Node 20.19+ with an explicit `ws` transport, or Node 22+. MCP 0.5.0
|
|
86
86
|
requires Node 22+. The SDK's REST client continues to support Node 18+.
|
|
87
87
|
|
|
88
88
|
```sh
|
|
89
|
-
npm install @pyai/sdk@0.
|
|
89
|
+
npm install @pyai/sdk@0.8.0
|
|
90
90
|
npx pyai init voice-demo --template omni
|
|
91
91
|
cd voice-demo
|
|
92
92
|
npm install
|
|
@@ -443,3 +443,43 @@ npm install
|
|
|
443
443
|
npm test # node --test, fetch injected (no network)
|
|
444
444
|
npm run build # emits dist/ (incl. the pyai CLI bin)
|
|
445
445
|
```
|
|
446
|
+
|
|
447
|
+
## Network recovery
|
|
448
|
+
|
|
449
|
+
REST automatically retries GET/HEAD only: transient connection failures with a
|
|
450
|
+
recognized cause, and HTTP 429/500/502/503/504. `maxRetries` defaults to 2;
|
|
451
|
+
`retryWindowMs` defaults to 10000. Backoff uses jitter and respects both numeric
|
|
452
|
+
and HTTP-date `Retry-After`. A delay outside the remaining window returns the
|
|
453
|
+
failure instead of retrying early. Original transport exceptions retain their
|
|
454
|
+
cause. An unclassified browser `TypeError` is surfaced without retry.
|
|
455
|
+
|
|
456
|
+
The window bounds admission of retries, **not** an in-flight request or response
|
|
457
|
+
body. Pass a caller-owned `signal` (for example `AbortSignal.timeout(10000)` where
|
|
458
|
+
supported) to bound the REST operation and cancel backoff. A signal belongs to
|
|
459
|
+
its deadline: create a new client/signal for a later independent operation.
|
|
460
|
+
Custom fetch implementations must honor the signal. Response-body failures after
|
|
461
|
+
successful headers are surfaced; the SDK does not replay a partially read body.
|
|
462
|
+
|
|
463
|
+
Writes, including speech and job creation, are no longer automatically retried
|
|
464
|
+
on server errors. The server may already have acted even when no response audio
|
|
465
|
+
arrived. Where the endpoint documents idempotency, applications can explicitly
|
|
466
|
+
retry the identical operation with the same supported idempotency key. An
|
|
467
|
+
arbitrary header does not make every endpoint safe to replay.
|
|
468
|
+
|
|
469
|
+
For an explicit Omni restoration attempt, use `omni.connect({ resumeToken,
|
|
470
|
+
resumeTimeoutMs: 5000, ... })` with the single-use token from the previous
|
|
471
|
+
`session_started`. The SDK waits for `session_started.resumed === true` before
|
|
472
|
+
sending automatic configuration or delivering media/tools. Sending audio or
|
|
473
|
+
controls earlier throws. Rejection or a missing acknowledgement closes with
|
|
474
|
+
`resume_failed`; it cannot silently become a new conversation. `onOpen` only
|
|
475
|
+
means the socket opened. After successful restoration, follow the normal
|
|
476
|
+
`configured` readiness rules before sending microphone audio. Fresh connections
|
|
477
|
+
are unchanged. The same guard applies to `query.resume_token`.
|
|
478
|
+
|
|
479
|
+
This is an explicit, bounded restoration attempt, not automatic reconnection.
|
|
480
|
+
Do not loop a consumed token, replay unacknowledged audio, or replay tool actions.
|
|
481
|
+
Store rotated tokens privately; do not log connection URLs containing them.
|
|
482
|
+
Clear queued playback after a disconnect and tell the caller when speech was
|
|
483
|
+
lost. Conversation-state restoration does not guarantee recovery of audio sent
|
|
484
|
+
immediately before a disconnect. Python currently provides URL helpers rather
|
|
485
|
+
than this WebSocket lifecycle guard.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export declare const omniTemplate: Readonly<{
|
|
2
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
3
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
2
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.8.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nCapture ends after the simulated queue drains and two seconds of quiet. Omni\nhas no protocol reply-end marker, so this is a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n";
|
|
3
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.8.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n";
|
|
4
4
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n";
|
|
5
5
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n";
|
|
6
6
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.8.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nCapture ends after the simulated queue drains and two seconds of quiet. Omni\nhas no protocol reply-end marker, so this is a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.8.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/dist/index.d.ts
CHANGED
|
@@ -13,8 +13,12 @@ export interface PyAIOptions {
|
|
|
13
13
|
baseURL?: string;
|
|
14
14
|
/** Injectable for tests / custom transports. Defaults to global fetch. */
|
|
15
15
|
fetch?: typeof fetch;
|
|
16
|
-
/** Retries on
|
|
16
|
+
/** Retries safe reads on transient failures. Writes are never replayed. Default 2. */
|
|
17
17
|
maxRetries?: number;
|
|
18
|
+
/** Retry admission window, in ms. Does not time out an in-flight request/body. Default 10000. */
|
|
19
|
+
retryWindowMs?: number;
|
|
20
|
+
/** Optional caller-owned cancellation/deadline signal for REST requests and backoff. */
|
|
21
|
+
signal?: AbortSignal;
|
|
18
22
|
}
|
|
19
23
|
/** Stable, machine-readable error. Branch on `code`, not `message`. */
|
|
20
24
|
export declare class PyAIError extends Error {
|
|
@@ -184,12 +188,16 @@ export interface HearVocabulary {
|
|
|
184
188
|
/** Unix milliseconds, or null before the first save. */
|
|
185
189
|
updated_at: number | null;
|
|
186
190
|
}
|
|
191
|
+
/** Canonical Omni session language identifiers accepted by the public API. */
|
|
192
|
+
export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh";
|
|
193
|
+
/** Internal candidate identifiers; the public control plane rejects these until qualified. */
|
|
194
|
+
export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
|
|
187
195
|
export interface AgentConfig {
|
|
188
196
|
name?: string;
|
|
189
197
|
persona_system_prompt?: string | null;
|
|
190
198
|
greeting?: string | null;
|
|
191
199
|
voice_id?: string | null;
|
|
192
|
-
language?:
|
|
200
|
+
language?: OmniLanguage | null;
|
|
193
201
|
/**
|
|
194
202
|
* Opt-in speech-recognition vocabulary. PyAI sanitizes and keeps at most
|
|
195
203
|
* five terms. An empty list or null turns it off.
|
|
@@ -632,7 +640,7 @@ export interface OmniConfigure {
|
|
|
632
640
|
* proceeds and bills as what was served. Per-language availability is
|
|
633
641
|
* staged, see the Language support reference.
|
|
634
642
|
*/
|
|
635
|
-
language?:
|
|
643
|
+
language?: OmniLanguage;
|
|
636
644
|
/** Function calling: hosted catalog names, client-loop schemas, or names of
|
|
637
645
|
* server tools already registered with POST /v1/tools. An inline `endpoint`
|
|
638
646
|
* is rejected with `unsupported_tool_transport`. */
|
|
@@ -643,6 +651,10 @@ export interface OmniConfigure {
|
|
|
643
651
|
export interface OmniConnectOptions {
|
|
644
652
|
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
645
653
|
sessionLabel?: string;
|
|
654
|
+
/** Single-use token from session_started. Failed restoration closes instead of starting fresh. */
|
|
655
|
+
resumeToken?: string;
|
|
656
|
+
/** Maximum wait for resume acknowledgement, including socket opening. Default 5000 ms. */
|
|
657
|
+
resumeTimeoutMs?: number;
|
|
646
658
|
/**
|
|
647
659
|
* Ephemeral browser token from `omni.createSession`. When set it is used as
|
|
648
660
|
* the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
|
|
@@ -713,11 +725,15 @@ export declare class OmniConnection {
|
|
|
713
725
|
private readonly ws;
|
|
714
726
|
private readonly opts;
|
|
715
727
|
private closed;
|
|
728
|
+
private awaitingResume;
|
|
729
|
+
private resumeTimer?;
|
|
716
730
|
/** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
|
|
717
731
|
private blobTail;
|
|
718
732
|
/** Serializes inbound Blob decoding so browser frames stay ordered. */
|
|
719
733
|
private inboundTail;
|
|
720
734
|
constructor(url: string, subprotocol: string, opts: OmniConnectOptions);
|
|
735
|
+
private rejectResume;
|
|
736
|
+
private assertSendable;
|
|
721
737
|
private handleMessage;
|
|
722
738
|
private dispatchFrame;
|
|
723
739
|
/**
|
|
@@ -1146,6 +1162,8 @@ export declare class PyAI {
|
|
|
1146
1162
|
private readonly baseURL;
|
|
1147
1163
|
private readonly fetchImpl;
|
|
1148
1164
|
private readonly maxRetries;
|
|
1165
|
+
private readonly retryWindowMs;
|
|
1166
|
+
private readonly signal?;
|
|
1149
1167
|
constructor(opts: PyAIOptions);
|
|
1150
1168
|
private authHeaders;
|
|
1151
1169
|
private request;
|
package/dist/index.js
CHANGED
|
@@ -429,19 +429,29 @@ export class OmniConnection {
|
|
|
429
429
|
ws;
|
|
430
430
|
opts;
|
|
431
431
|
closed = false;
|
|
432
|
+
awaitingResume = false;
|
|
433
|
+
resumeTimer;
|
|
432
434
|
/** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
|
|
433
435
|
blobTail = Promise.resolve();
|
|
434
436
|
/** Serializes inbound Blob decoding so browser frames stay ordered. */
|
|
435
437
|
inboundTail = Promise.resolve();
|
|
436
438
|
constructor(url, subprotocol, opts) {
|
|
437
439
|
this.opts = opts;
|
|
440
|
+
this.awaitingResume = new URL(url).searchParams.has("resume_token");
|
|
441
|
+
const resumeTimeout = opts.resumeTimeoutMs ?? 5000;
|
|
442
|
+
if (!Number.isFinite(resumeTimeout) || resumeTimeout <= 0 || resumeTimeout > 30_000) {
|
|
443
|
+
throw new Error("resumeTimeoutMs must be greater than zero and at most 30000");
|
|
444
|
+
}
|
|
438
445
|
const WS = opts.webSocket ?? globalThis.WebSocket;
|
|
439
446
|
if (!WS) {
|
|
440
447
|
throw new Error("No global WebSocket available; pass options.webSocket (e.g. the `ws` package) to omni.connect()");
|
|
441
448
|
}
|
|
442
449
|
this.ws = new WS(url, [REALTIME_SUBPROTOCOL_MARKER, subprotocol]);
|
|
450
|
+
if (this.awaitingResume) {
|
|
451
|
+
this.resumeTimer = setTimeout(() => this.rejectResume("Resume acknowledgement timed out"), resumeTimeout);
|
|
452
|
+
}
|
|
443
453
|
this.ws.onopen = () => {
|
|
444
|
-
if (opts.configure) {
|
|
454
|
+
if (opts.configure && !this.awaitingResume) {
|
|
445
455
|
try {
|
|
446
456
|
this.configure(opts.configure);
|
|
447
457
|
}
|
|
@@ -464,11 +474,26 @@ export class OmniConnection {
|
|
|
464
474
|
};
|
|
465
475
|
this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
|
|
466
476
|
this.ws.onclose = (ev) => {
|
|
477
|
+
clearTimeout(this.resumeTimer);
|
|
467
478
|
this.closed = true;
|
|
468
479
|
opts.onClose?.(ev.code, ev.reason);
|
|
469
480
|
};
|
|
470
481
|
}
|
|
482
|
+
rejectResume(message) {
|
|
483
|
+
if (this.closed)
|
|
484
|
+
return;
|
|
485
|
+
this.close(4000, "resume_failed");
|
|
486
|
+
this.opts.onError?.(new PyAIError(0, message, "resume_failed"));
|
|
487
|
+
}
|
|
488
|
+
assertSendable() {
|
|
489
|
+
if (this.closed)
|
|
490
|
+
throw new Error("Omni connection is closed");
|
|
491
|
+
if (this.awaitingResume)
|
|
492
|
+
throw new Error("Wait for a confirmed session resume before sending");
|
|
493
|
+
}
|
|
471
494
|
async handleMessage(data) {
|
|
495
|
+
if (this.closed)
|
|
496
|
+
return;
|
|
472
497
|
// Server → client binary frames are TYPE-TAGGED by their first byte:
|
|
473
498
|
// 0x01 = agent audio (PCM16) · 0x02 = transcript UTF-8 · 0x03 = control JSON.
|
|
474
499
|
// (Treating every binary frame as audio, the old behavior, plays the
|
|
@@ -481,7 +506,11 @@ export class OmniConnection {
|
|
|
481
506
|
this.opts.onError?.(new Error("Unsupported Omni binary frame"));
|
|
482
507
|
return;
|
|
483
508
|
}
|
|
509
|
+
if (this.closed)
|
|
510
|
+
return;
|
|
484
511
|
const tag = bytes[0];
|
|
512
|
+
if (this.awaitingResume && tag !== 0x03)
|
|
513
|
+
return;
|
|
485
514
|
if (tag === 0x01) {
|
|
486
515
|
this.opts.onAudio?.(bytes.slice(1)); // copy → aligned PCM16
|
|
487
516
|
return;
|
|
@@ -497,6 +526,8 @@ export class OmniConnection {
|
|
|
497
526
|
if (tag === 0x03) {
|
|
498
527
|
try {
|
|
499
528
|
const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(1)));
|
|
529
|
+
if (this.awaitingResume && parsed?.event !== OmniEvent.Hello && parsed?.event !== OmniEvent.SessionStarted && parsed?.event !== OmniEvent.Error)
|
|
530
|
+
return;
|
|
500
531
|
if (parsed?.event === OmniEvent.Transcript) {
|
|
501
532
|
// The deployed engine also sends this bounded synthesis-input
|
|
502
533
|
// advisory. It is not caller ASR or proof that speech was played.
|
|
@@ -531,6 +562,23 @@ export class OmniConnection {
|
|
|
531
562
|
this.opts.onError?.(new Error("Omni server control frame is missing its event key"));
|
|
532
563
|
return;
|
|
533
564
|
}
|
|
565
|
+
if (this.closed)
|
|
566
|
+
return;
|
|
567
|
+
if (this.awaitingResume) {
|
|
568
|
+
if (eventName === OmniEvent.SessionStarted) {
|
|
569
|
+
if (frame.resumed !== true) {
|
|
570
|
+
this.rejectResume("The server could not restore the previous conversation");
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
clearTimeout(this.resumeTimer);
|
|
574
|
+
this.awaitingResume = false;
|
|
575
|
+
if (this.opts.configure)
|
|
576
|
+
this.configure(this.opts.configure);
|
|
577
|
+
}
|
|
578
|
+
else if (eventName !== OmniEvent.Hello && eventName !== OmniEvent.Error) {
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
}
|
|
534
582
|
this.opts.onEvent?.(frame);
|
|
535
583
|
switch (eventName) {
|
|
536
584
|
case OmniEvent.Hello:
|
|
@@ -572,6 +620,7 @@ export class OmniConnection {
|
|
|
572
620
|
* `{"event":"configure"}` is acked but silently dropped by the engine.)
|
|
573
621
|
*/
|
|
574
622
|
configure(cfg) {
|
|
623
|
+
this.assertSendable();
|
|
575
624
|
const payload = { ...cfg };
|
|
576
625
|
delete payload.type;
|
|
577
626
|
delete payload.event;
|
|
@@ -583,6 +632,7 @@ export class OmniConnection {
|
|
|
583
632
|
* default branch, so an untagged chunk is dropped with no error.
|
|
584
633
|
*/
|
|
585
634
|
sendAudio(chunk) {
|
|
635
|
+
this.assertSendable();
|
|
586
636
|
const bytes = omniToBytes(chunk);
|
|
587
637
|
if (bytes) {
|
|
588
638
|
this.ws.send(omniAudioFrame(bytes));
|
|
@@ -593,16 +643,19 @@ export class OmniConnection {
|
|
|
593
643
|
this.blobTail = this.blobTail
|
|
594
644
|
.then(async () => {
|
|
595
645
|
const buf = await chunk.arrayBuffer();
|
|
646
|
+
this.assertSendable();
|
|
596
647
|
this.ws.send(omniAudioFrame(new Uint8Array(buf)));
|
|
597
648
|
})
|
|
598
649
|
.catch((err) => this.opts.onError?.(err instanceof Error ? err : new Error(String(err))));
|
|
599
650
|
}
|
|
600
651
|
/** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
|
|
601
652
|
sendDtmf(digits) {
|
|
653
|
+
this.assertSendable();
|
|
602
654
|
this.ws.send(omniControlFrame({ type: "dtmf", digits }));
|
|
603
655
|
}
|
|
604
656
|
/** Reply to a client-loop {@link OmniEvent.ToolCall}. */
|
|
605
657
|
toolResult(callId, payload) {
|
|
658
|
+
this.assertSendable();
|
|
606
659
|
this.ws.send(omniControlFrame({ type: "tool_result", call_id: callId, ...payload }));
|
|
607
660
|
}
|
|
608
661
|
/**
|
|
@@ -611,6 +664,7 @@ export class OmniConnection {
|
|
|
611
664
|
* never `event`.
|
|
612
665
|
*/
|
|
613
666
|
send(frame) {
|
|
667
|
+
this.assertSendable();
|
|
614
668
|
if (typeof frame.type !== "string" || !frame.type) {
|
|
615
669
|
throw new TypeError("Omni client control frames must have a non-empty type key");
|
|
616
670
|
}
|
|
@@ -620,8 +674,11 @@ export class OmniConnection {
|
|
|
620
674
|
}
|
|
621
675
|
/** Close the session. */
|
|
622
676
|
close(code = WSCloseCode.Normal, reason = "") {
|
|
623
|
-
|
|
677
|
+
clearTimeout(this.resumeTimer);
|
|
678
|
+
if (!this.closed) {
|
|
679
|
+
this.closed = true;
|
|
624
680
|
this.ws.close(code, reason);
|
|
681
|
+
}
|
|
625
682
|
}
|
|
626
683
|
/** The underlying socket (escape hatch for advanced use). */
|
|
627
684
|
get socket() {
|
|
@@ -633,11 +690,46 @@ export class OmniConnection {
|
|
|
633
690
|
}
|
|
634
691
|
}
|
|
635
692
|
const RETRYABLE = new Set([429, 500, 502, 503, 504]);
|
|
693
|
+
// Only transport codes known to be transient; do not retry TLS, URL or DNS
|
|
694
|
+
// configuration errors. Browsers often hide the cause: those propagate intact.
|
|
695
|
+
const TRANSIENT_TRANSPORT = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
|
|
696
|
+
function transientTransport(error) {
|
|
697
|
+
const seen = new Set();
|
|
698
|
+
while (error && typeof error === "object" && !seen.has(error)) {
|
|
699
|
+
seen.add(error);
|
|
700
|
+
const value = error;
|
|
701
|
+
if (value.name === "AbortError" || value.name === "TimeoutError")
|
|
702
|
+
return false;
|
|
703
|
+
if (value.code && TRANSIENT_TRANSPORT.has(value.code))
|
|
704
|
+
return true;
|
|
705
|
+
error = value.cause;
|
|
706
|
+
}
|
|
707
|
+
return false;
|
|
708
|
+
}
|
|
709
|
+
function retryDelay(header, attempt) {
|
|
710
|
+
const jitter = Math.random() * Math.min(2 ** attempt * 250, 2000);
|
|
711
|
+
if (header === null)
|
|
712
|
+
return jitter;
|
|
713
|
+
const seconds = /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) : NaN;
|
|
714
|
+
const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
|
|
715
|
+
return Number.isFinite(delay) ? Math.max(0, delay) + jitter : jitter;
|
|
716
|
+
}
|
|
717
|
+
function retryPause(ms, signal) {
|
|
718
|
+
return new Promise((resolve, reject) => {
|
|
719
|
+
const cancel = () => { clearTimeout(timer); signal?.removeEventListener("abort", cancel); reject(signal?.reason); };
|
|
720
|
+
const timer = setTimeout(() => { signal?.removeEventListener("abort", cancel); resolve(); }, ms);
|
|
721
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
722
|
+
if (signal?.aborted)
|
|
723
|
+
cancel();
|
|
724
|
+
});
|
|
725
|
+
}
|
|
636
726
|
export class PyAI {
|
|
637
727
|
apiKey;
|
|
638
728
|
baseURL;
|
|
639
729
|
fetchImpl;
|
|
640
730
|
maxRetries;
|
|
731
|
+
retryWindowMs;
|
|
732
|
+
signal;
|
|
641
733
|
constructor(opts) {
|
|
642
734
|
if (!opts.apiKey)
|
|
643
735
|
throw new Error("apiKey is required");
|
|
@@ -645,6 +737,12 @@ export class PyAI {
|
|
|
645
737
|
this.baseURL = (opts.baseURL ?? "https://api.pyai.com").replace(/\/$/, "");
|
|
646
738
|
this.fetchImpl = opts.fetch ?? globalThis.fetch;
|
|
647
739
|
this.maxRetries = opts.maxRetries ?? 2;
|
|
740
|
+
this.retryWindowMs = opts.retryWindowMs ?? 10_000;
|
|
741
|
+
this.signal = opts.signal;
|
|
742
|
+
if (!Number.isSafeInteger(this.maxRetries) || this.maxRetries < 0)
|
|
743
|
+
throw new Error("maxRetries must be a non-negative integer");
|
|
744
|
+
if (!Number.isFinite(this.retryWindowMs) || this.retryWindowMs < 0)
|
|
745
|
+
throw new Error("retryWindowMs must be finite and non-negative");
|
|
648
746
|
if (!this.fetchImpl)
|
|
649
747
|
throw new Error("No fetch implementation available; pass opts.fetch");
|
|
650
748
|
}
|
|
@@ -652,25 +750,57 @@ export class PyAI {
|
|
|
652
750
|
authHeaders(extra = {}) {
|
|
653
751
|
return { Authorization: `Bearer ${this.apiKey}`, ...extra };
|
|
654
752
|
}
|
|
655
|
-
async request(path, init
|
|
656
|
-
const
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
753
|
+
async request(path, init) {
|
|
754
|
+
const signal = init.signal ?? this.signal;
|
|
755
|
+
const safe = ["GET", "HEAD"].includes((init.method ?? "GET").toUpperCase());
|
|
756
|
+
const deadline = performance.now() + this.retryWindowMs;
|
|
757
|
+
for (let attempt = 0;; attempt++) {
|
|
758
|
+
signal?.throwIfAborted();
|
|
759
|
+
let res;
|
|
760
|
+
let failure;
|
|
761
|
+
try {
|
|
762
|
+
res = await this.fetchImpl(`${this.baseURL}${path}`, { ...init, signal });
|
|
763
|
+
}
|
|
764
|
+
catch (error) {
|
|
765
|
+
// Preserve the original exception and cause; an arbitrary TypeError may
|
|
766
|
+
// be a programming/CORS/configuration error, not a transient connection.
|
|
767
|
+
if (signal?.aborted || !transientTransport(error))
|
|
768
|
+
throw error;
|
|
769
|
+
failure = error;
|
|
770
|
+
}
|
|
771
|
+
if (res?.ok)
|
|
772
|
+
return res;
|
|
773
|
+
if (!safe || attempt >= this.maxRetries || (res && !RETRYABLE.has(res.status))) {
|
|
774
|
+
if (res)
|
|
775
|
+
throw await this.toError(res, signal);
|
|
776
|
+
throw failure;
|
|
777
|
+
}
|
|
778
|
+
const delay = retryDelay(res?.headers.get("retry-after") ?? null, attempt);
|
|
779
|
+
if (performance.now() + delay >= deadline) {
|
|
780
|
+
if (res)
|
|
781
|
+
throw await this.toError(res, signal);
|
|
782
|
+
throw failure;
|
|
783
|
+
}
|
|
784
|
+
// Dispose the failed response before waiting/reusing its connection.
|
|
785
|
+
if (res?.body)
|
|
786
|
+
await res.body.cancel().catch(() => { });
|
|
787
|
+
await retryPause(delay, signal);
|
|
788
|
+
if (performance.now() >= deadline) {
|
|
789
|
+
if (res)
|
|
790
|
+
throw new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, res.headers.get("x-request-id") ?? undefined);
|
|
791
|
+
throw failure;
|
|
792
|
+
}
|
|
664
793
|
}
|
|
665
|
-
throw await this.toError(res);
|
|
666
794
|
}
|
|
667
|
-
async toError(res) {
|
|
795
|
+
async toError(res, signal) {
|
|
668
796
|
const requestId = res.headers.get("x-request-id") ?? undefined;
|
|
669
797
|
let body;
|
|
670
798
|
try {
|
|
671
799
|
body = await res.json();
|
|
800
|
+
signal?.throwIfAborted();
|
|
672
801
|
}
|
|
673
802
|
catch {
|
|
803
|
+
signal?.throwIfAborted();
|
|
674
804
|
return new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, requestId);
|
|
675
805
|
}
|
|
676
806
|
const b = body;
|
|
@@ -1081,6 +1211,11 @@ export class PyAI {
|
|
|
1081
1211
|
*/
|
|
1082
1212
|
connect: (opts = {}) => {
|
|
1083
1213
|
const query = { ...(opts.query ?? {}) };
|
|
1214
|
+
if (opts.resumeToken !== undefined) {
|
|
1215
|
+
if (!opts.resumeToken.trim())
|
|
1216
|
+
throw new Error("resumeToken must not be empty");
|
|
1217
|
+
query.resume_token = opts.resumeToken;
|
|
1218
|
+
}
|
|
1084
1219
|
if (opts.format)
|
|
1085
1220
|
query.format = opts.format;
|
|
1086
1221
|
if (opts.rate)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pyai/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Official TypeScript/JavaScript SDK for PyAI, speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
package/src/cli-omni-template.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.8.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nCapture ends after the simulated queue drains and two seconds of quiet. Omni\nhas no protocol reply-end marker, so this is a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.8.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/src/index.ts
CHANGED
|
@@ -14,8 +14,12 @@ export interface PyAIOptions {
|
|
|
14
14
|
baseURL?: string;
|
|
15
15
|
/** Injectable for tests / custom transports. Defaults to global fetch. */
|
|
16
16
|
fetch?: typeof fetch;
|
|
17
|
-
/** Retries on
|
|
17
|
+
/** Retries safe reads on transient failures. Writes are never replayed. Default 2. */
|
|
18
18
|
maxRetries?: number;
|
|
19
|
+
/** Retry admission window, in ms. Does not time out an in-flight request/body. Default 10000. */
|
|
20
|
+
retryWindowMs?: number;
|
|
21
|
+
/** Optional caller-owned cancellation/deadline signal for REST requests and backoff. */
|
|
22
|
+
signal?: AbortSignal;
|
|
19
23
|
}
|
|
20
24
|
|
|
21
25
|
/** Stable, machine-readable error. Branch on `code`, not `message`. */
|
|
@@ -215,12 +219,17 @@ export interface HearVocabulary {
|
|
|
215
219
|
updated_at: number | null;
|
|
216
220
|
}
|
|
217
221
|
|
|
222
|
+
/** Canonical Omni session language identifiers accepted by the public API. */
|
|
223
|
+
export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh";
|
|
224
|
+
/** Internal candidate identifiers; the public control plane rejects these until qualified. */
|
|
225
|
+
export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
|
|
226
|
+
|
|
218
227
|
export interface AgentConfig {
|
|
219
228
|
name?: string;
|
|
220
229
|
persona_system_prompt?: string | null;
|
|
221
230
|
greeting?: string | null;
|
|
222
231
|
voice_id?: string | null;
|
|
223
|
-
language?:
|
|
232
|
+
language?: OmniLanguage | null;
|
|
224
233
|
/**
|
|
225
234
|
* Opt-in speech-recognition vocabulary. PyAI sanitizes and keeps at most
|
|
226
235
|
* five terms. An empty list or null turns it off.
|
|
@@ -927,7 +936,7 @@ export interface OmniConfigure {
|
|
|
927
936
|
* proceeds and bills as what was served. Per-language availability is
|
|
928
937
|
* staged, see the Language support reference.
|
|
929
938
|
*/
|
|
930
|
-
language?:
|
|
939
|
+
language?: OmniLanguage;
|
|
931
940
|
/** Function calling: hosted catalog names, client-loop schemas, or names of
|
|
932
941
|
* server tools already registered with POST /v1/tools. An inline `endpoint`
|
|
933
942
|
* is rejected with `unsupported_tool_transport`. */
|
|
@@ -939,6 +948,10 @@ export interface OmniConfigure {
|
|
|
939
948
|
export interface OmniConnectOptions {
|
|
940
949
|
/** Optional opaque per-session tag echoed to your `kb_endpoint`. Omni is zero-state. */
|
|
941
950
|
sessionLabel?: string;
|
|
951
|
+
/** Single-use token from session_started. Failed restoration closes instead of starting fresh. */
|
|
952
|
+
resumeToken?: string;
|
|
953
|
+
/** Maximum wait for resume acknowledgement, including socket opening. Default 5000 ms. */
|
|
954
|
+
resumeTimeoutMs?: number;
|
|
942
955
|
/**
|
|
943
956
|
* Ephemeral browser token from `omni.createSession`. When set it is used as
|
|
944
957
|
* the WS subprotocol (`pyai-key.<token>`) instead of the client's secret key,
|
|
@@ -1010,6 +1023,8 @@ export class OmniConnection {
|
|
|
1010
1023
|
private readonly ws: WebSocketLike;
|
|
1011
1024
|
private readonly opts: OmniConnectOptions;
|
|
1012
1025
|
private closed = false;
|
|
1026
|
+
private awaitingResume = false;
|
|
1027
|
+
private resumeTimer?: ReturnType<typeof setTimeout>;
|
|
1013
1028
|
/** Serializes async Blob reads in {@link sendAudio} so frames stay ordered. */
|
|
1014
1029
|
private blobTail: Promise<void> = Promise.resolve();
|
|
1015
1030
|
/** Serializes inbound Blob decoding so browser frames stay ordered. */
|
|
@@ -1017,6 +1032,11 @@ export class OmniConnection {
|
|
|
1017
1032
|
|
|
1018
1033
|
constructor(url: string, subprotocol: string, opts: OmniConnectOptions) {
|
|
1019
1034
|
this.opts = opts;
|
|
1035
|
+
this.awaitingResume = new URL(url).searchParams.has("resume_token");
|
|
1036
|
+
const resumeTimeout = opts.resumeTimeoutMs ?? 5000;
|
|
1037
|
+
if (!Number.isFinite(resumeTimeout) || resumeTimeout <= 0 || resumeTimeout > 30_000) {
|
|
1038
|
+
throw new Error("resumeTimeoutMs must be greater than zero and at most 30000");
|
|
1039
|
+
}
|
|
1020
1040
|
const WS = opts.webSocket ?? (globalThis as { WebSocket?: WebSocketCtor }).WebSocket;
|
|
1021
1041
|
if (!WS) {
|
|
1022
1042
|
throw new Error(
|
|
@@ -1024,8 +1044,11 @@ export class OmniConnection {
|
|
|
1024
1044
|
);
|
|
1025
1045
|
}
|
|
1026
1046
|
this.ws = new WS(url, [REALTIME_SUBPROTOCOL_MARKER, subprotocol]);
|
|
1047
|
+
if (this.awaitingResume) {
|
|
1048
|
+
this.resumeTimer = setTimeout(() => this.rejectResume("Resume acknowledgement timed out"), resumeTimeout);
|
|
1049
|
+
}
|
|
1027
1050
|
this.ws.onopen = () => {
|
|
1028
|
-
if (opts.configure) {
|
|
1051
|
+
if (opts.configure && !this.awaitingResume) {
|
|
1029
1052
|
try {
|
|
1030
1053
|
this.configure(opts.configure);
|
|
1031
1054
|
} catch {
|
|
@@ -1046,12 +1069,25 @@ export class OmniConnection {
|
|
|
1046
1069
|
};
|
|
1047
1070
|
this.ws.onerror = (ev) => opts.onError?.(ev instanceof Error ? ev : new Error("WebSocket error"));
|
|
1048
1071
|
this.ws.onclose = (ev) => {
|
|
1072
|
+
clearTimeout(this.resumeTimer);
|
|
1049
1073
|
this.closed = true;
|
|
1050
1074
|
opts.onClose?.(ev.code, ev.reason);
|
|
1051
1075
|
};
|
|
1052
1076
|
}
|
|
1053
1077
|
|
|
1078
|
+
private rejectResume(message: string): void {
|
|
1079
|
+
if (this.closed) return;
|
|
1080
|
+
this.close(4000, "resume_failed");
|
|
1081
|
+
this.opts.onError?.(new PyAIError(0, message, "resume_failed"));
|
|
1082
|
+
}
|
|
1083
|
+
|
|
1084
|
+
private assertSendable(): void {
|
|
1085
|
+
if (this.closed) throw new Error("Omni connection is closed");
|
|
1086
|
+
if (this.awaitingResume) throw new Error("Wait for a confirmed session resume before sending");
|
|
1087
|
+
}
|
|
1088
|
+
|
|
1054
1089
|
private async handleMessage(data: unknown): Promise<void> {
|
|
1090
|
+
if (this.closed) return;
|
|
1055
1091
|
// Server → client binary frames are TYPE-TAGGED by their first byte:
|
|
1056
1092
|
// 0x01 = agent audio (PCM16) · 0x02 = transcript UTF-8 · 0x03 = control JSON.
|
|
1057
1093
|
// (Treating every binary frame as audio, the old behavior, plays the
|
|
@@ -1064,7 +1100,9 @@ export class OmniConnection {
|
|
|
1064
1100
|
this.opts.onError?.(new Error("Unsupported Omni binary frame"));
|
|
1065
1101
|
return;
|
|
1066
1102
|
}
|
|
1103
|
+
if (this.closed) return;
|
|
1067
1104
|
const tag = bytes[0];
|
|
1105
|
+
if (this.awaitingResume && tag !== 0x03) return;
|
|
1068
1106
|
if (tag === 0x01) {
|
|
1069
1107
|
this.opts.onAudio?.(bytes.slice(1) as OmniAudioChunk); // copy → aligned PCM16
|
|
1070
1108
|
return;
|
|
@@ -1078,6 +1116,7 @@ export class OmniConnection {
|
|
|
1078
1116
|
if (tag === 0x03) {
|
|
1079
1117
|
try {
|
|
1080
1118
|
const parsed = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(1))) as OmniServerFrame;
|
|
1119
|
+
if (this.awaitingResume && parsed?.event !== OmniEvent.Hello && parsed?.event !== OmniEvent.SessionStarted && parsed?.event !== OmniEvent.Error) return;
|
|
1081
1120
|
if (parsed?.event === OmniEvent.Transcript) {
|
|
1082
1121
|
// The deployed engine also sends this bounded synthesis-input
|
|
1083
1122
|
// advisory. It is not caller ASR or proof that speech was played.
|
|
@@ -1112,6 +1151,20 @@ export class OmniConnection {
|
|
|
1112
1151
|
this.opts.onError?.(new Error("Omni server control frame is missing its event key"));
|
|
1113
1152
|
return;
|
|
1114
1153
|
}
|
|
1154
|
+
if (this.closed) return;
|
|
1155
|
+
if (this.awaitingResume) {
|
|
1156
|
+
if (eventName === OmniEvent.SessionStarted) {
|
|
1157
|
+
if (frame.resumed !== true) {
|
|
1158
|
+
this.rejectResume("The server could not restore the previous conversation");
|
|
1159
|
+
return;
|
|
1160
|
+
}
|
|
1161
|
+
clearTimeout(this.resumeTimer);
|
|
1162
|
+
this.awaitingResume = false;
|
|
1163
|
+
if (this.opts.configure) this.configure(this.opts.configure);
|
|
1164
|
+
} else if (eventName !== OmniEvent.Hello && eventName !== OmniEvent.Error) {
|
|
1165
|
+
return;
|
|
1166
|
+
}
|
|
1167
|
+
}
|
|
1115
1168
|
this.opts.onEvent?.(frame);
|
|
1116
1169
|
switch (eventName) {
|
|
1117
1170
|
case OmniEvent.Hello:
|
|
@@ -1154,6 +1207,7 @@ export class OmniConnection {
|
|
|
1154
1207
|
* `{"event":"configure"}` is acked but silently dropped by the engine.)
|
|
1155
1208
|
*/
|
|
1156
1209
|
configure(cfg: OmniConfigure): void {
|
|
1210
|
+
this.assertSendable();
|
|
1157
1211
|
const payload: Record<string, unknown> = { ...cfg };
|
|
1158
1212
|
delete payload.type;
|
|
1159
1213
|
delete payload.event;
|
|
@@ -1166,6 +1220,7 @@ export class OmniConnection {
|
|
|
1166
1220
|
* default branch, so an untagged chunk is dropped with no error.
|
|
1167
1221
|
*/
|
|
1168
1222
|
sendAudio(chunk: ArrayBufferLike | ArrayBufferView | Blob): void {
|
|
1223
|
+
this.assertSendable();
|
|
1169
1224
|
const bytes = omniToBytes(chunk);
|
|
1170
1225
|
if (bytes) {
|
|
1171
1226
|
this.ws.send(omniAudioFrame(bytes));
|
|
@@ -1176,6 +1231,7 @@ export class OmniConnection {
|
|
|
1176
1231
|
this.blobTail = this.blobTail
|
|
1177
1232
|
.then(async () => {
|
|
1178
1233
|
const buf = await (chunk as Blob).arrayBuffer();
|
|
1234
|
+
this.assertSendable();
|
|
1179
1235
|
this.ws.send(omniAudioFrame(new Uint8Array(buf)));
|
|
1180
1236
|
})
|
|
1181
1237
|
.catch((err) => this.opts.onError?.(err instanceof Error ? err : new Error(String(err))));
|
|
@@ -1183,11 +1239,13 @@ export class OmniConnection {
|
|
|
1183
1239
|
|
|
1184
1240
|
/** Send DTMF digits as a `{"type":"dtmf"}` control frame. */
|
|
1185
1241
|
sendDtmf(digits: string): void {
|
|
1242
|
+
this.assertSendable();
|
|
1186
1243
|
this.ws.send(omniControlFrame({ type: "dtmf", digits }));
|
|
1187
1244
|
}
|
|
1188
1245
|
|
|
1189
1246
|
/** Reply to a client-loop {@link OmniEvent.ToolCall}. */
|
|
1190
1247
|
toolResult(callId: string, payload: { result?: unknown; error?: string }): void {
|
|
1248
|
+
this.assertSendable();
|
|
1191
1249
|
this.ws.send(omniControlFrame({ type: "tool_result", call_id: callId, ...payload }));
|
|
1192
1250
|
}
|
|
1193
1251
|
|
|
@@ -1197,6 +1255,7 @@ export class OmniConnection {
|
|
|
1197
1255
|
* never `event`.
|
|
1198
1256
|
*/
|
|
1199
1257
|
send(frame: Record<string, unknown>): void {
|
|
1258
|
+
this.assertSendable();
|
|
1200
1259
|
if (typeof frame.type !== "string" || !frame.type) {
|
|
1201
1260
|
throw new TypeError("Omni client control frames must have a non-empty type key");
|
|
1202
1261
|
}
|
|
@@ -1207,7 +1266,11 @@ export class OmniConnection {
|
|
|
1207
1266
|
|
|
1208
1267
|
/** Close the session. */
|
|
1209
1268
|
close(code: number = WSCloseCode.Normal, reason = ""): void {
|
|
1210
|
-
|
|
1269
|
+
clearTimeout(this.resumeTimer);
|
|
1270
|
+
if (!this.closed) {
|
|
1271
|
+
this.closed = true;
|
|
1272
|
+
this.ws.close(code, reason);
|
|
1273
|
+
}
|
|
1211
1274
|
}
|
|
1212
1275
|
|
|
1213
1276
|
/** The underlying socket (escape hatch for advanced use). */
|
|
@@ -1660,11 +1723,43 @@ export interface AmdStreamOptions {
|
|
|
1660
1723
|
|
|
1661
1724
|
const RETRYABLE = new Set([429, 500, 502, 503, 504]);
|
|
1662
1725
|
|
|
1726
|
+
// Only transport codes known to be transient; do not retry TLS, URL or DNS
|
|
1727
|
+
// configuration errors. Browsers often hide the cause: those propagate intact.
|
|
1728
|
+
const TRANSIENT_TRANSPORT = new Set(["ECONNRESET", "ECONNREFUSED", "EPIPE", "ETIMEDOUT", "EAI_AGAIN", "UND_ERR_SOCKET", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_HEADERS_TIMEOUT", "UND_ERR_BODY_TIMEOUT"]);
|
|
1729
|
+
function transientTransport(error: unknown): boolean {
|
|
1730
|
+
const seen = new Set<unknown>();
|
|
1731
|
+
while (error && typeof error === "object" && !seen.has(error)) {
|
|
1732
|
+
seen.add(error);
|
|
1733
|
+
const value = error as { name?: string; code?: string; cause?: unknown };
|
|
1734
|
+
if (value.name === "AbortError" || value.name === "TimeoutError") return false;
|
|
1735
|
+
if (value.code && TRANSIENT_TRANSPORT.has(value.code)) return true;
|
|
1736
|
+
error = value.cause;
|
|
1737
|
+
}
|
|
1738
|
+
return false;
|
|
1739
|
+
}
|
|
1740
|
+
function retryDelay(header: string | null, attempt: number): number {
|
|
1741
|
+
const jitter = Math.random() * Math.min(2 ** attempt * 250, 2000);
|
|
1742
|
+
if (header === null) return jitter;
|
|
1743
|
+
const seconds = /^\d+(?:\.\d+)?$/.test(header.trim()) ? Number(header) : NaN;
|
|
1744
|
+
const delay = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now();
|
|
1745
|
+
return Number.isFinite(delay) ? Math.max(0, delay) + jitter : jitter;
|
|
1746
|
+
}
|
|
1747
|
+
function retryPause(ms: number, signal?: AbortSignal | null): Promise<void> {
|
|
1748
|
+
return new Promise((resolve, reject) => {
|
|
1749
|
+
const cancel = () => { clearTimeout(timer); signal?.removeEventListener("abort", cancel); reject(signal?.reason); };
|
|
1750
|
+
const timer = setTimeout(() => { signal?.removeEventListener("abort", cancel); resolve(); }, ms);
|
|
1751
|
+
signal?.addEventListener("abort", cancel, { once: true });
|
|
1752
|
+
if (signal?.aborted) cancel();
|
|
1753
|
+
});
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1663
1756
|
export class PyAI {
|
|
1664
1757
|
private readonly apiKey: string;
|
|
1665
1758
|
private readonly baseURL: string;
|
|
1666
1759
|
private readonly fetchImpl: typeof fetch;
|
|
1667
1760
|
private readonly maxRetries: number;
|
|
1761
|
+
private readonly retryWindowMs: number;
|
|
1762
|
+
private readonly signal?: AbortSignal;
|
|
1668
1763
|
|
|
1669
1764
|
constructor(opts: PyAIOptions) {
|
|
1670
1765
|
if (!opts.apiKey) throw new Error("apiKey is required");
|
|
@@ -1672,6 +1767,10 @@ export class PyAI {
|
|
|
1672
1767
|
this.baseURL = (opts.baseURL ?? "https://api.pyai.com").replace(/\/$/, "");
|
|
1673
1768
|
this.fetchImpl = opts.fetch ?? globalThis.fetch;
|
|
1674
1769
|
this.maxRetries = opts.maxRetries ?? 2;
|
|
1770
|
+
this.retryWindowMs = opts.retryWindowMs ?? 10_000;
|
|
1771
|
+
this.signal = opts.signal;
|
|
1772
|
+
if (!Number.isSafeInteger(this.maxRetries) || this.maxRetries < 0) throw new Error("maxRetries must be a non-negative integer");
|
|
1773
|
+
if (!Number.isFinite(this.retryWindowMs) || this.retryWindowMs < 0) throw new Error("retryWindowMs must be finite and non-negative");
|
|
1675
1774
|
if (!this.fetchImpl) throw new Error("No fetch implementation available; pass opts.fetch");
|
|
1676
1775
|
}
|
|
1677
1776
|
|
|
@@ -1681,24 +1780,50 @@ export class PyAI {
|
|
|
1681
1780
|
return { Authorization: `Bearer ${this.apiKey}`, ...extra };
|
|
1682
1781
|
}
|
|
1683
1782
|
|
|
1684
|
-
private async request(path: string, init: RequestInit
|
|
1685
|
-
const
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1783
|
+
private async request(path: string, init: RequestInit): Promise<Response> {
|
|
1784
|
+
const signal = init.signal ?? this.signal;
|
|
1785
|
+
const safe = ["GET", "HEAD"].includes((init.method ?? "GET").toUpperCase());
|
|
1786
|
+
const deadline = performance.now() + this.retryWindowMs;
|
|
1787
|
+
for (let attempt = 0; ; attempt++) {
|
|
1788
|
+
signal?.throwIfAborted();
|
|
1789
|
+
let res: Response | undefined;
|
|
1790
|
+
let failure: unknown;
|
|
1791
|
+
try {
|
|
1792
|
+
res = await this.fetchImpl(`${this.baseURL}${path}`, { ...init, signal });
|
|
1793
|
+
} catch (error) {
|
|
1794
|
+
// Preserve the original exception and cause; an arbitrary TypeError may
|
|
1795
|
+
// be a programming/CORS/configuration error, not a transient connection.
|
|
1796
|
+
if (signal?.aborted || !transientTransport(error)) throw error;
|
|
1797
|
+
failure = error;
|
|
1798
|
+
}
|
|
1799
|
+
if (res?.ok) return res;
|
|
1800
|
+
if (!safe || attempt >= this.maxRetries || (res && !RETRYABLE.has(res.status))) {
|
|
1801
|
+
if (res) throw await this.toError(res, signal);
|
|
1802
|
+
throw failure;
|
|
1803
|
+
}
|
|
1804
|
+
const delay = retryDelay(res?.headers.get("retry-after") ?? null, attempt);
|
|
1805
|
+
if (performance.now() + delay >= deadline) {
|
|
1806
|
+
if (res) throw await this.toError(res, signal);
|
|
1807
|
+
throw failure;
|
|
1808
|
+
}
|
|
1809
|
+
// Dispose the failed response before waiting/reusing its connection.
|
|
1810
|
+
if (res?.body) await res.body.cancel().catch(() => {});
|
|
1811
|
+
await retryPause(delay, signal);
|
|
1812
|
+
if (performance.now() >= deadline) {
|
|
1813
|
+
if (res) throw new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, res.headers.get("x-request-id") ?? undefined);
|
|
1814
|
+
throw failure;
|
|
1815
|
+
}
|
|
1692
1816
|
}
|
|
1693
|
-
throw await this.toError(res);
|
|
1694
1817
|
}
|
|
1695
1818
|
|
|
1696
|
-
private async toError(res: Response): Promise<PyAIError> {
|
|
1819
|
+
private async toError(res: Response, signal?: AbortSignal | null): Promise<PyAIError> {
|
|
1697
1820
|
const requestId = res.headers.get("x-request-id") ?? undefined;
|
|
1698
1821
|
let body: unknown;
|
|
1699
1822
|
try {
|
|
1700
1823
|
body = await res.json();
|
|
1824
|
+
signal?.throwIfAborted();
|
|
1701
1825
|
} catch {
|
|
1826
|
+
signal?.throwIfAborted();
|
|
1702
1827
|
return new PyAIError(res.status, `HTTP ${res.status}`, undefined, undefined, requestId);
|
|
1703
1828
|
}
|
|
1704
1829
|
const b = body as {
|
|
@@ -2177,6 +2302,10 @@ export class PyAI {
|
|
|
2177
2302
|
*/
|
|
2178
2303
|
connect: (opts: OmniConnectOptions = {}): OmniConnection => {
|
|
2179
2304
|
const query: Record<string, string> = { ...(opts.query ?? {}) };
|
|
2305
|
+
if (opts.resumeToken !== undefined) {
|
|
2306
|
+
if (!opts.resumeToken.trim()) throw new Error("resumeToken must not be empty");
|
|
2307
|
+
query.resume_token = opts.resumeToken;
|
|
2308
|
+
}
|
|
2180
2309
|
if (opts.format) query.format = opts.format;
|
|
2181
2310
|
if (opts.rate) query.rate = String(opts.rate);
|
|
2182
2311
|
const url = this.realtimeURL({ sessionLabel: opts.sessionLabel, query });
|