@pyai/sdk 0.9.3 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CLI.schema.json +11 -0
  2. package/README.md +106 -3
  3. package/dist/cli-omni-template.d.ts +2 -2
  4. package/dist/cli-omni-template.js +2 -2
  5. package/dist/cli.js +39 -4
  6. package/dist/dialer/appServer.d.ts +21 -0
  7. package/dist/dialer/appServer.js +195 -0
  8. package/dist/dialer/dialerElement.d.ts +23 -0
  9. package/dist/dialer/dialerElement.js +108 -0
  10. package/dist/dialer/dialingQueue.d.ts +31 -0
  11. package/dist/dialer/dialingQueue.js +78 -0
  12. package/dist/dialer/index.d.ts +6 -0
  13. package/dist/dialer/index.js +6 -0
  14. package/dist/dialer/indexedDbJournal.d.ts +11 -0
  15. package/dist/dialer/indexedDbJournal.js +62 -0
  16. package/dist/dialer/managedDialer.d.ts +65 -0
  17. package/dist/dialer/managedDialer.js +217 -0
  18. package/dist/dialer/meetingRecorder.d.ts +83 -0
  19. package/dist/dialer/meetingRecorder.js +261 -0
  20. package/dist/dialer/meetingRecorderElement.d.ts +20 -0
  21. package/dist/dialer/meetingRecorderElement.js +138 -0
  22. package/dist/dialer/meetingRecordingStore.d.ts +17 -0
  23. package/dist/dialer/meetingRecordingStore.js +129 -0
  24. package/dist/dialer/serverBroker.d.ts +3 -0
  25. package/dist/dialer/serverBroker.js +23 -0
  26. package/dist/dialer/voiceSdkAudio.d.ts +33 -0
  27. package/dist/dialer/voiceSdkAudio.js +66 -0
  28. package/dist/index.d.ts +122 -6
  29. package/dist/index.js +36 -3
  30. package/package.json +24 -1
  31. package/src/cli-omni-template.ts +2 -2
  32. package/src/cli.ts +25 -4
  33. package/src/dialer/app/app.css +1 -0
  34. package/src/dialer/app/app.html +7 -0
  35. package/src/dialer/app/app.js +88 -0
  36. package/src/dialer/app/icon.svg +1 -0
  37. package/src/dialer/app/manifest.webmanifest +1 -0
  38. package/src/dialer/appServer.ts +151 -0
  39. package/src/dialer/dialerElement.ts +80 -0
  40. package/src/dialer/dialingQueue.ts +65 -0
  41. package/src/dialer/index.ts +6 -0
  42. package/src/dialer/indexedDbJournal.ts +56 -0
  43. package/src/dialer/managedDialer.ts +187 -0
  44. package/src/dialer/meetingRecorder.ts +220 -0
  45. package/src/dialer/meetingRecorderElement.ts +99 -0
  46. package/src/dialer/meetingRecordingStore.ts +99 -0
  47. package/src/dialer/serverBroker.ts +23 -0
  48. package/src/dialer/voiceSdkAudio.ts +61 -0
  49. package/src/index.ts +118 -5
@@ -0,0 +1,23 @@
1
+ /** Server-only broker. Never import this module into a browser bundle. */
2
+ import { createHash } from "node:crypto";
3
+ export function createServerDialerBroker(sdk, authenticatedUserId) {
4
+ if (!authenticatedUserId.trim())
5
+ throw new Error("An authenticated user is required.");
6
+ const ownedCall = async (id) => {
7
+ const call = await sdk.telephony.calls.get(id);
8
+ if (call.call_type !== "human" || call.user_id !== authenticatedUserId)
9
+ throw new Error("No such call.");
10
+ return call;
11
+ };
12
+ return {
13
+ async create(input, key) {
14
+ if (!key || key.length > 255 || /[\r\n]/.test(key))
15
+ throw new Error("A valid Idempotency-Key is required.");
16
+ // Separate users cannot collide on an organization-wide caller-chosen key.
17
+ const scopedKey = "dialer:" + createHash("sha256").update(authenticatedUserId).update("\0").update(key).digest("hex");
18
+ return sdk.telephony.browserSessions.create({ ...input, user_id: authenticatedUserId }, { idempotencyKey: scopedKey });
19
+ },
20
+ get: ownedCall,
21
+ async hangup(id) { await ownedCall(id); return sdk.telephony.calls.hangup(id); },
22
+ };
23
+ }
@@ -0,0 +1,33 @@
1
+ import type { TelephonyBrowserSession } from "../index.ts";
2
+ import type { DialerAudio } from "./managedDialer.ts";
3
+ interface VoiceCall {
4
+ on(event: string, callback: (...args: unknown[]) => void): void;
5
+ removeAllListeners(): void;
6
+ disconnect(): void;
7
+ mute(value: boolean): void;
8
+ sendDigits(digits: string): void;
9
+ status(): string;
10
+ }
11
+ interface VoiceDevice {
12
+ connect(options: {
13
+ params: Record<string, string>;
14
+ }): Promise<VoiceCall>;
15
+ on(event: string, callback: (...args: unknown[]) => void): void;
16
+ removeAllListeners(): void;
17
+ destroy(): void;
18
+ }
19
+ /** Inject the supported Voice SDK's Device constructor. Lazy-load it only when
20
+ * calling starts; the dialer shell itself has no framework or media dependency. */
21
+ export declare class VoiceSdkAudio implements DialerAudio {
22
+ private device;
23
+ private call;
24
+ private generation;
25
+ private notify;
26
+ private readonly createDevice;
27
+ constructor(createDevice: (token: string) => VoiceDevice);
28
+ connect(session: TelephonyBrowserSession, event: (state: "ringing" | "active" | "disconnected") => void): Promise<void>;
29
+ disconnect(): void;
30
+ mute(value: boolean): void;
31
+ sendDigits(digits: string): void;
32
+ }
33
+ export {};
@@ -0,0 +1,66 @@
1
+ /** Inject the supported Voice SDK's Device constructor. Lazy-load it only when
2
+ * calling starts; the dialer shell itself has no framework or media dependency. */
3
+ export class VoiceSdkAudio {
4
+ device = null;
5
+ call = null;
6
+ generation = 0;
7
+ notify = null;
8
+ createDevice;
9
+ constructor(createDevice) { this.createDevice = createDevice; }
10
+ async connect(session, event) {
11
+ if (!session.access_token)
12
+ throw new Error("No unconsumed calling capability.");
13
+ this.disconnect();
14
+ const generation = this.generation;
15
+ const device = this.createDevice(session.access_token);
16
+ this.device = device;
17
+ this.notify = event;
18
+ const emit = (state) => { if (generation === this.generation)
19
+ this.notify?.(state); };
20
+ device.on("error", () => emit("disconnected"));
21
+ try {
22
+ const call = await device.connect({ params: { ...session.connect_params } });
23
+ if (generation !== this.generation) {
24
+ try {
25
+ call.removeAllListeners();
26
+ }
27
+ catch { /* Continue releasing stale audio. */ }
28
+ try {
29
+ call.disconnect();
30
+ }
31
+ catch { /* The owning device is already destroyed. */ }
32
+ return;
33
+ }
34
+ this.call = call;
35
+ call.on("ringing", () => emit("ringing"));
36
+ call.on("accept", () => emit("active"));
37
+ for (const name of ["disconnect", "cancel", "reject", "error"])
38
+ call.on(name, () => emit("disconnected"));
39
+ emit(call.status() === "open" ? "active" : "ringing");
40
+ }
41
+ catch (error) {
42
+ if (generation === this.generation)
43
+ this.disconnect();
44
+ throw error;
45
+ }
46
+ }
47
+ disconnect() {
48
+ this.generation++;
49
+ this.notify = null;
50
+ const call = this.call, device = this.device;
51
+ this.call = null;
52
+ this.device = null;
53
+ // Local SDK failures must not prevent the controller's durable remote stop.
54
+ for (const close of [() => call?.removeAllListeners(), () => call?.disconnect(),
55
+ () => device?.removeAllListeners(), () => device?.destroy()]) {
56
+ try {
57
+ close();
58
+ }
59
+ catch { /* All owned resources still get a cleanup attempt. */ }
60
+ }
61
+ }
62
+ mute(value) { if (!this.call)
63
+ throw new Error("No active audio."); this.call.mute(value); }
64
+ sendDigits(digits) { if (!this.call)
65
+ throw new Error("No active audio."); this.call.sendDigits(digits); }
66
+ }
package/dist/index.d.ts CHANGED
@@ -199,6 +199,7 @@ export type OmniLanguage = "en" | "fr" | "es" | "de" | "hi" | "zh" | "it" | "pt"
199
199
  export type OmniStagedLanguage = "ht" | "zh-CN" | "ar-MSA" | "ar-Gulf";
200
200
  export interface AgentConfig {
201
201
  name?: string;
202
+ /** Saved instructions: 8,000 Unicode characters by default; 32,000 on commitment plans. */
202
203
  persona_system_prompt?: string | null;
203
204
  /** False restricts retrieval to explicit agent bindings, including none. Default true. */
204
205
  inherit_org_knowledgebases?: boolean;
@@ -215,6 +216,7 @@ export interface AgentConfig {
215
216
  export interface Agent {
216
217
  object: "agent";
217
218
  agent_id: string;
219
+ prompt_limit_characters?: 8000 | 32000;
218
220
  name: string;
219
221
  vocabulary: string[];
220
222
  inherit_org_knowledgebases?: boolean;
@@ -827,6 +829,63 @@ export interface TelephonyCapabilities {
827
829
  voice?: boolean;
828
830
  sms?: boolean;
829
831
  }
832
+ export type TelephonyCallStatus = "queued" | "dispatching" | "ringing" | "in_progress" | "completed" | "failed" | "canceled" | "dispatch_unknown";
833
+ export interface TelephonyCall {
834
+ object: "telephony.call";
835
+ id: string;
836
+ project_id: string;
837
+ direction: "inbound" | "outbound";
838
+ status: TelephonyCallStatus;
839
+ version: number;
840
+ from: string;
841
+ to: string;
842
+ from_number_id: string | null;
843
+ agent_id: string | null;
844
+ call_type?: "human";
845
+ user_id?: string;
846
+ client_reference: string | null;
847
+ stop_requested: boolean;
848
+ created_at: number;
849
+ updated_at: number;
850
+ answered_at: number | null;
851
+ ended_at: number | null;
852
+ [key: string]: unknown;
853
+ }
854
+ export type TelephonyUseCase = "transactional" | "support" | "sales" | "collections" | "survey";
855
+ export interface TelephonyCallCreate {
856
+ from_number_id: string;
857
+ to: string;
858
+ agent_id: string;
859
+ use_case: TelephonyUseCase;
860
+ routing_profile?: "quality_first" | "balanced" | "cost_optimized";
861
+ client_reference?: string;
862
+ agent_release_id?: string;
863
+ max_duration_seconds?: number;
864
+ stop_requested?: boolean;
865
+ }
866
+ export interface TelephonyBrowserSessionCreate {
867
+ from_number_id: string;
868
+ to: string;
869
+ /** Resolve from your authenticated server session, never from untrusted form data. */
870
+ user_id: string;
871
+ client_reference: string;
872
+ recording: boolean;
873
+ use_case?: TelephonyUseCase;
874
+ max_duration_seconds?: number;
875
+ }
876
+ export interface TelephonyBrowserSession {
877
+ object: "telephony.browser_session";
878
+ id: string;
879
+ call_id: string;
880
+ transport: "voice-sdk";
881
+ /** Single-call capability. Keep in memory; never log or persist this token. */
882
+ access_token: string | null;
883
+ expires_at: number;
884
+ connect_params: {
885
+ session_id: string;
886
+ };
887
+ call: TelephonyCall;
888
+ }
830
889
  export interface TelephonyAvailableNumber {
831
890
  object?: "telephony.available_number";
832
891
  /** E.164. */
@@ -854,6 +913,26 @@ export interface TelephonyNumber {
854
913
  created_at?: number;
855
914
  released_at?: number | null;
856
915
  }
916
+ export interface TelephonyNumberOrder {
917
+ object: "telephony.number_order";
918
+ id: string;
919
+ status: "pending" | "unknown" | "completed" | "failed" | "released";
920
+ phone_number: string;
921
+ country: "AU";
922
+ project_id: string;
923
+ retry_after_seconds: number;
924
+ message: string;
925
+ failure_code?: string;
926
+ number?: TelephonyNumber;
927
+ }
928
+ export interface TelephonyNumberPurchase {
929
+ phone_number: string;
930
+ agent_id?: string | null;
931
+ provisioning_mode?: "automated" | "adopt_preowned";
932
+ project_id?: string;
933
+ compliance_case_id?: string;
934
+ max_monthly_cost_cents?: number;
935
+ }
857
936
  export type TraceVerdict = "PASS" | "WARN" | "FAIL";
858
937
  export type TraceSeverity = "low" | "medium" | "high" | "critical";
859
938
  export interface TraceInteraction {
@@ -1380,9 +1459,42 @@ export declare class PyAI {
1380
1459
  delete: (id: string) => Promise<void>;
1381
1460
  };
1382
1461
  telephony: {
1462
+ /** Server-side methods. Keep the PyAI key in your backend. */
1463
+ calls: {
1464
+ create: (input: TelephonyCallCreate, options: {
1465
+ idempotencyKey: string;
1466
+ }) => Promise<TelephonyCall>;
1467
+ get: (id: string) => Promise<TelephonyCall>;
1468
+ list: (input?: {
1469
+ limit?: number;
1470
+ cursor?: string;
1471
+ direction?: "inbound" | "outbound";
1472
+ status?: TelephonyCallStatus;
1473
+ }) => Promise<ListResponse<TelephonyCall> & {
1474
+ next_cursor: string | null;
1475
+ }>;
1476
+ /** Durable stop intent; use get() until a terminal status is confirmed. */
1477
+ hangup: (id: string) => Promise<TelephonyCall>;
1478
+ decideAdmission: (id: string, input: {
1479
+ decision: "accept" | "reject";
1480
+ client_reference: string;
1481
+ max_duration_seconds: number;
1482
+ }) => Promise<TelephonyCall>;
1483
+ recording: (id: string, options?: {
1484
+ range?: string;
1485
+ }) => Promise<Response>;
1486
+ deleteRecording: (id: string) => Promise<void>;
1487
+ };
1488
+ browserSessions: {
1489
+ /** Persist input and key on your server; retries recover the original session. */
1490
+ create: (input: TelephonyBrowserSessionCreate, options: {
1491
+ idempotencyKey: string;
1492
+ }) => Promise<TelephonyBrowserSession>;
1493
+ };
1383
1494
  numbers: {
1384
- /** Search the carrier's available US local numbers. */
1495
+ /** Search managed inventory; country availability and ordering may be gated. */
1385
1496
  available: (params?: {
1497
+ country?: "US" | "CA" | "IN" | "AU";
1386
1498
  areaCode?: string;
1387
1499
  contains?: string;
1388
1500
  limit?: number;
@@ -1391,16 +1503,20 @@ export declare class PyAI {
1391
1503
  list: (params?: {
1392
1504
  includeReleased?: boolean;
1393
1505
  }) => Promise<ListResponse<TelephonyNumber>>;
1394
- /** Provision (buy) a specific available number, optionally bound to an agent. */
1395
- buy: (params: {
1396
- phone_number: string;
1397
- agent_id?: string | null;
1398
- }) => Promise<TelephonyNumber>;
1506
+ /** Persist selection and key first. Pending/unknown orders must be recovered, never repurchased. */
1507
+ buy: (params: TelephonyNumberPurchase, options: {
1508
+ idempotencyKey: string;
1509
+ }) => Promise<TelephonyNumber | TelephonyNumberOrder>;
1399
1510
  /** Route a number to an agent (`agentId: null` to unassign). */
1400
1511
  assign: (id: string, agentId: string | null) => Promise<TelephonyNumber>;
1401
1512
  /** Release a number back to the carrier (idempotent). */
1402
1513
  release: (id: string) => Promise<TelephonyNumber>;
1403
1514
  };
1515
+ numberOrders: {
1516
+ get: (id: string) => Promise<TelephonyNumberOrder>;
1517
+ /** Read/reconcile the original order; this never purchases another number. */
1518
+ reconcile: (id: string) => Promise<TelephonyNumberOrder>;
1519
+ };
1404
1520
  };
1405
1521
  /** Customer-owned two-human calls. Server keys only; history is project scoped. */
1406
1522
  translation: {
package/dist/index.js CHANGED
@@ -830,6 +830,11 @@ export class OmniConnection {
830
830
  return this.ws.readyState;
831
831
  }
832
832
  }
833
+ function telephonyIdempotencyHeader(key) {
834
+ if (!key || key.length > 255 || /[\r\n]/.test(key))
835
+ throw new Error("A valid Idempotency-Key is required; persist it before creating a call.");
836
+ return { "Idempotency-Key": key };
837
+ }
833
838
  const RETRYABLE = new Set([429, 500, 502, 503, 504]);
834
839
  // Only transport codes known to be transient; do not retry TLS, URL or DNS
835
840
  // configuration errors. Browsers often hide the cause: those propagate intact.
@@ -1156,10 +1161,33 @@ export class PyAI {
1156
1161
  };
1157
1162
  // --- telephony (managed numbers) ---------------------------------------
1158
1163
  telephony = {
1164
+ /** Server-side methods. Keep the PyAI key in your backend. */
1165
+ calls: {
1166
+ create: (input, options) => this.postJson("/v1/telephony/calls", input, telephonyIdempotencyHeader(options.idempotencyKey)),
1167
+ get: (id) => this.getJson(`/v1/telephony/calls/${encodeURIComponent(id)}`),
1168
+ list: (input = {}) => {
1169
+ const query = new URLSearchParams();
1170
+ for (const [key, value] of Object.entries(input))
1171
+ if (value !== undefined)
1172
+ query.set(key, String(value));
1173
+ return this.getJson(`/v1/telephony/calls${query.size ? `?${query}` : ""}`);
1174
+ },
1175
+ /** Durable stop intent; use get() until a terminal status is confirmed. */
1176
+ hangup: (id) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/hangup`, {}),
1177
+ decideAdmission: (id, input) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/admission`, input),
1178
+ recording: (id, options = {}) => this.request(`/v1/telephony/calls/${encodeURIComponent(id)}/recording`, { headers: this.authHeaders(options.range ? { Range: options.range } : {}) }),
1179
+ deleteRecording: async (id) => { await this.deleteReq(`/v1/telephony/calls/${encodeURIComponent(id)}/recording`); },
1180
+ },
1181
+ browserSessions: {
1182
+ /** Persist input and key on your server; retries recover the original session. */
1183
+ create: (input, options) => this.postJson("/v1/telephony/browser-sessions", input, telephonyIdempotencyHeader(options.idempotencyKey)),
1184
+ },
1159
1185
  numbers: {
1160
- /** Search the carrier's available US local numbers. */
1186
+ /** Search managed inventory; country availability and ordering may be gated. */
1161
1187
  available: (params = {}) => {
1162
1188
  const q = new URLSearchParams();
1189
+ if (params.country)
1190
+ q.set("country", params.country);
1163
1191
  if (params.areaCode)
1164
1192
  q.set("area_code", params.areaCode);
1165
1193
  if (params.contains)
@@ -1174,8 +1202,8 @@ export class PyAI {
1174
1202
  const qs = params.includeReleased ? "?include_released=true" : "";
1175
1203
  return this.getJson(`/v1/telephony/numbers${qs}`);
1176
1204
  },
1177
- /** Provision (buy) a specific available number, optionally bound to an agent. */
1178
- buy: (params) => this.postJson("/v1/telephony/numbers", params),
1205
+ /** Persist selection and key first. Pending/unknown orders must be recovered, never repurchased. */
1206
+ buy: (params, options) => this.postJson("/v1/telephony/numbers", params, telephonyIdempotencyHeader(options.idempotencyKey)),
1179
1207
  /** Route a number to an agent (`agentId: null` to unassign). */
1180
1208
  assign: (id, agentId) => this.postJson(`/v1/telephony/numbers/${encodeURIComponent(id)}/assign`, { agent_id: agentId }),
1181
1209
  /** Release a number back to the carrier (idempotent). */
@@ -1184,6 +1212,11 @@ export class PyAI {
1184
1212
  return (await res.json());
1185
1213
  },
1186
1214
  },
1215
+ numberOrders: {
1216
+ get: (id) => this.getJson(`/v1/telephony/number-orders/${encodeURIComponent(id)}`),
1217
+ /** Read/reconcile the original order; this never purchases another number. */
1218
+ reconcile: (id) => this.postJson(`/v1/telephony/number-orders/${encodeURIComponent(id)}/reconcile`, {}),
1219
+ },
1187
1220
  };
1188
1221
  /** Customer-owned two-human calls. Server keys only; history is project scoped. */
1189
1222
  translation = {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pyai/sdk",
3
- "version": "0.9.3",
3
+ "version": "0.11.0",
4
4
  "description": "Official TypeScript/JavaScript SDK for PyAI, speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -9,6 +9,26 @@
9
9
  ".": {
10
10
  "types": "./dist/index.d.ts",
11
11
  "import": "./dist/index.js"
12
+ },
13
+ "./dialer": {
14
+ "types": "./dist/dialer/index.d.ts",
15
+ "import": "./dist/dialer/index.js"
16
+ },
17
+ "./dialer/element": {
18
+ "types": "./dist/dialer/dialerElement.d.ts",
19
+ "import": "./dist/dialer/dialerElement.js"
20
+ },
21
+ "./dialer/server": {
22
+ "types": "./dist/dialer/serverBroker.d.ts",
23
+ "import": "./dist/dialer/serverBroker.js"
24
+ },
25
+ "./dialer/meeting-element": {
26
+ "types": "./dist/dialer/meetingRecorderElement.d.ts",
27
+ "import": "./dist/dialer/meetingRecorderElement.js"
28
+ },
29
+ "./dialer/app": {
30
+ "types": "./dist/dialer/appServer.d.ts",
31
+ "import": "./dist/dialer/appServer.js"
12
32
  }
13
33
  },
14
34
  "bin": {
@@ -56,5 +76,8 @@
56
76
  "devDependencies": {
57
77
  "@types/node": "^22.0.0",
58
78
  "typescript": "^5.7.0"
79
+ },
80
+ "dependencies": {
81
+ "@twilio/voice-sdk": "2.18.5"
59
82
  }
60
83
  }
@@ -1,7 +1,7 @@
1
1
  // Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
2
2
  export const omniTemplate = Object.freeze({
3
- "README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.9.3\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
4
- "package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.9.3\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
3
+ "README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.11.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
4
+ "package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.11.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
5
5
  "timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
6
6
  "main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
7
7
  "session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
package/src/cli.ts CHANGED
@@ -4,16 +4,16 @@ import "./cli-runtime.ts";
4
4
  import { createWriteStream, unlinkSync } from "node:fs";
5
5
  import { readFile, link, rename, unlink, stat, lstat } from "node:fs/promises";
6
6
  import { basename, dirname, resolve, join } from "node:path";
7
- import { randomUUID } from "node:crypto";
7
+ import { randomUUID, createHash } from "node:crypto";
8
8
  import { Readable, Transform } from "node:stream";
9
9
  import { pipeline } from "node:stream/promises";
10
10
  import { CliHttp, CliError } from "./cli-http.ts";
11
- import { browserLogin } from "./cli-web-auth.ts";
11
+ import { browserLogin, openBrowser } from "./cli-web-auth.ts";
12
12
  import { CliConfigError, resolveConfig, saveProfile, removeProfile, useProfile, listProfiles, validateApiKey, validateProfileName } from "./cli-config.ts";
13
13
  import { routes } from "./cli-routes.ts";
14
14
  import { aliases, shortFlags, recipes } from "./cli-dx.ts";
15
15
  import { CLI_TEMPLATES, scaffoldProject, CliInitError } from "./cli-init.ts";
16
- import { SPEECH_FORMATS, SPEECH_SAMPLE_RATES } from "./index.ts";
16
+ import { PyAI, SPEECH_FORMATS, SPEECH_SAMPLE_RATES } from "./index.ts";
17
17
 
18
18
  type FlagType = "boolean" | "string" | "repeat";
19
19
  type Flags = Record<string, string | boolean | string[]>;
@@ -46,6 +46,7 @@ const commands: Command[] = [
46
46
  { command: "init", description: "Create a new project with API context and request examples offline", args: ["directory"], flags: { template: "string" }, example: "pyai init voice-demo --template typescript" },
47
47
  { command: "smoke", description: "Check models, voices, and speech synthesis", flags: { "tolerate-upstream": "boolean" } },
48
48
  { command: "doctor", description: "Diagnose key, catalogs, and a Speak to Hear round-trip" },
49
+ { command: "dialer", description: "Open the private desktop calling and meeting-recording app", flags: { port: "string", "no-browser": "boolean", name: "string", "white-label": "boolean" }, example: "pyai dialer --profile production" },
49
50
  ...routes.map((r): Command => ({ command: r.command, description: r.description, args: r.id ? ["id"] : [], flags: {
50
51
  ...(r.body ? bodyFlags : {}), ...(r.binary ? outputFlags : {}), ...(r.wait ? waitFlags : {}),
51
52
  ...(r.query ?? {}), ...(r.confirm ? { confirm: "boolean" as const } : {}),
@@ -270,7 +271,7 @@ async function main(): Promise<void> {
270
271
  const grouped = group ? commands.filter(c => c.command.startsWith(`${group} `)) : commands;
271
272
  const visible = spec ? [spec, ...commands.filter(c => c.command.startsWith(`${spec.command} `))] : grouped.length ? grouped : commands;
272
273
  if (flags.json) output({ name: "pyai", globals, aliases, short_flags: shortFlags, commands: visible }, flags, undefined, false);
273
- else if (!spec && !group) process.stdout.write(`PyAI CLI — speech, agents, and automation\n\nStart here\n pyai login Sign in through your browser\n pyai speak "Hello from PyAI" -o hello.wav Turn text into audio\n pyai hear call.wav --text-only Print a transcript\n pyai dub call.wav --from en --to hi -o hi.wav\n Submit, wait, and save dubbed audio\n pyai init voice-demo --template omni Create a runnable Omni starter offline\n\nCalling\n pyai numbers list List owned phone numbers\n pyai numbers search --country US Find available numbers\n pyai calls list Inspect outbound calls\n pyai recipes calling --json Setup and preview examples\n pyai calls create --data @call.json --idempotency-key REQUEST_KEY --dry-run\n Preview a call without submitting it\n --dry-run previews a change without sending the request.\n Number purchases and outbound calls require --confirm and --idempotency-key.\n Use --confirm only after that purchase or call is authorized.\n\nDiscover\n pyai recipes [name] Copyable workflows\n pyai help <command> Command options and examples\n pyai help all Every command\n pyai schema [command] -j Machine-readable command definitions\n\nGroups\n auth profiles models voices agents jobs clones design\n cast dub recap trace tools vocabulary amd numbers calls omni\n\nShortcuts: login, logout, whoami, use, say (speak), hear (transcribe)\nOptions: -o FILE -f FILE -t TEXT -p PROFILE -j (JSON)\nGlobal: --base-url URL --api-key KEY --timeout SECONDS --retries N --dry-run\nCredentials: pyai login or PYAI_API_KEY. Run pyai doctor for a usage-consuming diagnostic.\n`);
274
+ else if (!spec && !group) process.stdout.write(`PyAI CLI — speech, agents, and automation\n\nStart here\n pyai login Sign in through your browser\n pyai speak "Hello from PyAI" -o hello.wav Turn text into audio\n pyai hear call.wav --text-only Print a transcript\n pyai dub call.wav --from en --to hi -o hi.wav\n Submit, wait, and save dubbed audio\n pyai init voice-demo --template omni Create a runnable Omni starter offline\n\nCalling\n pyai dialer Open the calling and meeting app\n pyai numbers list List owned phone numbers\n pyai numbers search --country US Find available numbers\n pyai calls list Inspect outbound calls\n pyai recipes calling --json Setup and preview examples\n pyai calls create --data @call.json --idempotency-key REQUEST_KEY --dry-run\n Preview a call without submitting it\n --dry-run previews a change without sending the request.\n Number purchases and outbound calls require --confirm and --idempotency-key.\n Use --confirm only after that purchase or call is authorized.\n\nDiscover\n pyai recipes [name] Copyable workflows\n pyai help <command> Command options and examples\n pyai help all Every command\n pyai schema [command] -j Machine-readable command definitions\n\nGroups\n auth profiles models voices agents jobs clones design\n cast dub recap trace tools vocabulary amd numbers calls omni\n\nShortcuts: login, logout, whoami, use, say (speak), hear (transcribe)\nOptions: -o FILE -f FILE -t TEXT -p PROFILE -j (JSON)\nGlobal: --base-url URL --api-key KEY --timeout SECONDS --retries N --dry-run\nCredentials: pyai login or PYAI_API_KEY. Run pyai doctor for a usage-consuming diagnostic.\n`);
274
275
  else process.stdout.write(`PyAI CLI — speech, agents, and automation\n\n${visible.map(c => ` pyai ${c.command}${(c.args ?? []).map(a => c.optionalArgs ? ` [${a}${c.variadic ? "..." : ""}]` : ` <${a}>`).join("")}\n ${c.description}${c.flags ? `\n Options: ${Object.keys(c.flags).map(k => `--${k}${c.flags![k] === "boolean" ? "" : " VALUE"}`).join(" ")}` : ""}${c.example ? `\n ${c.example}` : ""}`).join("\n")}\n\nGlobal: --profile NAME --json --timeout SECONDS --retries N --dry-run\n --base-url URL --api-key KEY --help --version\nShort flags: -o FILE -f FILE -t TEXT -p PROFILE -j (JSON)\nAuth: pyai login or PYAI_API_KEY. Discover: pyai schema --json\n`);
275
276
  return;
276
277
  }
@@ -334,6 +335,26 @@ async function main(): Promise<void> {
334
335
  if (config.apiKey) secrets.add(config.apiKey);
335
336
  const retries = number(flags, "retries", 2, 0, 5); if (!Number.isInteger(retries)) usage("--retries must be an integer");
336
337
  const http = new CliHttp({ baseURL: config.baseURL, apiKey: config.apiKey, timeoutMs: number(flags, "timeout", 30, 0.001) * 1000, maxRetries: retries });
338
+ if (cmd === "dialer") {
339
+ const port = number(flags, "port", 43117, 0, 65535); if (!Number.isInteger(port)) usage("--port must be an integer");
340
+ if (flags["dry-run"]) { output({ dry_run: true, command: cmd, bind: "127.0.0.1", port, browser: !flags["no-browser"], calls: "Only after a foreground Call or Start calling action; no number is dialed at startup" }, flags); return; }
341
+ if (!config.apiKey) throw new CliError("authentication_required", "Run pyai login or set PYAI_API_KEY to a live project key with telephony:manage.", 3);
342
+ const { startDialerApp } = await import("./dialer/appServer.ts");
343
+ const fingerprint = createHash("sha256").update(config.baseURL).update("\0").update(config.apiKey).digest("hex");
344
+ const sdk = new PyAI({ apiKey: config.apiKey, baseURL: config.baseURL, fetch: (url, init) => fetch(url, { ...init, redirect: "error", signal: init?.signal ?? AbortSignal.timeout(15000) }) });
345
+ const app = await startDialerApp({ sdk, userId: `desktop_${fingerprint.slice(0,40)}`, scope: `desktop:${fingerprint}`, port, name: value(flags, "name"), whiteLabel: flags["white-label"] === true });
346
+ output({ url: app.url, local_only: true }, flags, `PyAI Phone is ready at ${app.url}\nKeep this window open. Press Ctrl+C to stop the app and end its owned calls.`);
347
+ if (flags["no-browser"] || !(await openBrowser(app.launchUrl))) process.stderr.write(`Private one-use launch link (valid for 60 seconds): ${app.launchUrl}\n`);
348
+ await new Promise<void>(resolve => {
349
+ let stopping = false;
350
+ const stop = () => { if (stopping) return; stopping = true; void app.close().then(result => {
351
+ if (result.unsettled.length) process.stderr.write("Some call completions are still unconfirmed. Reopen the same profile to recover them before starting new calls.\n");
352
+ resolve();
353
+ }); };
354
+ process.once("SIGINT", stop); process.once("SIGTERM", stop);
355
+ });
356
+ return;
357
+ }
337
358
  if (cmd === "auth logout") { if (!flags["dry-run"]) await removeProfile(config.profile); output({ profile: config.profile, removed: !flags["dry-run"] }, flags, `${flags["dry-run"] ? "Would remove" : "Removed"} saved profile ${config.profile}`); return; }
338
359
  if (sandbox) {
339
360
  if (preview(flags, config.baseURL, "POST", "/v1/sandbox/keys", {})) return;
@@ -0,0 +1 @@
1
+ :root{font:14px/1.5 system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif;color:#252638;background:#f6f6fa;--accent:#5b55e7}*{box-sizing:border-box}body{margin:0;display:grid;grid-template-columns:240px 1fr;min-height:100vh}aside{padding:34px 24px;border-right:1px solid #e6e7ee;background:#fff;display:flex;flex-direction:column;position:sticky;top:0;height:100vh}.brand{font-size:18px;font-weight:700;display:flex;align-items:center;gap:10px;color:inherit;text-decoration:none;letter-spacing:-.5px}.logo{display:grid;place-items:center;border-radius:13px;width:36px;height:36px;color:white;background:var(--accent);font-size:30px;line-height:1}nav{display:grid;gap:6px;margin-top:48px}button,select,textarea{font:inherit}button{cursor:pointer;border:0}nav button{padding:14px 13px;text-align:left;border-radius:12px;background:transparent;font-weight:550;color:#7e8193;display:flex;justify-content:space-between}nav button.selected{background:#eeedfd;color:var(--accent)}.aside-foot{margin-top:auto;font-size:12px;color:#73778c}.aside-foot small{font-size:11px;color:#a0a3b3;padding-left:16px}.online{display:inline-block;width:6px;height:6px;border-radius:50%;background:#24b18b;margin-right:6px}main{padding:44px clamp(22px,4.5vw,78px) 24px;max-width:1440px;width:100%;margin:auto}header{display:flex;justify-content:space-between;align-items:center;margin-bottom:38px}.eyebrow{font-size:10px;letter-spacing:1.7px;font-weight:650;color:#9294ab;margin:0 0 9px}h1{font-weight:600;letter-spacing:-1.3px;font-size:32px;line-height:1.2;margin:0 0 12px}.subtitle{margin:0;color:#83879c}.local-badge{border:1px solid #e3e5ee;border-radius:20px;padding:7px 12px;color:#888da2;font-size:11px;background:white}.calling-grid{display:grid;grid-template-columns:minmax(280px,380px) minmax(300px,1fr);gap:clamp(24px,4vw,72px);align-items:start}.source-label,.queue-card label{display:block;font-size:11px;font-weight:600;color:#7b8196;margin-bottom:8px}select{width:100%;padding:11px 14px;border:1px solid #e2e4ee;border-radius:11px;background:white;color:#464d65;min-height:43px}.recording-choice{display:flex;align-items:center;gap:7px;font-size:10px;color:#8b90a3;margin:12px 0 20px}.recording-choice input{accent-color:var(--accent)}.queue-card{background:white;padding:30px;border:1px solid #e7e8f0;border-radius:22px;margin-top:86px}.card-top{display:flex;justify-content:space-between}.quiet-icon{font-size:23px;color:#a5a1f0}.queue-card h2{font-size:24px;letter-spacing:-.8px;font-weight:550;line-height:1.3;margin:10px 0 15px}.queue-card>p{font-size:12px;color:#8c91a5;margin:0 0 26px;max-width:300px}.queue-card textarea{border:1px solid #e4e6f0;border-radius:12px;background:#fafaff;width:100%;height:114px;resize:vertical;padding:12px;font-size:13px;margin-bottom:15px}.queue-actions{display:flex;gap:7px;flex-wrap:wrap;margin-top:18px}.queue-actions button{border-radius:10px;background:#f1f2f8;color:#697087;padding:11px 13px;font-size:11px;font-weight:600}.queue-actions button.primary{background:var(--accent);color:white}.queue-actions button:disabled{opacity:.45;cursor:default}#queue-status{font-size:11px;color:#9398ab;margin:15px 0 0}.meeting-grid{display:grid;grid-template-columns:1fr minmax(300px,440px);gap:60px;align-items:center}.meeting-copy h2{font-size:48px;letter-spacing:-2px;font-weight:550;line-height:1.05;margin:18px 0 25px}.meeting-copy>p{font-size:14px;line-height:1.8;color:#81879b;max-width:320px}.feature{display:flex;gap:14px;max-width:330px;margin-top:23px;color:#8b91a5;font-size:12px}.feature b{background:#edebff;color:#837bdb;border-radius:10px;width:36px;min-width:36px;height:36px;display:grid;place-items:center}.feature strong{display:block;color:#5a617a;font-size:12px;font-weight:600;margin-bottom:3px}footer{font-size:10px;color:#a0a6b8;margin-top:38px;max-width:660px}#error{color:#b42338;font-size:12px;min-height:18px;margin-bottom:12px}button:focus-visible,select:focus-visible,textarea:focus-visible{outline:3px solid #bdb8ff;outline-offset:3px}[hidden]{display:none!important}@media(max-width:1000px){body{grid-template-columns:190px 1fr}aside{padding:28px 18px}.calling-grid{gap:25px;grid-template-columns:1fr}.queue-card{margin-top:0;max-width:380px}.meeting-grid{grid-template-columns:1fr;gap:24px}.meeting-copy h2{font-size:38px}.meeting-copy .feature{display:none}}@media(max-width:650px){body{display:block}aside{height:auto;padding:18px 22px;position:static;border:0;border-bottom:1px solid #e8e9f0}.brand{font-size:16px}nav{margin-top:20px;display:flex}nav button{flex:1;font-size:12px}.aside-foot{display:none}main{padding:28px 22px}header{margin-bottom:22px}h1{font-size:27px}.local-badge{display:none}.calling-grid{justify-items:center}.calling-grid>div,.queue-card{width:100%;max-width:380px}.meeting-copy h2{font-size:36px}.meeting-copy>p{font-size:12px}.meeting-grid{display:block}pyai-meeting-recorder{margin-top:25px}}
@@ -0,0 +1,7 @@
1
+ <!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="theme-color" content="#5b55e7"><title>PyAI Phone</title><link rel="icon" href="/icon.svg"><link rel="manifest" href="/manifest.webmanifest"><link rel="stylesheet" href="/app.css"><script type="module" src="/app.js"></script></head><body>
2
+ <aside><a class="brand" href="/" aria-label="PyAI Phone"><span class="logo">p</span> <span data-brand>PyAI Phone</span></a><nav aria-label="Workspace"><button class="selected" data-tab="calls">Phone <span>↗</span></button><button data-tab="meetings">Meeting recorder <span>◉</span></button></nav><div class="aside-foot"><span class="online"></span> Private workspace<br><small>Powered by PyAI</small></div></aside>
3
+ <main><header><div><p class="eyebrow">YOUR CONVERSATIONS, TOGETHER</p><h1 data-heading>Make the next connection.</h1><p class="subtitle" data-subtitle>A clear line. A little less busywork.</p></div><span class="local-badge">On this device</span></header>
4
+ <div id="error" role="alert"></div><button id="retry" hidden>Retry connection</button><section data-panel="calls"><div class="calling-grid"><div><label class="source-label" for="source">Calling from</label><select id="source" disabled><option>Loading workspace numbers…</option></select><label class="recording-choice"><input id="record-calls" type="checkbox">Record calls · tell the other person before recording</label><pyai-dialer></pyai-dialer></div>
5
+ <section class="queue-card"><div class="card-top"><span class="eyebrow">STAY IN YOUR FLOW</span><span class="quiet-icon">≋</span></div><h2>One good conversation<br>after another.</h2><p>Paste your contacts and move through them at your own pace. Each call waits for your wrap-up.</p><label for="contacts">Phone numbers, one per line</label><textarea id="contacts" placeholder="+61 2 5550 0100&#10;+1 415 555 0100" spellcheck="false"></textarea><label for="mode">Calling order</label><select id="mode"><option value="auto">In list order</option><option value="dynamic">Highest priority first · number, priority</option></select><div class="queue-actions"><button id="queue-start" class="primary" disabled>Start calling</button><button id="queue-pause" disabled>Pause queue</button><button id="queue-resume" disabled>Resume queue</button></div><p id="queue-status" role="status" aria-live="polite">You choose when to start. Your workspace calling rules apply to every call.</p></section></div></section>
6
+ <section data-panel="meetings" hidden><div class="meeting-grid"><div class="meeting-copy"><span class="eyebrow">LESS DISTRACTION, MORE PRESENCE</span><h2>Stay in the<br>conversation.</h2><p>Choose the audio you want to keep. Record your meeting and microphone together, pause when you need to, and save a private audio file.</p><div class="feature"><b>◉</b><span><strong>Your choice of audio</strong>Choose a supported meeting tab or system source in the browser picker.</span></div><div class="feature"><b>▦</b><span><strong>Saved as you go</strong>Committed audio can be recovered after an interrupted tab.</span></div><div class="feature"><b>↧</b><span><strong>Yours to keep</strong>Download your recording. No automatic uploads or hidden capture.</span></div></div><pyai-meeting-recorder></pyai-meeting-recorder></div></section>
7
+ <footer>Call history and call recordings are stored in your PyAI workspace. Meeting audio stays in this browser until you save or submit it.</footer></main></body></html>