@pyai/sdk 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.schema.json +2 -1
- package/README.md +58 -6
- package/dist/cli-omni-template.d.ts +2 -2
- package/dist/cli-omni-template.js +2 -2
- package/dist/cli-web-auth.d.ts +2 -0
- package/dist/cli-web-auth.js +2 -0
- package/dist/cli.js +2 -2
- package/dist/dialer/appServer.d.ts +4 -0
- package/dist/dialer/appServer.js +84 -4
- package/dist/dialer/dialerElement.js +28 -6
- package/dist/dialer/dialingQueue.d.ts +33 -10
- package/dist/dialer/dialingQueue.js +124 -41
- package/dist/dialer/fileMeetingIntelligenceJournal.d.ts +14 -0
- package/dist/dialer/fileMeetingIntelligenceJournal.js +147 -0
- package/dist/dialer/index.d.ts +1 -0
- package/dist/dialer/index.js +1 -0
- package/dist/dialer/indexedDbJournal.d.ts +9 -0
- package/dist/dialer/indexedDbJournal.js +62 -1
- package/dist/dialer/managedDialer.d.ts +40 -2
- package/dist/dialer/managedDialer.js +137 -10
- package/dist/dialer/meetingIntelligence.d.ts +28 -0
- package/dist/dialer/meetingIntelligence.js +3 -0
- package/dist/dialer/meetingIntelligenceElement.d.ts +21 -0
- package/dist/dialer/meetingIntelligenceElement.js +112 -0
- package/dist/dialer/serverBroker.js +19 -2
- package/dist/dialer/serverMeetingIntelligence.d.ts +60 -0
- package/dist/dialer/serverMeetingIntelligence.js +207 -0
- package/dist/index.d.ts +40 -0
- package/dist/index.js +24 -0
- package/package.json +9 -1
- package/src/cli-omni-template.ts +2 -2
- package/src/cli-web-auth.ts +3 -0
- package/src/cli.ts +2 -2
- package/src/dialer/MEETING_INTELLIGENCE.md +76 -0
- package/src/dialer/app/app.html +3 -3
- package/src/dialer/app/app.js +45 -16
- package/src/dialer/appServer.ts +63 -5
- package/src/dialer/dialerElement.ts +24 -5
- package/src/dialer/dialingQueue.ts +108 -45
- package/src/dialer/fileMeetingIntelligenceJournal.ts +77 -0
- package/src/dialer/index.ts +2 -0
- package/src/dialer/indexedDbJournal.ts +58 -1
- package/src/dialer/managedDialer.ts +115 -9
- package/src/dialer/meetingIntelligence.ts +21 -0
- package/src/dialer/meetingIntelligenceElement.ts +65 -0
- package/src/dialer/serverBroker.ts +17 -2
- package/src/dialer/serverMeetingIntelligence.ts +148 -0
- package/src/index.ts +37 -1
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/** Server-only. Bind one broker to a trusted org/user and an existing project
|
|
2
|
+
* key. Never fall back to an operator/parent key or change that key's grants. */
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
export class MeetingApiError extends Error {
|
|
5
|
+
status;
|
|
6
|
+
constructor(status) { super(`PyAI meeting request failed (${status}).`); this.status = status; }
|
|
7
|
+
}
|
|
8
|
+
/** Uses only public routes. Keyed multipart requires the durable job retry contract; no public bucket,
|
|
9
|
+
* arbitrary result_url fetch, or browser-held key is involved. */
|
|
10
|
+
export function createPyaiMeetingApi(options) {
|
|
11
|
+
const base = new URL(options.baseURL ?? "https://api.pyai.com");
|
|
12
|
+
if ((base.protocol !== "https:" && !(base.protocol === "http:" && ["127.0.0.1", "[::1]"].includes(base.hostname))) || base.username || base.password || base.search || base.hash || base.pathname !== "/")
|
|
13
|
+
throw new Error("Use a trusted HTTPS API origin.");
|
|
14
|
+
if (!options.apiKey || /[\r\n]/.test(options.apiKey))
|
|
15
|
+
throw new Error("An existing project key is required.");
|
|
16
|
+
const timeout = options.timeoutMs ?? 20000;
|
|
17
|
+
if (!Number.isInteger(timeout) || timeout < 1 || timeout > 60000)
|
|
18
|
+
throw new Error("Meeting requests require a deadline of 1–60000 ms.");
|
|
19
|
+
const request = async (path, method = "GET", input, key) => {
|
|
20
|
+
const response = await (options.fetch ?? fetch)(new URL(path, base), { method, redirect: "error", signal: AbortSignal.timeout(timeout),
|
|
21
|
+
headers: { Authorization: `Bearer ${options.apiKey}`, ...(input !== undefined && !(input instanceof FormData) ? { "content-type": "application/json" } : {}), ...(key ? { "Idempotency-Key": key } : {}) },
|
|
22
|
+
...(input !== undefined ? { body: input instanceof FormData ? input : JSON.stringify(input) } : {}) });
|
|
23
|
+
if (!response.ok) {
|
|
24
|
+
await response.body?.cancel();
|
|
25
|
+
throw new MeetingApiError(response.status);
|
|
26
|
+
}
|
|
27
|
+
// Avoid unbounded or attacker-selected artifact downloads. Recap supplies
|
|
28
|
+
// retained transcript text even when Hear returns only a signed result URL.
|
|
29
|
+
const reader = response.body?.getReader();
|
|
30
|
+
if (!reader)
|
|
31
|
+
throw new Error("PyAI returned an empty response.");
|
|
32
|
+
const chunks = [];
|
|
33
|
+
let bytes = 0;
|
|
34
|
+
try {
|
|
35
|
+
while (true) {
|
|
36
|
+
const chunk = await reader.read();
|
|
37
|
+
if (chunk.done)
|
|
38
|
+
break;
|
|
39
|
+
bytes += chunk.value.length;
|
|
40
|
+
if (bytes > 4 * 1024 * 1024)
|
|
41
|
+
throw new Error("Meeting response exceeds 4 MiB.");
|
|
42
|
+
chunks.push(chunk.value);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
finally {
|
|
46
|
+
await reader.cancel();
|
|
47
|
+
}
|
|
48
|
+
return JSON.parse(Buffer.concat(chunks).toString("utf8"));
|
|
49
|
+
};
|
|
50
|
+
return { me: () => request("/v1/me"), recapEnabled: async () => (await request("/v1/recap/config")).enabled === true,
|
|
51
|
+
upload: (audio, filename, callId, key) => { const form = new FormData(); form.set("call_id", callId); form.set("diarize", "true"); form.set("output_formats", "json"); form.set("audio", audio, filename); return request("/v1/transcription/jobs", "POST", form, key); },
|
|
52
|
+
job: id => request(`/v1/transcription/jobs/${encodeURIComponent(id)}`), cancel: id => request(`/v1/transcription/jobs/${encodeURIComponent(id)}`, "DELETE"),
|
|
53
|
+
recap: async (id) => { try {
|
|
54
|
+
return await request(`/v1/recap/calls/${encodeURIComponent(id)}`);
|
|
55
|
+
}
|
|
56
|
+
catch (error) {
|
|
57
|
+
if (error instanceof MeetingApiError && error.status === 404)
|
|
58
|
+
return null;
|
|
59
|
+
throw error;
|
|
60
|
+
} } };
|
|
61
|
+
}
|
|
62
|
+
const hash = (...values) => createHash("sha256").update(JSON.stringify(values)).digest("hex");
|
|
63
|
+
const view = (entry) => ({ recordingId: entry.recordingId, state: entry.state, transcript: entry.transcript, notes: entry.notes, actionItems: structuredClone(entry.actionItems), error: entry.error });
|
|
64
|
+
export function createServerMeetingIntelligence(options) {
|
|
65
|
+
const { api, journal } = options, now = options.now ?? Date.now;
|
|
66
|
+
const maxBytes = options.maxBytes ?? 128 * 1024 * 1024, retention = options.retentionMs ?? 24 * 3600000;
|
|
67
|
+
if (![options.orgId, options.projectId, options.userId, options.keyId].every(value => value.trim()) || !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 256 * 1024 * 1024 || !Number.isSafeInteger(retention) || retention < 60000 || retention > 7 * 24 * 3600000)
|
|
68
|
+
throw new Error("Trusted identity and bounded private recording retention are required.");
|
|
69
|
+
const owner = hash(options.orgId, options.projectId, options.userId);
|
|
70
|
+
const identity = (id) => { if (!/^[A-Za-z0-9_-]{1,200}$/.test(id))
|
|
71
|
+
throw new Error("Invalid recording ID."); return "meeting_" + hash(owner, id); };
|
|
72
|
+
const owned = async (key) => { const entry = await journal.get(key); if (!entry || entry.owner !== owner || entry.key !== key)
|
|
73
|
+
throw new Error("No meeting in this workspace."); return entry; };
|
|
74
|
+
const availability = async () => {
|
|
75
|
+
const me = await api.me();
|
|
76
|
+
if (me.org_id !== options.orgId || me.project_id !== options.projectId || me.key_id !== options.keyId)
|
|
77
|
+
throw new Error("The selected PyAI project identity changed. Select the original project.");
|
|
78
|
+
const required = ["hear:transcribe", "transcribe:jobs", "recap:read", "recap:configure"];
|
|
79
|
+
const missing = required.filter(scope => !me.scopes?.includes(scope));
|
|
80
|
+
if (missing.length)
|
|
81
|
+
return { available: false, reason: `Select a project permitted to transcribe and summarize meetings. Missing scopes: ${missing.join(", ")}. Managed calling access alone does not include this feature.` };
|
|
82
|
+
if (!await api.recapEnabled())
|
|
83
|
+
return { available: false, reason: "Enable Recap through your workspace's normal settings before sending meeting audio." };
|
|
84
|
+
return { available: true, reason: null };
|
|
85
|
+
};
|
|
86
|
+
const requireAvailable = async () => { const result = await availability(); if (!result.available)
|
|
87
|
+
throw new Error(result.reason); };
|
|
88
|
+
const refresh = async (entry) => {
|
|
89
|
+
if (entry.state === "removed")
|
|
90
|
+
return view(entry);
|
|
91
|
+
if (now() >= entry.expiresAt) {
|
|
92
|
+
entry.state = "removed";
|
|
93
|
+
entry.transcript = null;
|
|
94
|
+
entry.notes = null;
|
|
95
|
+
entry.actionItems = [];
|
|
96
|
+
entry.error = "Private meeting retention expired. PyAI results and Recap follow workspace retention.";
|
|
97
|
+
await journal.save(entry);
|
|
98
|
+
if (entry.jobId)
|
|
99
|
+
await api.cancel(entry.jobId);
|
|
100
|
+
return view(entry);
|
|
101
|
+
}
|
|
102
|
+
if (!entry.jobId)
|
|
103
|
+
return view(entry);
|
|
104
|
+
const job = await api.job(entry.jobId);
|
|
105
|
+
if (job.job_id !== entry.jobId)
|
|
106
|
+
throw new Error("PyAI returned a different meeting job.");
|
|
107
|
+
if (["failed", "cancelled"].includes(job.status)) {
|
|
108
|
+
entry.state = "failed";
|
|
109
|
+
entry.error = "Transcription did not complete. Saved audio remains available locally; review the job before starting another submission.";
|
|
110
|
+
await journal.save(entry);
|
|
111
|
+
return view(entry);
|
|
112
|
+
}
|
|
113
|
+
if (job.status !== "completed") {
|
|
114
|
+
entry.state = "transcribing";
|
|
115
|
+
await journal.save(entry);
|
|
116
|
+
return view(entry);
|
|
117
|
+
}
|
|
118
|
+
const result = job.result;
|
|
119
|
+
if (typeof result?.text === "string")
|
|
120
|
+
entry.transcript = result.text;
|
|
121
|
+
entry.state = "summarizing";
|
|
122
|
+
await journal.save(entry);
|
|
123
|
+
const recap = await api.recap(entry.key);
|
|
124
|
+
if (recap && recap.call_id !== entry.key)
|
|
125
|
+
throw new Error("PyAI returned a different meeting summary.");
|
|
126
|
+
if (recap?.status === "failed") {
|
|
127
|
+
entry.state = "failed";
|
|
128
|
+
entry.error = "The transcript is ready but Recap failed. Review the existing summary job; audio is not submitted again.";
|
|
129
|
+
}
|
|
130
|
+
if (recap?.status === "complete") {
|
|
131
|
+
entry.transcript ??= recap.transcript?.utterances.map(item => item.text).join("\n") ?? null;
|
|
132
|
+
entry.notes = recap.record?.summary ?? recap.record?.tldr ?? null;
|
|
133
|
+
entry.actionItems = structuredClone(recap.record?.action_items ?? []);
|
|
134
|
+
entry.state = "complete";
|
|
135
|
+
entry.error = null;
|
|
136
|
+
}
|
|
137
|
+
await journal.save(entry);
|
|
138
|
+
return view(entry);
|
|
139
|
+
};
|
|
140
|
+
return { availability,
|
|
141
|
+
async submit(id, audio, noticeAccepted) {
|
|
142
|
+
if (noticeAccepted !== true)
|
|
143
|
+
throw new Error("Accept the upload and retention notice before sending audio.");
|
|
144
|
+
if (!audio.size || audio.size > maxBytes || !/^audio\/(webm|mp4|wav|x-wav|mpeg|ogg)(?:;|$)/i.test(audio.type))
|
|
145
|
+
throw new Error("Choose a supported saved audio recording within the workspace size limit.");
|
|
146
|
+
const key = identity(id);
|
|
147
|
+
await requireAvailable();
|
|
148
|
+
const digest = createHash("sha256").update(new Uint8Array(await audio.arrayBuffer())).digest("hex");
|
|
149
|
+
return journal.run(key, async () => {
|
|
150
|
+
let entry = await journal.get(key);
|
|
151
|
+
if (entry && (entry.owner !== owner || entry.digest !== digest || entry.mimeType !== audio.type || entry.bytes !== audio.size))
|
|
152
|
+
throw new Error("This recording ID already belongs to different audio. Recover the original submission.");
|
|
153
|
+
if (entry?.state === "removed")
|
|
154
|
+
throw new Error("This meeting was removed. It cannot be resubmitted with the same ID.");
|
|
155
|
+
if (!entry) {
|
|
156
|
+
entry = { key, recordingId: id, owner, digest, mimeType: audio.type, bytes: audio.size, createdAt: now(), expiresAt: now() + retention, jobId: null, state: "submitting", transcript: null, notes: null, actionItems: [], error: null };
|
|
157
|
+
await journal.save(entry);
|
|
158
|
+
}
|
|
159
|
+
if (now() >= entry.expiresAt || entry.jobId)
|
|
160
|
+
return refresh(entry);
|
|
161
|
+
// Persist the same content identity before POST. A lost response is
|
|
162
|
+
// recovered with the SAME durable JSON/multipart job retry key.
|
|
163
|
+
const extension = { "audio/webm": "webm", "audio/mp4": "m4a", "audio/wav": "wav", "audio/x-wav": "wav", "audio/mpeg": "mp3", "audio/ogg": "ogg" }[audio.type.split(";")[0].toLowerCase()];
|
|
164
|
+
const job = await api.upload(audio, `meeting.${extension}`, key, key);
|
|
165
|
+
entry.jobId = job.job_id;
|
|
166
|
+
entry.state = "transcribing";
|
|
167
|
+
await journal.save(entry);
|
|
168
|
+
return refresh(entry);
|
|
169
|
+
});
|
|
170
|
+
},
|
|
171
|
+
async get(id) { const key = identity(id); await requireAvailable(); return journal.run(key, async () => refresh(await owned(key))); },
|
|
172
|
+
async remove(id) {
|
|
173
|
+
const key = identity(id);
|
|
174
|
+
return journal.run(key, async () => {
|
|
175
|
+
const entry = await owned(key);
|
|
176
|
+
entry.state = "removed";
|
|
177
|
+
entry.transcript = null;
|
|
178
|
+
entry.notes = null;
|
|
179
|
+
entry.actionItems = [];
|
|
180
|
+
entry.error = "Cached notes removed. PyAI uploaded audio may remain for up to 7 days and results up to 30 days; Recap follows workspace retention. Cancellation is not erasure.";
|
|
181
|
+
await journal.save(entry);
|
|
182
|
+
// Erase never submits or replays audio. If the initial POST outcome was
|
|
183
|
+
// unknown we cannot invent its job ID or claim it was cancelled.
|
|
184
|
+
if (!entry.jobId)
|
|
185
|
+
entry.error = "Cached notes removed. The earlier submission outcome is unknown; a PyAI job may remain under workspace retention. Uploaded audio may remain for up to 7 days and results up to 30 days.";
|
|
186
|
+
await journal.save(entry);
|
|
187
|
+
if (entry.jobId)
|
|
188
|
+
await api.cancel(entry.jobId);
|
|
189
|
+
return view(entry);
|
|
190
|
+
});
|
|
191
|
+
}, };
|
|
192
|
+
}
|
|
193
|
+
/** Reuses the host's already selected SDK instance and its normal transport.
|
|
194
|
+
* The SDK never exposes the key to the renderer. */
|
|
195
|
+
export function createSdkMeetingApi(sdk) {
|
|
196
|
+
return { me: () => sdk.me(), recapEnabled: async () => (await sdk.recap.config.get()).enabled === true,
|
|
197
|
+
upload: (audio, filename, callId, key) => sdk.transcriptionJobs.upload({ file: audio, filename, call_id: callId, diarize: true, output_formats: ["json"] }, { idempotencyKey: key }),
|
|
198
|
+
job: id => sdk.transcriptionJobs.get(id), cancel: id => sdk.transcriptionJobs.cancel(id),
|
|
199
|
+
recap: async (id) => { try {
|
|
200
|
+
return await sdk.recap.calls.get(id);
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
if (error.status === 404)
|
|
204
|
+
return null;
|
|
205
|
+
throw error;
|
|
206
|
+
} } };
|
|
207
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -201,6 +201,8 @@ export interface AgentConfig {
|
|
|
201
201
|
name?: string;
|
|
202
202
|
/** Saved instructions: 8,000 Unicode characters by default; 32,000 on commitment plans. */
|
|
203
203
|
persona_system_prompt?: string | null;
|
|
204
|
+
/** Separate setup rules, writable only by explicitly enrolled server credentials. */
|
|
205
|
+
platform_guidance?: string | null;
|
|
204
206
|
/** False restricts retrieval to explicit agent bindings, including none. Default true. */
|
|
205
207
|
inherit_org_knowledgebases?: boolean;
|
|
206
208
|
greeting?: string | null;
|
|
@@ -650,6 +652,8 @@ export interface OmniConfigure {
|
|
|
650
652
|
voice_id?: string;
|
|
651
653
|
/** System prompt / role + instructions for the agent. */
|
|
652
654
|
persona?: string;
|
|
655
|
+
/** Separate bounded platform guidance, applied before customer instructions. */
|
|
656
|
+
role_scaffold?: string;
|
|
653
657
|
/** Customer-hosted URL the engine calls per turn for grounding. */
|
|
654
658
|
kb_endpoint?: string;
|
|
655
659
|
/** Bearer the engine presents to `kb_endpoint`. */
|
|
@@ -815,6 +819,7 @@ export declare class OmniConnection {
|
|
|
815
819
|
}
|
|
816
820
|
/** Shape of `GET /v1/me`. Fields are best-effort / forward-compatible. */
|
|
817
821
|
export interface MeResponse {
|
|
822
|
+
project_id?: string;
|
|
818
823
|
object?: string;
|
|
819
824
|
key_id?: string;
|
|
820
825
|
org_id?: string;
|
|
@@ -851,6 +856,8 @@ export interface TelephonyCall {
|
|
|
851
856
|
from_number_id: string | null;
|
|
852
857
|
agent_id: string | null;
|
|
853
858
|
call_type?: "human";
|
|
859
|
+
transfer_enabled?: boolean;
|
|
860
|
+
transfers?: TelephonyTransfer[];
|
|
854
861
|
user_id?: string;
|
|
855
862
|
client_reference: string | null;
|
|
856
863
|
stop_requested: boolean;
|
|
@@ -861,6 +868,12 @@ export interface TelephonyCall {
|
|
|
861
868
|
[key: string]: unknown;
|
|
862
869
|
}
|
|
863
870
|
export type TelephonyUseCase = "transactional" | "support" | "sales" | "collections" | "survey";
|
|
871
|
+
export interface TelephonyTransfer {
|
|
872
|
+
id: string;
|
|
873
|
+
mode: "warm" | "cold";
|
|
874
|
+
status: "holding" | "dialing" | "ready" | "completing" | "canceling" | "completed" | "canceled" | "failed";
|
|
875
|
+
to: string | null;
|
|
876
|
+
}
|
|
864
877
|
export interface TelephonyCallCreate {
|
|
865
878
|
from_number_id: string;
|
|
866
879
|
to: string;
|
|
@@ -883,9 +896,11 @@ export type TelephonyBrowserSessionCreate = {
|
|
|
883
896
|
} & ({
|
|
884
897
|
to: string;
|
|
885
898
|
destinations?: never;
|
|
899
|
+
transfer_enabled?: boolean;
|
|
886
900
|
} | {
|
|
887
901
|
to?: never;
|
|
888
902
|
destinations: string[];
|
|
903
|
+
transfer_enabled?: false;
|
|
889
904
|
});
|
|
890
905
|
export interface TelephonyBrowserSession {
|
|
891
906
|
object: "telephony.browser_session";
|
|
@@ -1443,6 +1458,19 @@ export declare class PyAI {
|
|
|
1443
1458
|
create: (params: CreateJobParams, opts?: {
|
|
1444
1459
|
idempotencyKey?: string;
|
|
1445
1460
|
}) => Promise<TranscriptionJob>;
|
|
1461
|
+
/** Upload a recording with a durable retry key. Requires the current
|
|
1462
|
+
* keyed multipart job contract; never change the key after an unknown ACK. */
|
|
1463
|
+
upload: (params: {
|
|
1464
|
+
file: Blob;
|
|
1465
|
+
filename: string;
|
|
1466
|
+
call_id?: string;
|
|
1467
|
+
diarize?: boolean;
|
|
1468
|
+
output_formats?: Array<"json" | "srt" | "vtt">;
|
|
1469
|
+
}, opts: {
|
|
1470
|
+
idempotencyKey: string;
|
|
1471
|
+
}) => Promise<TranscriptionJob>;
|
|
1472
|
+
/** Cancels queued/running work; existing cloud retention still applies. */
|
|
1473
|
+
cancel: (jobId: string) => Promise<TranscriptionJob>;
|
|
1446
1474
|
get: (jobId: string) => Promise<TranscriptionJob>;
|
|
1447
1475
|
list: (params?: {
|
|
1448
1476
|
limit?: number;
|
|
@@ -1498,6 +1526,18 @@ export declare class PyAI {
|
|
|
1498
1526
|
range?: string;
|
|
1499
1527
|
}) => Promise<Response>;
|
|
1500
1528
|
deleteRecording: (id: string) => Promise<void>;
|
|
1529
|
+
transfers: {
|
|
1530
|
+
/** Persist the key and target before requesting a consultation. */
|
|
1531
|
+
create: (id: string, input: {
|
|
1532
|
+
to: string;
|
|
1533
|
+
mode: "warm" | "cold";
|
|
1534
|
+
user_id: string;
|
|
1535
|
+
}, options: {
|
|
1536
|
+
idempotencyKey: string;
|
|
1537
|
+
}) => Promise<TelephonyCall>;
|
|
1538
|
+
complete: (id: string, transferId: string, userId: string) => Promise<TelephonyCall>;
|
|
1539
|
+
cancel: (id: string, transferId: string, userId: string) => Promise<TelephonyCall>;
|
|
1540
|
+
};
|
|
1501
1541
|
};
|
|
1502
1542
|
browserSessions: {
|
|
1503
1543
|
/** Persist input and key on your server; retries recover the original session. */
|
package/dist/index.js
CHANGED
|
@@ -1113,6 +1113,24 @@ export class PyAI {
|
|
|
1113
1113
|
});
|
|
1114
1114
|
return (await res.json());
|
|
1115
1115
|
},
|
|
1116
|
+
/** Upload a recording with a durable retry key. Requires the current
|
|
1117
|
+
* keyed multipart job contract; never change the key after an unknown ACK. */
|
|
1118
|
+
upload: async (params, opts) => {
|
|
1119
|
+
if (!opts.idempotencyKey || opts.idempotencyKey.length > 255 || /[\r\n]/.test(opts.idempotencyKey))
|
|
1120
|
+
throw new Error("A valid recording retry key is required.");
|
|
1121
|
+
const form = new FormData();
|
|
1122
|
+
if (params.call_id)
|
|
1123
|
+
form.set("call_id", params.call_id);
|
|
1124
|
+
if (params.diarize !== undefined)
|
|
1125
|
+
form.set("diarize", String(params.diarize));
|
|
1126
|
+
if (params.output_formats)
|
|
1127
|
+
form.set("output_formats", params.output_formats.join(","));
|
|
1128
|
+
form.set("audio", params.file, params.filename);
|
|
1129
|
+
const response = await this.request("/v1/transcription/jobs", { method: "POST", headers: this.authHeaders({ "Idempotency-Key": opts.idempotencyKey }), body: form });
|
|
1130
|
+
return await response.json();
|
|
1131
|
+
},
|
|
1132
|
+
/** Cancels queued/running work; existing cloud retention still applies. */
|
|
1133
|
+
cancel: async (jobId) => { const response = await this.request(`/v1/transcription/jobs/${encodeURIComponent(jobId)}`, { method: "DELETE", headers: this.authHeaders() }); return await response.json(); },
|
|
1116
1134
|
get: (jobId) => this.getJson(`/v1/transcription/jobs/${encodeURIComponent(jobId)}`),
|
|
1117
1135
|
list: (params = {}) => {
|
|
1118
1136
|
const q = new URLSearchParams();
|
|
@@ -1177,6 +1195,12 @@ export class PyAI {
|
|
|
1177
1195
|
decideAdmission: (id, input) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/admission`, input),
|
|
1178
1196
|
recording: (id, options = {}) => this.request(`/v1/telephony/calls/${encodeURIComponent(id)}/recording`, { headers: this.authHeaders(options.range ? { Range: options.range } : {}) }),
|
|
1179
1197
|
deleteRecording: async (id) => { await this.deleteReq(`/v1/telephony/calls/${encodeURIComponent(id)}/recording`); },
|
|
1198
|
+
transfers: {
|
|
1199
|
+
/** Persist the key and target before requesting a consultation. */
|
|
1200
|
+
create: (id, input, options) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/transfers`, input, telephonyIdempotencyHeader(options.idempotencyKey)),
|
|
1201
|
+
complete: (id, transferId, userId) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/transfers/${encodeURIComponent(transferId)}/complete`, { user_id: userId }),
|
|
1202
|
+
cancel: (id, transferId, userId) => this.postJson(`/v1/telephony/calls/${encodeURIComponent(id)}/transfers/${encodeURIComponent(transferId)}/cancel`, { user_id: userId }),
|
|
1203
|
+
},
|
|
1180
1204
|
},
|
|
1181
1205
|
browserSessions: {
|
|
1182
1206
|
/** Persist input and key on your server; retries recover the original session. */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pyai/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Official TypeScript/JavaScript SDK for PyAI, speech-to-text (Hear), text-to-speech (Speak), realtime voice agents (Omni), and call compliance (Trace).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -29,6 +29,14 @@
|
|
|
29
29
|
"./dialer/app": {
|
|
30
30
|
"types": "./dist/dialer/appServer.d.ts",
|
|
31
31
|
"import": "./dist/dialer/appServer.js"
|
|
32
|
+
},
|
|
33
|
+
"./dialer/meeting-intelligence": {
|
|
34
|
+
"types": "./dist/dialer/meetingIntelligenceElement.d.ts",
|
|
35
|
+
"import": "./dist/dialer/meetingIntelligenceElement.js"
|
|
36
|
+
},
|
|
37
|
+
"./dialer/meeting-server": {
|
|
38
|
+
"types": "./dist/dialer/serverMeetingIntelligence.d.ts",
|
|
39
|
+
"import": "./dist/dialer/serverMeetingIntelligence.js"
|
|
32
40
|
}
|
|
33
41
|
},
|
|
34
42
|
"bin": {
|
package/src/cli-omni-template.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Generated by scripts/generate-omni-starter.mjs; edit examples/omni-starter instead.
|
|
2
2
|
export const omniTemplate = Object.freeze({
|
|
3
|
-
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.
|
|
4
|
-
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.
|
|
3
|
+
"README.md": "# Runnable Omni starter\n\nRequires Node 20.19+ and ESM. Node 20 uses the included `ws` transport;\nNode 22 also works. The separate MCP server requires Node 22+.\nThis server-side example uses a synthetic read-only office-hours tool. Read\n`PYAI.md` when generating the project through the CLI.\n\n<!-- omni-install:start -->\n```sh\nnpm install @pyai/sdk@0.14.0\nnpx pyai init voice-demo --template omni\ncd voice-demo\nnpm install\n```\n\nInitialization is offline; installing dependencies and running `main.mjs` are\nseparate steps. For an existing checkout of this example, run `npm install`\nin its directory instead.\n<!-- omni-install:end -->\n\nInject `PYAI_API_KEY` through your secret manager or shell environment. Do not\npaste a real key into source, a prompt, or shell history. `.env.example` documents\nthe variable; this program does not load `.env` or saved CLI profiles.\n\nPrepare a 24 kHz PCM16 mono WAV, at most 20 seconds, saying “Please look up the\noffice opening time.” Then run:\n\n```sh\nnode main.mjs caller.wav\n```\n\nRunning the program explicitly opens one bounded Omni session and uploads its\ncaptured reply to Hear. Both consume your key's allowed usage. Automatic retries\nare disabled. No managed number is bought and no phone call is placed.\n\n`session.mjs` waits for configuration and the greeting to drain, sends exactly\none paced input stream (caller PCM or silence), returns the read tool's result,\nand captures output using the rate from `hello.audio_out`. Never run a separate\nsilence timer alongside caller audio. `timing.mjs` is generated from the existing\nOmni evaluation harness's timing utilities.\n\nThe new private `omni-run-*` folder contains `session.wav`, `reply.wav`, and\n`report.json`. The report separates received audio, an office-hours answer\nrecovered by Hear from captured audio, and physical playback (not tested).\nThe office-hours check accepts formatting such as `opens@9a.m`, `9 a.m.` and\n`nine in the morning` in a complete affirmative opening-time sentence. Wrong\ntimes, negation, conflicting or truncated text remain unverified. This is a\nconservative check for the example fact, not a general answer-quality score.\nA synthesis transcript is advisory; it is not evidence that speech arrived.\nThis starter does not negotiate playback receipts. Capture ends after the\nsimulated queue drains and two seconds of quiet, a bounded capture heuristic.\nListen to the saved file when qualifying actual sound quality or completeness.\n\nTo exercise interruption, supply a second WAV saying “Stop speaking now”:\n\n```sh\nnode main.mjs caller.wav interruption.wav\n```\n\nThe second clip starts while reply audio is queued. `onBargeIn` clears that\nsimulated queue, and capture observes at least five seconds after that clip ends.\nAn interrupted answer can correctly fail the full-answer check;\ninspect the separate interruption evidence. This is not a physical speaker test.\nFor a real playback adapter, cancel both queued and currently playing audio.\nReplace the example lookup with authorized application data before product use.\n\nOn a session error, the program exits with JSON containing a recognized machine\n`code` (for example `media_dead` or `invalid_configure`). Unknown server codes\nbecome `unrecognized_server_error`; transport or decoding errors without a\nrecognized transport code become `transport_or_protocol_error`. Raw error text,\ncall IDs and unknown code values are omitted. No automatic retry is attempted;\ninspect the code and your authorized diagnostics before deciding to retry.\n",
|
|
4
|
+
"package.json": "{\n \"name\": \"pyai-omni-starter\",\n \"version\": \"0.0.0\",\n \"private\": true,\n \"type\": \"module\",\n \"engines\": {\n \"node\": \">=20.19.0\"\n },\n \"scripts\": {\n \"start\": \"node main.mjs\"\n },\n \"dependencies\": {\n \"@pyai/sdk\": \"0.14.0\",\n \"ws\": \"^8.18.3\"\n }\n}\n",
|
|
5
5
|
"timing.mjs": "// Monotonic capture timing. Arrival time is distinct from queued playback:\n// a single packet may contain seconds of audio and must finish before settling.\nexport const FRAME_MS = 20;\nexport const REALTIME_GAP_LIMIT_MS = 100;\n// Interruption tests need an observation window after caller transmission,\n// even when an earlier cue/response already finished. These are test bounds,\n// not an inferred engine reply-end or a naturalness threshold.\nexport const INTERRUPTION_OBSERVATION = Object.freeze({\n minimumAfterCallerStreamMs: 5000, quietMs: 2000, maximumAfterCallerStreamMs: 25000,\n});\n\n/** Approximate audible bounds; this is energy detection, not a speech/VAD claim. */\nexport function audibleBounds(pcm, rate, rmsFloor = 160) {\n const window = Math.max(1, Math.round(rate / 100)); // 10 ms windows\n let first = null;\n let last = null;\n for (let off = 0; off < pcm.length; off += window) {\n const end = Math.min(pcm.length, off + window);\n let energy = 0;\n for (let i = off; i < end; i++) energy += pcm[i] * pcm[i];\n if (Math.sqrt(energy / (end - off)) >= rmsFloor) {\n first ??= off;\n last = end;\n }\n }\n return { first, last };\n}\n\nexport function newAudioCapture(started) {\n return {\n started, firstPacketAt: null, lastPacketAt: null,\n firstAudioAt: null, lastAudioAt: null, playbackEndAt: null,\n samples: 0, pcm: [], turnBeginAt: null, eouMs: null,\n latestTurnBeginAt: null, postTurnBeginFirstPacketAt: null,\n postTurnBeginFirstAudioAt: null, lastAssistantTranscriptAt: null, lastCallerTranscriptAt: null,\n callerTranscriptEvents: [], assistantTranscriptEvents: [],\n turnBegins: [], unheardCancellations: [], unheardSupersededTurns: [], tools: [], toolResults: [], kb: null, events: [],\n };\n}\n\nexport function recordTurnBegin(ctx, at, turn, origin = 0) {\n const prior = ctx.turnBegins.at(-1);\n const cancellation = ctx.unheardCancellations.at(-1);\n if (prior && cancellation && !cancellation.admitted\n && cancellation.turn === prior.turn && cancellation.turnBeginIndex === ctx.turnBegins.length - 1\n && Number.isSafeInteger(turn) && turn > prior.turn && at > cancellation.at\n && ctx.samples === 0 && ctx.assistantTranscriptEvents.length === 0\n && ctx.tools.length === 0 && ctx.toolResults.length === 0) {\n cancellation.admitted = true;\n ctx.unheardSupersededTurns.push({ turn: prior.turn, beginAtMs: prior.atMs,\n cancelledAtMs: cancellation.atMs, cancellationEventIndex: cancellation.eventIndex,\n replacementTurn: turn, outputPacketsBeforeReplacement: 0 });\n }\n ctx.turnBeginAt ??= at;\n ctx.latestTurnBeginAt = at;\n // Earlier PCM can be a listening cue. A new reply is still pending even\n // when that cue has already finished playing; retain all of its PCM.\n ctx.postTurnBeginFirstPacketAt = null;\n ctx.postTurnBeginFirstAudioAt = null;\n ctx.turnBegins.push({ atMs: Math.round(at - origin), turn: turn ?? null });\n}\n\nexport function recordUnheardCancellation(ctx, event, at, origin, eventIndex) {\n const prior = ctx.turnBegins.at(-1);\n if (event.event !== \"flush\" || event.reason !== \"turn_merge\"\n || !Number.isSafeInteger(event.cancelled_turn) || event.cancelled_turn <= 0\n || prior?.turn !== event.cancelled_turn || at < ctx.latestTurnBeginAt\n || ctx.samples !== 0 || ctx.assistantTranscriptEvents.length\n || ctx.tools.length || ctx.toolResults.length\n || ctx.unheardCancellations.some(c => c.turnBeginIndex === ctx.turnBegins.length - 1)) return;\n ctx.unheardCancellations.push({ turn: event.cancelled_turn, at,\n atMs: Math.round(at - origin), eventIndex, turnBeginIndex: ctx.turnBegins.length - 1,\n admitted: false });\n}\n\nexport function recordAudio(ctx, pcm, at, rate) {\n if (!(pcm instanceof Int16Array) || !pcm.length) return null;\n const copy = pcm.slice(); // Some transports reuse callback buffers.\n const playbackAt = Math.max(at, ctx.playbackEndAt ?? at);\n const endAt = playbackAt + (copy.length / rate) * 1000;\n const bounds = audibleBounds(copy, rate);\n ctx.firstPacketAt ??= at;\n ctx.lastPacketAt = at;\n ctx.playbackEndAt = endAt;\n const afterTurnBegin = ctx.latestTurnBeginAt != null && at >= ctx.latestTurnBeginAt;\n if (afterTurnBegin) ctx.postTurnBeginFirstPacketAt ??= at;\n if (bounds.first != null) {\n ctx.firstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n ctx.lastAudioAt = playbackAt + (bounds.last / rate) * 1000;\n if (afterTurnBegin) ctx.postTurnBeginFirstAudioAt ??= playbackAt + (bounds.first / rate) * 1000;\n }\n ctx.samples += copy.length;\n ctx.pcm.push(copy);\n return { playbackAt, endAt, pcm: copy };\n}\n\n/** Preserve negative gaps: they are overlaps, never missing measurements. */\nexport function captureTiming(ctx, caller, origin = 0) {\n const relative = (at) => at == null ? null : Math.round(at - origin);\n const gap = (at) => at == null || caller.speechOffsetAt == null\n ? null : Math.round(at - caller.speechOffsetAt);\n return {\n ttfbMs: gap(ctx.firstAudioAt),\n anyAudioTtfbMs: gap(ctx.firstAudioAt),\n postTurnBeginTtfbMs: gap(ctx.postTurnBeginFirstAudioAt),\n turnMs: gap(ctx.lastAudioAt),\n packetTtfbMs: gap(ctx.firstPacketAt),\n callerStartMs: relative(caller.startedAt),\n callerSpeechOnsetMs: relative(caller.speechOnsetAt),\n callerSpeechOffsetMs: relative(caller.speechOffsetAt),\n callerStreamEndMs: relative(caller.streamEndAt),\n agentFirstPacketMs: relative(ctx.firstPacketAt),\n agentLastPacketMs: relative(ctx.lastPacketAt),\n agentSpeechOnsetMs: relative(ctx.firstAudioAt),\n agentSpeechOffsetMs: relative(ctx.lastAudioAt),\n agentPlaybackEndMs: relative(ctx.playbackEndAt),\n latestTurnBeginMs: relative(ctx.latestTurnBeginAt),\n postTurnBeginFirstPacketMs: relative(ctx.postTurnBeginFirstPacketAt),\n postTurnBeginSpeechOnsetMs: relative(ctx.postTurnBeginFirstAudioAt),\n lastAssistantTranscriptMs: relative(ctx.lastAssistantTranscriptAt),\n callerMaxFrameGapMs: Math.round(caller.maxFrameGapMs ?? 0),\n method: \"client-monotonic-queued-playback-energy-bounds\",\n };\n}\n\nexport async function streamPcmRealtime(omni, pcm, rate, clock, onFrame = () => {}) {\n const frameSize = Math.max(1, Math.round((rate * FRAME_MS) / 1000));\n const bounds = audibleBounds(pcm, rate);\n const startedAt = clock.now();\n let speechOnsetAt = null;\n let speechOffsetAt = null;\n let previousFrameEnd = startedAt;\n let maxFrameGapMs = 0;\n for (let off = 0; off < pcm.length; off += frameSize) {\n const frame = pcm.subarray(off, Math.min(off + frameSize, pcm.length));\n const at = clock.now();\n maxFrameGapMs = Math.max(maxFrameGapMs, at - previousFrameEnd);\n const durationMs = (frame.length / rate) * 1000;\n omni.sendAudio(frame);\n onFrame(frame, at);\n if (bounds.first != null && bounds.first >= off && bounds.first < off + frame.length) {\n speechOnsetAt = at + ((bounds.first - off) / rate) * 1000;\n }\n if (bounds.last != null && bounds.last > off && bounds.last <= off + frame.length) {\n speechOffsetAt = at + ((bounds.last - off) / rate) * 1000;\n }\n previousFrameEnd = at + durationMs;\n // Pace from the actual send. Never burst late frames to \"catch up\".\n await clock.sleep(Math.max(0, previousFrameEnd - clock.now()));\n }\n return { startedAt, speechOnsetAt, speechOffsetAt,\n streamEndAt: clock.now(), maxFrameGapMs };\n}\n\nexport async function streamSilenceWhile(omni, rate, shouldContinue, clock, onFrame = () => {}) {\n const silence = new Int16Array(Math.max(1, Math.round((rate * FRAME_MS) / 1000)));\n while (shouldContinue()) {\n const at = clock.now();\n omni.sendAudio(silence);\n onFrame(silence, at);\n await clock.sleep(FRAME_MS);\n }\n}\n\nexport async function waitForAgentSettle(getCtx, clock, opts = {}) {\n const started = clock.now();\n const observation = opts.callerStreamEndAt != null;\n if (observation && (!Number.isFinite(opts.callerStreamEndAt)\n || opts.callerStreamEndAt < 0 || opts.callerStreamEndAt > started)) {\n throw new Error(\"Invalid caller observation boundary\");\n }\n const timeoutMs = observation ? INTERRUPTION_OBSERVATION.maximumAfterCallerStreamMs : opts.timeoutMs ?? 25000;\n const settleMs = observation ? INTERRUPTION_OBSERVATION.quietMs : opts.settleMs ?? 2000;\n const deadlineAt = (observation ? opts.callerStreamEndAt : started) + timeoutMs;\n const minimumAt = observation ? opts.callerStreamEndAt + INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : -Infinity;\n while (true) {\n const ctx = getCtx();\n const at = clock.now();\n if (opts.isClosed?.()) return { reason: \"closed\", at };\n if (observation && at >= deadlineAt) return { reason: \"timeout\", at };\n const hasResponseAudio = !opts.requireTurnBegin || (ctx.latestTurnBeginAt != null\n && ctx.postTurnBeginFirstAudioAt != null);\n // Synthesis text is progress, not a completion event. It can arrive after\n // early/final audio, so extend the quiet floor without demanding another\n // packet after every advisory. The protocol still has no reply-end marker.\n const activityEnd = Math.max(ctx.playbackEndAt ?? -Infinity,\n ctx.latestTurnBeginAt ?? -Infinity, ctx.lastAssistantTranscriptAt ?? -Infinity,\n observation ? ctx.lastCallerTranscriptAt ?? -Infinity : -Infinity);\n // Do not demand a new post-caller turn_begin: an earlier turn can be\n // coalesced and speak later. This records a bounded quiet observation;\n // downstream response-timing checks can impose a stricter begin boundary.\n if (ctx.firstPacketAt != null && hasResponseAudio && at >= minimumAt && at >= activityEnd + settleMs) {\n return { reason: \"settled\", at };\n }\n if (opts.allowEmpty && ctx.firstPacketAt == null && at >= minimumAt && at - started >= (opts.emptyWaitMs ?? 800)) {\n return { reason: \"empty\", at };\n }\n if (at - started >= timeoutMs) return { reason: \"timeout\", at };\n await clock.sleep(observation ? Math.min(20, deadlineAt - at) : 20);\n }\n}\n\nexport async function withTimeout(promise, ms, message, clock) {\n let timer;\n try {\n return await Promise.race([\n promise,\n new Promise((_, reject) => {\n timer = clock.setTimeout(() => reject(new Error(message)), ms);\n }),\n ]);\n } finally {\n clock.clearTimeout(timer);\n }\n}\n",
|
|
6
6
|
"main.mjs": "import PyAI from \"@pyai/sdk\";\nimport WebSocket from \"ws\"; // Explicit transport makes this work on Node 20 too.\nimport { readFile, writeFile, mkdtemp } from \"node:fs/promises\";\nimport { join } from \"node:path\";\nimport { runSession } from \"./session.mjs\";\nimport { readWav, writeWav, assessAnswer } from \"./audio.mjs\";\n\nasync function main() {\n const [callerPath, interruptionPath, extra] = process.argv.slice(2);\n if (!callerPath || extra) throw new Error(\"Usage: node main.mjs caller.wav [interruption.wav]\");\n if (!process.env.PYAI_API_KEY) throw new Error(\"Inject PYAI_API_KEY through your environment first\");\n const caller = readWav(await readFile(callerPath));\n const interruption = interruptionPath ? readWav(await readFile(interruptionPath)) : undefined;\n const pyai = new PyAI({ apiKey: process.env.PYAI_API_KEY, maxRetries: 0,\n fetch: (url, options) => fetch(url, { ...options, signal: AbortSignal.timeout(30000) }),\n });\n const { report, audio, replyAudio } = await runSession({ pyai, webSocket: WebSocket, caller, interruption });\n const directory = await mkdtemp(join(process.cwd(), \"omni-run-\"));\n const save = (name, data) => writeFile(join(directory, name), data, { mode: 0o600, flag: \"wx\" });\n await save(\"session.wav\", writeWav(audio, report.output_rate ?? 24000));\n const reply = writeWav(replyAudio, report.output_rate ?? 24000);\n await save(\"reply.wav\", reply);\n // Hear the captured bytes, rather than trusting synthesis-advisory text.\n let capturedTranscript = \"\";\n if (report.reply_audio_bytes) {\n try {\n const heard = await pyai.audio.transcriptions.create({ file: new File([reply], \"reply.wav\", { type: \"audio/wav\" }), model: \"pyai-hear\" });\n capturedTranscript = heard.text;\n } catch { report.hear_error = \"Captured audio saved; Hear verification failed\"; }\n }\n const result = { ...report, captured_transcript: capturedTranscript, evidence: assessAnswer(report, capturedTranscript) };\n await save(\"report.json\", JSON.stringify(result, null, 2) + \"\\n\");\n console.log(JSON.stringify({ directory, ...result }, null, 2));\n if (result.evidence.captured_answer !== \"verified_by_hear\" || (interruption && result.evidence.interruption !== \"simulated_queue_cleared\")) process.exitCode = 1;\n}\n\nmain().catch(error => {\n const message = String(error.message).replaceAll(process.env.PYAI_API_KEY || \"\\0\", \"[REDACTED]\");\n console.error(JSON.stringify({ error: message, ...(error.omni_code ? { code: error.omni_code } : {}) })); process.exitCode = 1;\n});\n",
|
|
7
7
|
"session.mjs": "import { newAudioCapture, recordAudio, recordTurnBegin, streamPcmRealtime, INTERRUPTION_OBSERVATION } from \"./timing.mjs\";\n\nexport const realClock = {\n now: () => performance.now(),\n sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),\n};\n\n// Only recognized machine codes may leave the example. Error messages, call\n// IDs and arbitrary code values can contain credentials or customer content.\nconst publicErrorCodes = new Set([\n \"invalid_configure\", \"unsupported_tool_transport\", \"tool_configuration_locked\",\n \"media_dead\", \"unauthorized\", \"insufficient_scope\", \"rate_limit_exceeded\",\n \"concurrency_limit_exceeded\", \"daily_cap_exceeded\", \"credit_exhausted\",\n]);\nconst transportErrorCodes = new Set([\"ECONNRESET\", \"ECONNREFUSED\", \"ETIMEDOUT\", \"ENOTFOUND\", \"EAI_AGAIN\"]);\nexport function omniFailure(error) {\n const code = error instanceof Error\n ? (transportErrorCodes.has(error.code) ? error.code : \"transport_or_protocol_error\")\n : (publicErrorCodes.has(error?.code) ? error.code : \"unrecognized_server_error\");\n return Object.assign(new Error(`Omni failed (${code}); no automatic retry was attempted`), { omni_code: code });\n}\n\n/** One bounded, server-side example. Its playback queue is simulated. */\nexport async function runSession({ pyai, webSocket, caller, interruption, clock = realClock, timeoutMs = 60000 }) {\n const rate = 24000, frameSize = 480, silence = new Int16Array(frameSize);\n if (!(caller instanceof Int16Array) || !caller.length || caller.length > rate * 20) throw new Error(\"Provide 1–20 seconds of caller PCM16 at 24 kHz\");\n if (interruption && (!(interruption instanceof Int16Array) || !interruption.length || interruption.length > rate * 20)) throw new Error(\"Invalid interruption PCM\");\n const started = clock.now(), all = newAudioCapture(started), reply = newAudioCapture(started);\n const report = { input_rate: rate, output_rate: null, playback_sink: \"simulated\", tool_calls: 0, tool_executions: 0, duplicate_tool_calls: 0, flushes_after_interruption: 0, cleared_queue_ms: 0, interruption_requested: !!interruption, caller_started_ms: null, interruption_started_ms: null, advisory_transcripts: [], end_reason: \"timeout\" };\n let session, configuredAt = null, closed = false, failure = null, phase = \"greeting\", offset = 0, queueEnd = started, interruptionSent = false, inputEndAt = started;\n const handled = new Set();\n const fail = message => { failure ??= new Error(message); };\n try {\n session = pyai.omni.connect({\n webSocket, rate,\n configure: {\n voice_id: \"stock_amos_en_us\", language: \"en\", greeting: \"Hello. How can I help?\",\n persona: \"When asked about office hours, call lookup_office_hours. Never guess. After the result, say one short sentence giving its opening_time.\",\n tools: [{ name: \"lookup_office_hours\", description: \"Read the synthetic example office opening time.\", side_effect: \"read\", parameters: { type: \"object\", properties: {}, required: [], additionalProperties: false } }],\n },\n onHello: frame => {\n const match = /^pcm16@(8000|24000)$/.exec(frame.audio_out ?? \"\");\n if (!match) return fail(\"Unsupported Omni output format\");\n report.output_rate = Number(match[1]);\n },\n onConfigured: () => { configuredAt ??= clock.now(); },\n onAudio: chunk => {\n if (!report.output_rate) return fail(\"Audio arrived before its output format\");\n const bytes = chunk instanceof ArrayBuffer ? new Uint8Array(chunk) : new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength);\n if (bytes.byteLength % 2) return fail(\"Invalid PCM16 output\");\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);\n const pcm = Int16Array.from({ length: bytes.length / 2 }, (_, i) => view.getInt16(i * 2, true));\n const now = clock.now();\n recordAudio(all, pcm, now, report.output_rate);\n queueEnd = Math.max(now, queueEnd) + pcm.length / report.output_rate * 1000;\n if (report.tool_executions) recordAudio(reply, pcm, now, report.output_rate);\n },\n onEvent: frame => {\n if (frame.event === \"turn_begin\") recordTurnBegin(all, clock.now(), frame.turn);\n },\n onTranscript: frame => {\n // An assistant advisory is progress; never treat it as received speech.\n if (report.advisory_transcripts.length < 100) report.advisory_transcripts.push({ role: frame.role, text: frame.text });\n if (frame.role === \"assistant\") all.lastAssistantTranscriptAt = clock.now();\n },\n onToolCall: frame => {\n report.tool_calls++;\n if (handled.has(frame.call_id)) { report.duplicate_tool_calls++; return; }\n handled.add(frame.call_id);\n if (frame.name !== \"lookup_office_hours\") { session.toolResult(frame.call_id, { error: \"Unknown example tool\" }); return; }\n // Replace this read-only fixture with your authorized data lookup.\n report.tool_executions++;\n session.toolResult(frame.call_id, { result: { opening_time: \"9 a.m.\", source: \"synthetic example office\" } });\n },\n onBargeIn: () => {\n const now = clock.now(), queued = Math.max(0, queueEnd - now);\n if (report.interruption_started_ms != null) { report.flushes_after_interruption++; report.cleared_queue_ms += queued; }\n // A real speaker adapter must cancel scheduled and currently playing audio.\n queueEnd = now; all.playbackEndAt = now; reply.playbackEndAt = now;\n },\n onError: error => { failure ??= omniFailure(error); },\n onClose: () => { closed = true; },\n });\n while (clock.now() - started < timeoutMs) {\n if (failure) throw failure;\n if (closed) { report.end_reason = \"closed\"; break; }\n const now = clock.now();\n if (configuredAt == null || !report.output_rate) {\n if (now - started >= 10000) throw new Error(\"Omni connection/configuration timed out\");\n await clock.sleep(20); continue;\n }\n const quietAt = Math.max(queueEnd, all.lastPacketAt ?? now, all.lastAssistantTranscriptAt ?? 0, inputEndAt);\n if (phase === \"greeting\") {\n if (all.firstPacketAt != null && now - configuredAt >= 2000 && now >= quietAt + 800) {\n phase = \"caller\"; report.caller_started_ms = now - started;\n } else if (now - configuredAt > 15000) throw new Error(\"Greeting did not drain within 15 seconds\");\n }\n if (phase === \"reply\" && interruption && !interruptionSent && reply.samples && queueEnd - now >= 200) {\n phase = \"interruption\"; offset = 0; interruptionSent = true; report.interruption_started_ms = now - started;\n }\n const observationEnd = inputEndAt + (interruptionSent ? INTERRUPTION_OBSERVATION.minimumAfterCallerStreamMs : 0);\n if (phase === \"reply\" && reply.samples && now >= quietAt + 2000 && now >= observationEnd && (!interruption || interruptionSent)) {\n report.end_reason = \"quiet_window\"; break;\n }\n // Exactly one producer: each 20 ms slot contains caller PCM OR silence.\n let pcm = silence;\n if (phase === \"caller\" || phase === \"interruption\") {\n const source = phase === \"caller\" ? caller : interruption;\n pcm = source.subarray(offset, offset + frameSize); offset += pcm.length;\n if (offset >= source.length) {\n inputEndAt = now + pcm.length / rate * 1000;\n report[phase === \"caller\" ? \"caller_completed_ms\" : \"interruption_completed_ms\"] = inputEndAt - started;\n phase = \"reply\"; offset = 0;\n }\n }\n await streamPcmRealtime(session, pcm, rate, clock);\n }\n report.audio_bytes = all.samples * 2;\n report.reply_audio_bytes = reply.samples * 2;\n report.elapsed_ms = Math.round(clock.now() - started);\n return { report, audio: all.pcm, replyAudio: reply.pcm };\n } finally { session?.close(); }\n}\n",
|
package/src/cli-web-auth.ts
CHANGED
|
@@ -17,6 +17,8 @@ export interface BrowserCredential {
|
|
|
17
17
|
key_id: string;
|
|
18
18
|
org_id: string;
|
|
19
19
|
project_id: string;
|
|
20
|
+
/** Verified approving human, when returned by the current device flow. */
|
|
21
|
+
user_id?: string;
|
|
20
22
|
environment: "live" | "test";
|
|
21
23
|
scopes: string[];
|
|
22
24
|
expires_at: number;
|
|
@@ -132,6 +134,7 @@ export async function browserLogin(options: BrowserLoginOptions): Promise<Browse
|
|
|
132
134
|
if (!["key_id", "org_id", "project_id"].every(key => typeof result[key] === "string" && result[key].length > 0)
|
|
133
135
|
|| !["test", "live"].includes(result.environment) || !Array.isArray(result.scopes) || !result.scopes.every((s: unknown) => typeof s === "string")
|
|
134
136
|
|| typeof result.expires_at !== "number" || !Number.isFinite(result.expires_at) || result.expires_at <= now()) invalidResponse();
|
|
137
|
+
if (result.user_id !== undefined && (typeof result.user_id !== "string" || !result.user_id.trim() || result.user_id.length > 200 || /[\x00-\x1f\x7f]/.test(result.user_id))) throw new CliError("invalid_response", "The login response contained an invalid user identity.", 5);
|
|
135
138
|
return result as BrowserCredential;
|
|
136
139
|
} catch (error) {
|
|
137
140
|
if (!(error instanceof CliError)) throw error;
|
package/src/cli.ts
CHANGED
|
@@ -46,7 +46,7 @@ const commands: Command[] = [
|
|
|
46
46
|
{ command: "init", description: "Create a new project with API context and request examples offline", args: ["directory"], flags: { template: "string" }, example: "pyai init voice-demo --template typescript" },
|
|
47
47
|
{ command: "smoke", description: "Check models, voices, and speech synthesis", flags: { "tolerate-upstream": "boolean" } },
|
|
48
48
|
{ command: "doctor", description: "Diagnose key, catalogs, and a Speak to Hear round-trip" },
|
|
49
|
-
{ command: "dialer", description: "Open the private desktop calling and meeting-recording app", flags: { port: "string", "no-browser": "boolean", name: "string", "white-label": "boolean" }, example: "pyai dialer --profile production" },
|
|
49
|
+
{ command: "dialer", description: "Open the private desktop calling and meeting-recording app", flags: { port: "string", "no-browser": "boolean", name: "string", "white-label": "boolean", transfers: "boolean" }, example: "pyai dialer --profile production" },
|
|
50
50
|
...routes.map((r): Command => ({ command: r.command, description: r.description, args: r.id ? ["id"] : [], flags: {
|
|
51
51
|
...(r.body ? bodyFlags : {}), ...(r.binary ? outputFlags : {}), ...(r.wait ? waitFlags : {}),
|
|
52
52
|
...(r.query ?? {}), ...(r.confirm ? { confirm: "boolean" as const } : {}),
|
|
@@ -342,7 +342,7 @@ async function main(): Promise<void> {
|
|
|
342
342
|
const { startDialerApp } = await import("./dialer/appServer.ts");
|
|
343
343
|
const fingerprint = createHash("sha256").update(config.baseURL).update("\0").update(config.apiKey).digest("hex");
|
|
344
344
|
const sdk = new PyAI({ apiKey: config.apiKey, baseURL: config.baseURL, fetch: (url, init) => fetch(url, { ...init, redirect: "error", signal: init?.signal ?? AbortSignal.timeout(15000) }) });
|
|
345
|
-
const app = await startDialerApp({ sdk, userId: `desktop_${fingerprint.slice(0,40)}`, scope: `desktop:${fingerprint}`, port, name: value(flags, "name"), whiteLabel: flags["white-label"] === true });
|
|
345
|
+
const app = await startDialerApp({ sdk, userId: `desktop_${fingerprint.slice(0,40)}`, scope: `desktop:${fingerprint}`, port, name: value(flags, "name"), whiteLabel: flags["white-label"] === true, allowTransfers: flags.transfers === true });
|
|
346
346
|
output({ url: app.url, local_only: true }, flags, `PyAI Phone is ready at ${app.url}\nKeep this window open. Press Ctrl+C to stop the app and end its owned calls.`);
|
|
347
347
|
if (flags["no-browser"] || !(await openBrowser(app.launchUrl))) process.stderr.write(`Private one-use launch link (valid for 60 seconds): ${app.launchUrl}\n`);
|
|
348
348
|
await new Promise<void>(resolve => {
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# Meeting transcripts, notes and action items
|
|
2
|
+
|
|
3
|
+
The CLI and desktop meeting app now include **Send for transcription** beside
|
|
4
|
+
saved recordings. Capture stays in the browser until the user accepts the
|
|
5
|
+
upload/retention notice and chooses Send. The app uses its selected existing
|
|
6
|
+
project key, the normal private Hear multipart job upload, and existing
|
|
7
|
+
Hear-to-Recap delivery. It does not enable Recap, grant scopes, select a parent
|
|
8
|
+
credential, or upload automatically when recording stops.
|
|
9
|
+
|
|
10
|
+
The host verifies organization/key identity and requires `hear:transcribe`,
|
|
11
|
+
`transcribe:jobs`, `recap:read`, `recap:configure` and enabled Recap. Managed calling
|
|
12
|
+
credentials with only Omni/telephony scopes display an unavailable state with
|
|
13
|
+
the missing access. Choose an already permitted project, or complete the normal
|
|
14
|
+
explicitly authorized feature grant. No operator credential fallback exists.
|
|
15
|
+
|
|
16
|
+
`startDialerApp` includes authenticated same-origin meeting routes, the visible
|
|
17
|
+
browser controls, and a private metadata journal by default. No external asset
|
|
18
|
+
adapter or cloud bucket setup is needed. Project credentials remain in the host.
|
|
19
|
+
The renderer receives only its existing private local session. Uploads require
|
|
20
|
+
that session, the current local nonce, an exact trusted Origin, the accepted
|
|
21
|
+
notice, a supported audio type, and a maximum of 128 MiB. No public route accepts
|
|
22
|
+
an organization, user, job ID, Recap call ID, or audio source URL from the browser.
|
|
23
|
+
|
|
24
|
+
The journal contains an opaque owner/recording key, content digest, job ID and
|
|
25
|
+
cached output. It never stores audio or API keys. It uses a private per-scope
|
|
26
|
+
directory, private files, atomic replacement and durable saves, plus a filesystem
|
|
27
|
+
lease across host instances. A removed entry cannot be resurrected. A crashed
|
|
28
|
+
writer with a recorded dead process can be recovered; an interrupted lease
|
|
29
|
+
recovery fails closed rather than guessing at ownership. The default metadata
|
|
30
|
+
root is the user's `.config/pyai/meetings`; a trusted host can select
|
|
31
|
+
`meetingJournalDirectory`. The normal startup and hourly sweep clear expired
|
|
32
|
+
cached text; the default cache lifetime is 24 hours. Opaque submission tombstones
|
|
33
|
+
remain to prevent unintended re-upload. Windows inherits the private profile's
|
|
34
|
+
access controls; Unix additionally enforces owner-only directory/file modes.
|
|
35
|
+
|
|
36
|
+
The public job API now supports optional Idempotency-Key on both JSON and
|
|
37
|
+
multipart submissions. Its deterministic job identity is scoped to the
|
|
38
|
+
organization and authenticated project. The stored fingerprint binds normalized
|
|
39
|
+
caller options, uploaded bytes and original upload filename. The database
|
|
40
|
+
primary key selects the durable winner even across processes or a lost create
|
|
41
|
+
acknowledgement. Each candidate upload has a content-addressed name, and the
|
|
42
|
+
worker reads the winner's exact file. A losing candidate cannot overwrite its
|
|
43
|
+
audio. Original billing/key/environment and resolved vocabulary snapshots stay
|
|
44
|
+
unchanged. Known jobs recover before new funding/add-on gates; new work still
|
|
45
|
+
passes them. Changed explicit input returns 409. Unkeyed uploads and existing
|
|
46
|
+
GPT-Live deduplication remain compatible.
|
|
47
|
+
|
|
48
|
+
A lost submission response is recovered by sending the **same saved recording**
|
|
49
|
+
with the same owner/recording identity. Do not create a new recording ID after an
|
|
50
|
+
uncertain response. Refresh checks the existing job once, then displays the
|
|
51
|
+
Hear transcript and Recap notes/actions. It does not fetch arbitrary result URLs
|
|
52
|
+
or automatically resubmit failed work. Recap can provide the retained transcript
|
|
53
|
+
when Hear offloads its result. The UI renders output as text nodes.
|
|
54
|
+
|
|
55
|
+
**Retention is explicit.** Uploaded audio may remain in PyAI for up to seven days
|
|
56
|
+
and job result artifacts up to 30 days. Recap records follow workspace retention.
|
|
57
|
+
Removing the meeting from the app clears cached notes, preserves a tombstone and
|
|
58
|
+
requests cancellation of a known unfinished job. Existing DELETE is not a cloud
|
|
59
|
+
erasure API. An uncertain first submission can leave an unknown existing job;
|
|
60
|
+
removal says so and never replays a POST. The browser's saved recording remains
|
|
61
|
+
available. Do not enable submission for a policy requiring immediate cloud
|
|
62
|
+
input/result/Recap erasure until the normal erasure mechanism supports it.
|
|
63
|
+
|
|
64
|
+
The desktop close lifecycle uses `pyai-desktop-prepare-close` and a matching
|
|
65
|
+
`pyai-desktop-close-result` ticket. The app pauses the queue, finalizes capture
|
|
66
|
+
and in-flight journal writes, and stops/reconciles known owned calls. An unknown
|
|
67
|
+
admission returns `ready: false` without replaying a create. Transferred calls
|
|
68
|
+
continue independently. The native host retains credentials when recovery is
|
|
69
|
+
still needed.
|
|
70
|
+
|
|
71
|
+
`softphone-desktop/test-browser/run-meetings.mjs` proves the assembled app UI,
|
|
72
|
+
file journal, SDK upload, actual API and worker, Recap ingest/callback/read,
|
|
73
|
+
lost-response recovery, reload, readable transcript/notes/actions, removal and
|
|
74
|
+
close lifecycle. It uses a tiny synthetic WAV and controlled transcription and
|
|
75
|
+
summary outputs. This proves integration, not ASR or summary quality. It uses
|
|
76
|
+
no customer audio, public model, carrier call or real account credential.
|
package/src/dialer/app/app.html
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><meta name="theme-color" content="#5b55e7"><title>PyAI Phone</title><link rel="icon" href="/icon.svg"><link rel="manifest" href="/manifest.webmanifest"><link rel="stylesheet" href="/app.css"><script type="module" src="/app.js"></script></head><body>
|
|
2
2
|
<aside><a class="brand" href="/" aria-label="PyAI Phone"><span class="logo">p</span> <span data-brand>PyAI Phone</span></a><nav aria-label="Workspace"><button class="selected" data-tab="calls">Phone <span>↗</span></button><button data-tab="meetings">Meeting recorder <span>◉</span></button></nav><div class="aside-foot"><span class="online"></span> Private workspace<br><small>Powered by PyAI</small></div></aside>
|
|
3
3
|
<main><header><div><p class="eyebrow">YOUR CONVERSATIONS, TOGETHER</p><h1 data-heading>Make the next connection.</h1><p class="subtitle" data-subtitle>A clear line. A little less busywork.</p></div><span class="local-badge">On this device</span></header>
|
|
4
|
-
<div id="error" role="alert"></div><button id="retry" hidden>Retry connection</button><section data-panel="calls"><div class="calling-grid"><div><label class="source-label" for="source">Calling from</label><select id="source" disabled><option>Loading workspace numbers…</option></select><label class="recording-choice"><input id="record-calls" type="checkbox">Record calls · tell the other person before recording</label><pyai-dialer></pyai-dialer></div>
|
|
5
|
-
<section class="queue-card"><div class="card-top"><span class="eyebrow">STAY IN YOUR FLOW</span><span class="quiet-icon">≋</span></div><h2>One good conversation<br>after another.</h2><p>Paste your contacts and move through them at your own pace. Each call waits for your wrap-up.</p><label for="contacts">Phone numbers, one per line</label><textarea id="contacts" placeholder="+61 2 5550 0100 +1 415 555 0100" spellcheck="false"></textarea><label for="mode">Calling order</label><select id="mode"><option value="auto">In list order</option><option value="dynamic">Highest priority first · number, priority</option><option value="parallel">
|
|
6
|
-
<section data-panel="meetings" hidden><div class="meeting-grid"><div class="meeting-copy"><span class="eyebrow">LESS DISTRACTION, MORE PRESENCE</span><h2>Stay in the<br>conversation.</h2><p>Choose the audio you want to keep. Record your meeting and microphone together, pause when you need to, and save a private audio file.</p><div class="feature"><b>◉</b><span><strong>Your choice of audio</strong>Choose a supported meeting tab or system source in the browser picker.</span></div><div class="feature"><b>▦</b><span><strong>Saved as you go</strong>Committed audio can be recovered after an interrupted tab.</span></div><div class="feature"><b>↧</b><span><strong>Yours to keep</strong>Download your recording. No automatic uploads or hidden capture.</span></div></div><pyai-meeting-recorder></pyai-meeting-recorder></div></section>
|
|
4
|
+
<div id="error" role="alert"></div><button id="retry" hidden>Retry connection</button><section data-panel="calls"><div class="calling-grid"><div><label class="source-label" for="source">Calling from</label><select id="source" disabled><option>Loading workspace numbers…</option></select><label class="recording-choice"><input id="record-calls" type="checkbox">Record calls · tell the other person before recording</label><label class="recording-choice" id="transfer-choice" hidden><input id="allow-transfers" type="checkbox">Allow transfers · single-line calls</label><pyai-dialer></pyai-dialer></div>
|
|
5
|
+
<section class="queue-card"><div class="card-top"><span class="eyebrow">STAY IN YOUR FLOW</span><span class="quiet-icon">≋</span></div><h2>One good conversation<br>after another.</h2><p>Paste your contacts and move through them at your own pace. Each call waits for your wrap-up.</p><label for="contacts">Phone numbers, one per line</label><textarea id="contacts" placeholder="+61 2 5550 0100 +1 415 555 0100" spellcheck="false"></textarea><label for="mode">Calling order</label><select id="mode"><option value="auto">In list order</option><option value="dynamic">Highest priority first · number, priority</option><option value="parallel">Parallel lines · first answer connects</option></select><div id="parallel-choice" hidden><label for="parallel-lines">Parallel lines</label><select id="parallel-lines"><option value="2">2 lines</option><option value="3">3 lines</option><option value="4">4 lines</option></select></div><div class="queue-actions"><button id="queue-start" class="primary" disabled>Start calling</button><button id="queue-pause" disabled>Pause queue</button><button id="queue-resume" disabled>Resume queue</button><button id="queue-clear" disabled>Clear queue</button></div><p id="queue-status" role="status" aria-live="polite">You choose when to start. Your workspace calling rules apply to every call.</p></section></div></section>
|
|
6
|
+
<section data-panel="meetings" hidden><div class="meeting-grid"><div class="meeting-copy"><span class="eyebrow">LESS DISTRACTION, MORE PRESENCE</span><h2>Stay in the<br>conversation.</h2><p>Choose the audio you want to keep. Record your meeting and microphone together, pause when you need to, and save a private audio file. When you choose, send it for a transcript, notes and action items.</p><div class="feature"><b>◉</b><span><strong>Your choice of audio</strong>Choose a supported meeting tab or system source in the browser picker.</span></div><div class="feature"><b>▦</b><span><strong>Saved as you go</strong>Committed audio can be recovered after an interrupted tab.</span></div><div class="feature"><b>↧</b><span><strong>Yours to keep</strong>Download your recording. No automatic uploads or hidden capture.</span></div></div><pyai-meeting-recorder></pyai-meeting-recorder><pyai-meeting-intelligence></pyai-meeting-intelligence></div></section>
|
|
7
7
|
<footer>Call history and call recordings are stored in your PyAI workspace. Meeting audio stays in this browser until you save or submit it.</footer></main></body></html>
|