@talqing/client 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +121 -0
- package/dist/cjs/images.d.ts +62 -0
- package/dist/cjs/images.d.ts.map +1 -0
- package/dist/cjs/images.js +137 -0
- package/dist/cjs/images.js.map +1 -0
- package/dist/cjs/index.d.ts +10 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +37 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/json.d.ts +8 -0
- package/dist/cjs/json.d.ts.map +1 -0
- package/dist/cjs/json.js +3 -0
- package/dist/cjs/json.js.map +1 -0
- package/dist/cjs/media.d.ts +9 -0
- package/dist/cjs/media.d.ts.map +1 -0
- package/dist/cjs/media.js +53 -0
- package/dist/cjs/media.js.map +1 -0
- package/dist/cjs/package.json +3 -0
- package/dist/cjs/rpc.d.ts +25 -0
- package/dist/cjs/rpc.d.ts.map +1 -0
- package/dist/cjs/rpc.js +48 -0
- package/dist/cjs/rpc.js.map +1 -0
- package/dist/cjs/token.d.ts +29 -0
- package/dist/cjs/token.d.ts.map +1 -0
- package/dist/cjs/token.js +40 -0
- package/dist/cjs/token.js.map +1 -0
- package/dist/cjs/userdata.d.ts +22 -0
- package/dist/cjs/userdata.d.ts.map +1 -0
- package/dist/cjs/userdata.js +51 -0
- package/dist/cjs/userdata.js.map +1 -0
- package/dist/esm/images.d.ts +62 -0
- package/dist/esm/images.d.ts.map +1 -0
- package/dist/esm/images.js +129 -0
- package/dist/esm/images.js.map +1 -0
- package/dist/esm/index.d.ts +10 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +16 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/json.d.ts +8 -0
- package/dist/esm/json.d.ts.map +1 -0
- package/dist/esm/json.js +2 -0
- package/dist/esm/json.js.map +1 -0
- package/dist/esm/media.d.ts +9 -0
- package/dist/esm/media.d.ts.map +1 -0
- package/dist/esm/media.js +50 -0
- package/dist/esm/media.js.map +1 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/rpc.d.ts +25 -0
- package/dist/esm/rpc.d.ts.map +1 -0
- package/dist/esm/rpc.js +43 -0
- package/dist/esm/rpc.js.map +1 -0
- package/dist/esm/token.d.ts +29 -0
- package/dist/esm/token.d.ts.map +1 -0
- package/dist/esm/token.js +37 -0
- package/dist/esm/token.js.map +1 -0
- package/dist/esm/userdata.d.ts +22 -0
- package/dist/esm/userdata.d.ts.map +1 -0
- package/dist/esm/userdata.js +46 -0
- package/dist/esm/userdata.js.map +1 -0
- package/package.json +62 -0
- package/src/images.ts +160 -0
- package/src/index.ts +48 -0
- package/src/json.ts +17 -0
- package/src/media.ts +53 -0
- package/src/rpc.ts +81 -0
- package/src/token.ts +49 -0
- package/src/userdata.ts +57 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.createTalqingTokenSource = createTalqingTokenSource;
|
|
4
|
+
const livekit_client_1 = require("livekit-client");
|
|
5
|
+
/** Turn a one-shot mint into the token source LiveKit's session takes.
|
|
6
|
+
*
|
|
7
|
+
* `fetchToken` is called at most once per source, no matter how often LiveKit
|
|
8
|
+
* asks. That is load-bearing rather than an optimization: minting is not free,
|
|
9
|
+
* because `POST /v1/calls/token` files the session row the call is later read
|
|
10
|
+
* back from. The framework asks three times per call — to warm the connection,
|
|
11
|
+
* to connect, and once more as the call ends to drop what it assumes is a
|
|
12
|
+
* cached token — and only the middle one is a call anybody placed. Minting for
|
|
13
|
+
* the third filed a session nobody ever joined, which sat in the workspace's
|
|
14
|
+
* call list as queued for good.
|
|
15
|
+
*
|
|
16
|
+
* `TokenSource.literal` is the LiveKit variant that leaves caching to us, and
|
|
17
|
+
* this is us doing it.
|
|
18
|
+
*
|
|
19
|
+
* Freshness comes from a new source per call, never from re-minting inside
|
|
20
|
+
* one: the token names the room, and this call has exactly one. Mount the
|
|
21
|
+
* session per call and end it on unmount, as LiveKit's own session lifecycle
|
|
22
|
+
* does, and that holds by construction. */
|
|
23
|
+
function createTalqingTokenSource(fetchToken) {
|
|
24
|
+
let minted = null;
|
|
25
|
+
return livekit_client_1.TokenSource.literal(() => {
|
|
26
|
+
// A mint that failed is not this call's token, and the surface will offer
|
|
27
|
+
// the call again — so keep the rejection out of the cache.
|
|
28
|
+
minted ?? (minted = fetchToken()
|
|
29
|
+
.then((credentials) => ({
|
|
30
|
+
serverUrl: credentials.server_url,
|
|
31
|
+
participantToken: credentials.participant_token,
|
|
32
|
+
}))
|
|
33
|
+
.catch((error) => {
|
|
34
|
+
minted = null;
|
|
35
|
+
throw error;
|
|
36
|
+
}));
|
|
37
|
+
return minted;
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=token.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../src/token.ts"],"names":[],"mappings":";;AA6BA,4DAmBC;AAhDD,mDAAoE;AAWpE;;;;;;;;;;;;;;;;;4CAiB4C;AAC5C,SAAgB,wBAAwB,CACtC,UAAiD;IAEjD,IAAI,MAAM,GAAoE,IAAI,CAAC;IAEnF,OAAO,4BAAW,CAAC,OAAO,CAAC,GAAG,EAAE;QAC9B,0EAA0E;QAC1E,2DAA2D;QAC3D,MAAM,KAAN,MAAM,GAAK,UAAU,EAAE;aACpB,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACtB,SAAS,EAAE,WAAW,CAAC,UAAU;YACjC,gBAAgB,EAAE,WAAW,CAAC,iBAAiB;SAChD,CAAC,CAAC;aACF,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACxB,MAAM,GAAG,IAAI,CAAC;YACd,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,EAAC;QACL,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type RemoteParticipant, type Room } from "livekit-client";
|
|
2
|
+
import type { JsonObject } from "./json.js";
|
|
3
|
+
/** The agent on this call, or undefined before it has joined.
|
|
4
|
+
*
|
|
5
|
+
* On a video call the avatar worker is an agent participant too, but it joins
|
|
6
|
+
* on the agent's behalf and never answers RPCs, so the first one found is the
|
|
7
|
+
* one to talk to. */
|
|
8
|
+
export declare function talqingAgentParticipant(room: Room): RemoteParticipant | undefined;
|
|
9
|
+
/** Merge a patch into the live session's userdata, over RPC to the agent.
|
|
10
|
+
*
|
|
11
|
+
* Resolves rather than throws when the agent is not there: a write that did
|
|
12
|
+
* not land is a state a page can render. */
|
|
13
|
+
export declare function setTalqingUserdata(room: Room, patch: JsonObject, responseTimeout?: number): Promise<{
|
|
14
|
+
ok: boolean;
|
|
15
|
+
error?: string;
|
|
16
|
+
}>;
|
|
17
|
+
/** Read the live session's userdata, or only the keys named.
|
|
18
|
+
*
|
|
19
|
+
* Throws when the agent is not there, unlike `setTalqingUserdata`: a read that
|
|
20
|
+
* returned nothing is not a state anything can render. */
|
|
21
|
+
export declare function getTalqingUserdata(room: Room, keys?: string[], responseTimeout?: number): Promise<JsonObject>;
|
|
22
|
+
//# sourceMappingURL=userdata.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"userdata.d.ts","sourceRoot":"","sources":["../../src/userdata.ts"],"names":[],"mappings":"AAAA,OAAO,EAAmB,KAAK,iBAAiB,EAAE,KAAK,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACpF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;sBAIsB;AACtB,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,IAAI,GAAG,iBAAiB,GAAG,SAAS,CAIjF;AAED;;;6CAG6C;AAC7C,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,IAAI,EACV,KAAK,EAAE,UAAU,EACjB,eAAe,SAAI,GAClB,OAAO,CAAC;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAU1C;AAED;;;2DAG2D;AAC3D,wBAAsB,kBAAkB,CACtC,IAAI,EAAE,IAAI,EACV,IAAI,CAAC,EAAE,MAAM,EAAE,EACf,eAAe,SAAI,GAClB,OAAO,CAAC,UAAU,CAAC,CAcrB"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.talqingAgentParticipant = talqingAgentParticipant;
|
|
4
|
+
exports.setTalqingUserdata = setTalqingUserdata;
|
|
5
|
+
exports.getTalqingUserdata = getTalqingUserdata;
|
|
6
|
+
const livekit_client_1 = require("livekit-client");
|
|
7
|
+
/** The agent on this call, or undefined before it has joined.
|
|
8
|
+
*
|
|
9
|
+
* On a video call the avatar worker is an agent participant too, but it joins
|
|
10
|
+
* on the agent's behalf and never answers RPCs, so the first one found is the
|
|
11
|
+
* one to talk to. */
|
|
12
|
+
function talqingAgentParticipant(room) {
|
|
13
|
+
return [...room.remoteParticipants.values()].find((participant) => participant.kind === livekit_client_1.ParticipantKind.AGENT);
|
|
14
|
+
}
|
|
15
|
+
/** Merge a patch into the live session's userdata, over RPC to the agent.
|
|
16
|
+
*
|
|
17
|
+
* Resolves rather than throws when the agent is not there: a write that did
|
|
18
|
+
* not land is a state a page can render. */
|
|
19
|
+
async function setTalqingUserdata(room, patch, responseTimeout = 3) {
|
|
20
|
+
const agent = talqingAgentParticipant(room);
|
|
21
|
+
if (!agent)
|
|
22
|
+
return { ok: false, error: "agent participant is not connected" };
|
|
23
|
+
const response = await room.localParticipant.performRpc({
|
|
24
|
+
destinationIdentity: agent.identity,
|
|
25
|
+
method: "talqing.client.userdata_set",
|
|
26
|
+
payload: JSON.stringify(patch),
|
|
27
|
+
responseTimeout,
|
|
28
|
+
});
|
|
29
|
+
return JSON.parse(response || "{}");
|
|
30
|
+
}
|
|
31
|
+
/** Read the live session's userdata, or only the keys named.
|
|
32
|
+
*
|
|
33
|
+
* Throws when the agent is not there, unlike `setTalqingUserdata`: a read that
|
|
34
|
+
* returned nothing is not a state anything can render. */
|
|
35
|
+
async function getTalqingUserdata(room, keys, responseTimeout = 3) {
|
|
36
|
+
const agent = talqingAgentParticipant(room);
|
|
37
|
+
if (!agent)
|
|
38
|
+
throw new Error("agent participant is not connected");
|
|
39
|
+
const response = await room.localParticipant.performRpc({
|
|
40
|
+
destinationIdentity: agent.identity,
|
|
41
|
+
method: "talqing.client.userdata_get",
|
|
42
|
+
payload: JSON.stringify(keys ? { keys } : {}),
|
|
43
|
+
responseTimeout,
|
|
44
|
+
});
|
|
45
|
+
const parsed = JSON.parse(response || "{}");
|
|
46
|
+
if (parsed.ok === false) {
|
|
47
|
+
throw new Error(String(parsed.error || "userdata RPC failed"));
|
|
48
|
+
}
|
|
49
|
+
return parsed;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=userdata.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"userdata.js","sourceRoot":"","sources":["../../src/userdata.ts"],"names":[],"mappings":";;AAQA,0DAIC;AAMD,gDAcC;AAMD,gDAkBC;AAxDD,mDAAoF;AAGpF;;;;sBAIsB;AACtB,SAAgB,uBAAuB,CAAC,IAAU;IAChD,OAAO,CAAC,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAC/C,CAAC,WAA8B,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,KAAK,gCAAe,CAAC,KAAK,CAC/E,CAAC;AACJ,CAAC;AAED;;;6CAG6C;AACtC,KAAK,UAAU,kBAAkB,CACtC,IAAU,EACV,KAAiB,EACjB,eAAe,GAAG,CAAC;IAEnB,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,oCAAoC,EAAE,CAAC;IAC9E,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC;QACtD,mBAAmB,EAAE,KAAK,CAAC,QAAQ;QACnC,MAAM,EAAE,6BAA6B;QACrC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;QAC9B,eAAe;KAChB,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAoC,CAAC;AACzE,CAAC;AAED;;;2DAG2D;AACpD,KAAK,UAAU,kBAAkB,CACtC,IAAU,EACV,IAAe,EACf,eAAe,GAAG,CAAC;IAEnB,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC;QACtD,mBAAmB,EAAE,KAAK,CAAC,QAAQ;QACnC,MAAM,EAAE,6BAA6B;QACrC,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,eAAe;KAChB,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAkD,CAAC;IAC7F,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,IAAI,qBAAqB,CAAC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { Room } from "livekit-client";
|
|
2
|
+
export declare const TALQING_IMAGE_TOPIC = "talqing.images";
|
|
3
|
+
/** Decoded size the server refuses above. The downscale below keeps a phone
|
|
4
|
+
* photo far under it; this is the belt for a file that is not a photo. */
|
|
5
|
+
export declare const TALQING_IMAGE_MAX_BYTES: number;
|
|
6
|
+
/** Longest side the server stores. Downscaling to it before sending is a
|
|
7
|
+
* courtesy to the data channel, which is ordered and reliable and will
|
|
8
|
+
* happily spend a minute pushing 10 MB past a call's audio. */
|
|
9
|
+
export declare const TALQING_IMAGE_MAX_EDGE_PX = 1568;
|
|
10
|
+
/** How long to wait for the agent's ack before saying we do not know. Without
|
|
11
|
+
* it a thumbnail sits on "sending" for the rest of the call when the worker
|
|
12
|
+
* dies mid-upload. */
|
|
13
|
+
export declare const TALQING_IMAGE_ACK_TIMEOUT_MS = 30000;
|
|
14
|
+
/** Refuse what the browser can see is wrong, instantly and with no round trip.
|
|
15
|
+
* Returns the sentence to show, or null when the file is fine. The server
|
|
16
|
+
* re-validates everything regardless — this exists so the common mistakes cost
|
|
17
|
+
* nothing to catch. */
|
|
18
|
+
export declare function talqingImageRejection(file: File): string | null;
|
|
19
|
+
/** Downscale to the longest edge the server keeps, re-encoding as JPEG or PNG.
|
|
20
|
+
*
|
|
21
|
+
* Also what normalizes whatever the OS handed the file input — this is how a
|
|
22
|
+
* Safari HEIC becomes a JPEG. Throws if the image cannot be decoded, which is
|
|
23
|
+
* the third dropzone check. */
|
|
24
|
+
export declare function talqingDownscaleImage(file: File): Promise<Blob>;
|
|
25
|
+
/** Validate and downscale a file into the `data:` URL a text message carries.
|
|
26
|
+
*
|
|
27
|
+
* `createTextMessage` takes `images: [{data_url, filename}]`; this produces the
|
|
28
|
+
* `data_url`. It doubles as the thumbnail's `src`, so there is no object URL to
|
|
29
|
+
* create and revoke. Throws the sentence to show if the file is refused. */
|
|
30
|
+
export declare function talqingImageDataUrl(file: File): Promise<string>;
|
|
31
|
+
export type TalqingImageStatus = "sending" | "sent" | "failed" | "unknown";
|
|
32
|
+
/** One image the browser sent on a call, and what became of it. */
|
|
33
|
+
export interface TalqingSentImage {
|
|
34
|
+
/** The stream id `sendFile` returned, which the ack comes back under. */
|
|
35
|
+
streamId: string;
|
|
36
|
+
name: string;
|
|
37
|
+
/** An object URL for the downscaled image, for rendering the thumbnail.
|
|
38
|
+
* Revoke it when you are done with it — `useTalqingImages` does. */
|
|
39
|
+
previewUrl: string;
|
|
40
|
+
/** When it was handed to the room, so a transcript can put it in the sequence
|
|
41
|
+
* it was sent in rather than at the end. */
|
|
42
|
+
sentAt: number;
|
|
43
|
+
status: TalqingImageStatus;
|
|
44
|
+
/** Set when `status` is "failed" or "unknown" — a sentence, shown as-is. */
|
|
45
|
+
error?: string;
|
|
46
|
+
}
|
|
47
|
+
/** Validate, downscale and send one image to the agent on this call.
|
|
48
|
+
*
|
|
49
|
+
* Resolves once the bytes are on their way, with the entry to render. The
|
|
50
|
+
* outcome arrives later, on the ack — see `onTalqingImageResult`. Throws the
|
|
51
|
+
* sentence to show if the file is refused or the agent is not there yet. */
|
|
52
|
+
export declare function sendTalqingImage(room: Room, file: File): Promise<TalqingSentImage>;
|
|
53
|
+
/** Listen for the agent's verdict on each image sent. Returns the unregister.
|
|
54
|
+
*
|
|
55
|
+
* There is one ack per image even on the happy path, because the agent stays
|
|
56
|
+
* silent when a photo lands — so silence cannot double as "it arrived". */
|
|
57
|
+
export declare function onTalqingImageResult(room: Room, onResult: (result: {
|
|
58
|
+
streamId: string;
|
|
59
|
+
ok: boolean;
|
|
60
|
+
error?: string;
|
|
61
|
+
}) => void): () => void;
|
|
62
|
+
//# sourceMappingURL=images.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images.d.ts","sourceRoot":"","sources":["../../src/images.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAqB,MAAM,gBAAgB,CAAC;AAgB9D,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AACpD;2EAC2E;AAC3E,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD;;gEAEgE;AAChE,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;uBAEuB;AACvB,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;wBAGwB;AACxB,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAM/D;AAED;;;;gCAIgC;AAChC,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBrE;AAED;;;;6EAI6E;AAC7E,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAUrE;AAED,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE3E,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb;yEACqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB;iDAC6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,kBAAkB,CAAC;IAC3B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;6EAI6E;AAC7E,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAwBxF;AAED;;;4EAG4E;AAC5E,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,CAAC,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,KAAK,IAAI,GAC5E,MAAM,IAAI,CAeZ"}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { talqingAgentParticipant } from "./userdata.js";
|
|
2
|
+
/* ── images ───────────────────────────────────────────────────────────────
|
|
3
|
+
*
|
|
4
|
+
* A caller attaches a photo on a web voice or video call and the agent sees
|
|
5
|
+
* it. The bytes go over a LiveKit byte stream on the `talqing.images` topic,
|
|
6
|
+
* addressed to the agent; the agent answers on an RPC back, and the id
|
|
7
|
+
* `sendFile` returns is the id that ack carries, so there is nothing to
|
|
8
|
+
* correlate by hand.
|
|
9
|
+
*
|
|
10
|
+
* The agent deliberately does NOT speak when a photo lands — the caller is
|
|
11
|
+
* mid-conversation and about to say what it is for. The image is in the
|
|
12
|
+
* model's view for their next turn.
|
|
13
|
+
*/
|
|
14
|
+
export const TALQING_IMAGE_TOPIC = "talqing.images";
|
|
15
|
+
/** Decoded size the server refuses above. The downscale below keeps a phone
|
|
16
|
+
* photo far under it; this is the belt for a file that is not a photo. */
|
|
17
|
+
export const TALQING_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
18
|
+
/** Longest side the server stores. Downscaling to it before sending is a
|
|
19
|
+
* courtesy to the data channel, which is ordered and reliable and will
|
|
20
|
+
* happily spend a minute pushing 10 MB past a call's audio. */
|
|
21
|
+
export const TALQING_IMAGE_MAX_EDGE_PX = 1568;
|
|
22
|
+
const TALQING_IMAGE_TYPES = ["image/jpeg", "image/png", "image/webp"];
|
|
23
|
+
/** How long to wait for the agent's ack before saying we do not know. Without
|
|
24
|
+
* it a thumbnail sits on "sending" for the rest of the call when the worker
|
|
25
|
+
* dies mid-upload. */
|
|
26
|
+
export const TALQING_IMAGE_ACK_TIMEOUT_MS = 30000;
|
|
27
|
+
/** Refuse what the browser can see is wrong, instantly and with no round trip.
|
|
28
|
+
* Returns the sentence to show, or null when the file is fine. The server
|
|
29
|
+
* re-validates everything regardless — this exists so the common mistakes cost
|
|
30
|
+
* nothing to catch. */
|
|
31
|
+
export function talqingImageRejection(file) {
|
|
32
|
+
if (!TALQING_IMAGE_TYPES.includes(file.type)) {
|
|
33
|
+
return "that file is not a JPEG, PNG or WebP";
|
|
34
|
+
}
|
|
35
|
+
if (file.size > TALQING_IMAGE_MAX_BYTES)
|
|
36
|
+
return "images are 10 MB or smaller";
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
/** Downscale to the longest edge the server keeps, re-encoding as JPEG or PNG.
|
|
40
|
+
*
|
|
41
|
+
* Also what normalizes whatever the OS handed the file input — this is how a
|
|
42
|
+
* Safari HEIC becomes a JPEG. Throws if the image cannot be decoded, which is
|
|
43
|
+
* the third dropzone check. */
|
|
44
|
+
export async function talqingDownscaleImage(file) {
|
|
45
|
+
const bitmap = await createImageBitmap(file);
|
|
46
|
+
try {
|
|
47
|
+
const longest = Math.max(bitmap.width, bitmap.height);
|
|
48
|
+
const scale = longest > TALQING_IMAGE_MAX_EDGE_PX ? TALQING_IMAGE_MAX_EDGE_PX / longest : 1;
|
|
49
|
+
const canvas = document.createElement("canvas");
|
|
50
|
+
canvas.width = Math.max(1, Math.round(bitmap.width * scale));
|
|
51
|
+
canvas.height = Math.max(1, Math.round(bitmap.height * scale));
|
|
52
|
+
const context = canvas.getContext("2d");
|
|
53
|
+
if (!context)
|
|
54
|
+
throw new Error("this browser cannot resize images");
|
|
55
|
+
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
|
|
56
|
+
// PNG keeps transparency; everything else becomes a JPEG, including the
|
|
57
|
+
// HEIC the OS decoded for us.
|
|
58
|
+
const type = file.type === "image/png" ? "image/png" : "image/jpeg";
|
|
59
|
+
const blob = await new Promise((resolve) => canvas.toBlob(resolve, type, 0.85));
|
|
60
|
+
if (!blob)
|
|
61
|
+
throw new Error("that image could not be read");
|
|
62
|
+
return blob;
|
|
63
|
+
}
|
|
64
|
+
finally {
|
|
65
|
+
bitmap.close();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/** Validate and downscale a file into the `data:` URL a text message carries.
|
|
69
|
+
*
|
|
70
|
+
* `createTextMessage` takes `images: [{data_url, filename}]`; this produces the
|
|
71
|
+
* `data_url`. It doubles as the thumbnail's `src`, so there is no object URL to
|
|
72
|
+
* create and revoke. Throws the sentence to show if the file is refused. */
|
|
73
|
+
export async function talqingImageDataUrl(file) {
|
|
74
|
+
const rejection = talqingImageRejection(file);
|
|
75
|
+
if (rejection)
|
|
76
|
+
throw new Error(rejection);
|
|
77
|
+
const blob = await talqingDownscaleImage(file);
|
|
78
|
+
return await new Promise((resolve, reject) => {
|
|
79
|
+
const reader = new FileReader();
|
|
80
|
+
reader.onerror = () => reject(new Error("that image could not be read"));
|
|
81
|
+
reader.onload = () => resolve(String(reader.result));
|
|
82
|
+
reader.readAsDataURL(blob);
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
/** Validate, downscale and send one image to the agent on this call.
|
|
86
|
+
*
|
|
87
|
+
* Resolves once the bytes are on their way, with the entry to render. The
|
|
88
|
+
* outcome arrives later, on the ack — see `onTalqingImageResult`. Throws the
|
|
89
|
+
* sentence to show if the file is refused or the agent is not there yet. */
|
|
90
|
+
export async function sendTalqingImage(room, file) {
|
|
91
|
+
const agent = talqingAgentParticipant(room);
|
|
92
|
+
if (!agent)
|
|
93
|
+
throw new Error("the agent is not on the call yet");
|
|
94
|
+
const rejection = talqingImageRejection(file);
|
|
95
|
+
if (rejection)
|
|
96
|
+
throw new Error(rejection);
|
|
97
|
+
const blob = await talqingDownscaleImage(file);
|
|
98
|
+
// Addressed to the agent, never broadcast: on video the avatar worker is
|
|
99
|
+
// a participant too, and a broadcast sends the bytes twice.
|
|
100
|
+
const { id } = await room.localParticipant.sendFile(new File([blob], file.name, { type: blob.type }), {
|
|
101
|
+
topic: TALQING_IMAGE_TOPIC,
|
|
102
|
+
mimeType: blob.type,
|
|
103
|
+
destinationIdentities: [agent.identity],
|
|
104
|
+
});
|
|
105
|
+
return {
|
|
106
|
+
streamId: id,
|
|
107
|
+
name: file.name,
|
|
108
|
+
previewUrl: URL.createObjectURL(blob),
|
|
109
|
+
sentAt: Date.now(),
|
|
110
|
+
status: "sending",
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
/** Listen for the agent's verdict on each image sent. Returns the unregister.
|
|
114
|
+
*
|
|
115
|
+
* There is one ack per image even on the happy path, because the agent stays
|
|
116
|
+
* silent when a photo lands — so silence cannot double as "it arrived". */
|
|
117
|
+
export function onTalqingImageResult(room, onResult) {
|
|
118
|
+
room.registerRpcMethod("talqing.image_result", async (data) => {
|
|
119
|
+
const { stream_id, ok, error } = JSON.parse(data.payload || "{}");
|
|
120
|
+
onResult({
|
|
121
|
+
streamId: stream_id,
|
|
122
|
+
ok,
|
|
123
|
+
error: ok ? undefined : error || "that image was not accepted",
|
|
124
|
+
});
|
|
125
|
+
return "{}";
|
|
126
|
+
});
|
|
127
|
+
return () => room.unregisterRpcMethod("talqing.image_result");
|
|
128
|
+
}
|
|
129
|
+
//# sourceMappingURL=images.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/images.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAExD;;;;;;;;;;;GAWG;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gBAAgB,CAAC;AACpD;2EAC2E;AAC3E,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACxD;;gEAEgE;AAChE,MAAM,CAAC,MAAM,yBAAyB,GAAG,IAAI,CAAC;AAC9C,MAAM,mBAAmB,GAAG,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAU,CAAC;AAC/E;;uBAEuB;AACvB,MAAM,CAAC,MAAM,4BAA4B,GAAG,KAAM,CAAC;AAEnD;;;wBAGwB;AACxB,MAAM,UAAU,qBAAqB,CAAC,IAAU;IAC9C,IAAI,CAAE,mBAAyC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,OAAO,sCAAsC,CAAC;IAChD,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,GAAG,uBAAuB;QAAE,OAAO,6BAA6B,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;gCAIgC;AAChC,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,IAAU;IACpD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,OAAO,GAAG,yBAAyB,CAAC,CAAC,CAAC,yBAAyB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnE,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7D,wEAAwE;QACxE,8BAA8B;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,EAAE,CACtD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CACnC,CAAC;QACF,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;;;6EAI6E;AAC7E,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAU;IAClD,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnD,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAChC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACzE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAoBD;;;;6EAI6E;AAC7E,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,IAAU;IAC3D,MAAM,KAAK,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC/C,yEAAyE;IACzE,4DAA4D;IAC5D,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CACjD,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,EAChD;QACE,KAAK,EAAE,mBAAmB;QAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,qBAAqB,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC;KACxC,CACF,CAAC;IACF,OAAO;QACL,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,UAAU,EAAE,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC;QACrC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE;QAClB,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;;4EAG4E;AAC5E,MAAM,UAAU,oBAAoB,CAClC,IAAU,EACV,QAA6E;IAE7E,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,IAAuB,EAAE,EAAE;QAC/E,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAI/D,CAAC;QACF,QAAQ,CAAC;YACP,QAAQ,EAAE,SAAS;YACnB,EAAE;YACF,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,6BAA6B;SAC/D,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { createTalqingTokenSource } from "./token.js";
|
|
2
|
+
export type { TalqingCallCredentials } from "./token.js";
|
|
3
|
+
export { getTalqingUserdata, setTalqingUserdata, talqingAgentParticipant, } from "./userdata.js";
|
|
4
|
+
export { onTalqingImageResult, sendTalqingImage, talqingDownscaleImage, talqingImageDataUrl, talqingImageRejection, TALQING_IMAGE_ACK_TIMEOUT_MS, TALQING_IMAGE_MAX_BYTES, TALQING_IMAGE_MAX_EDGE_PX, TALQING_IMAGE_TOPIC, } from "./images.js";
|
|
5
|
+
export type { TalqingImageStatus, TalqingSentImage } from "./images.js";
|
|
6
|
+
export { registerTalqingRpcHandlers, TalqingRpcError } from "./rpc.js";
|
|
7
|
+
export type { TalqingFrontendRpcEnvelope, TalqingRpcHandler, TalqingRpcHandlers, TalqingRpcInvocation, } from "./rpc.js";
|
|
8
|
+
export { TALQING_AUDIO_CAPTURE_DEFAULTS, TALQING_SCREEN_SHARE_CAPTURE, TALQING_SCREEN_SHARE_PUBLISH, } from "./media.js";
|
|
9
|
+
export type { JsonObject, JsonPrimitive, JsonValue } from "./json.js";
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACtD,YAAY,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEzD,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAExE,OAAO,EAAE,0BAA0B,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACvE,YAAY,EACV,0BAA0B,EAC1B,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,8BAA8B,EAC9B,4BAA4B,EAC5B,4BAA4B,GAC7B,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/* The Talqing call protocol, over a LiveKit `Room`. Framework-free.
|
|
2
|
+
*
|
|
3
|
+
* Everything here is something only Talqing can write: the mint-once token
|
|
4
|
+
* source, the two userdata RPC methods, the `talqing.images` byte-stream topic
|
|
5
|
+
* and its ack, the `talqing.frontend_rpc` envelope, and the measured capture
|
|
6
|
+
* settings. Nothing here renames LiveKit — for a room, a track or a
|
|
7
|
+
* participant, use `livekit-client` directly.
|
|
8
|
+
*
|
|
9
|
+
* React bindings for all of it are `@talqing/react`.
|
|
10
|
+
*/
|
|
11
|
+
export { createTalqingTokenSource } from "./token.js";
|
|
12
|
+
export { getTalqingUserdata, setTalqingUserdata, talqingAgentParticipant, } from "./userdata.js";
|
|
13
|
+
export { onTalqingImageResult, sendTalqingImage, talqingDownscaleImage, talqingImageDataUrl, talqingImageRejection, TALQING_IMAGE_ACK_TIMEOUT_MS, TALQING_IMAGE_MAX_BYTES, TALQING_IMAGE_MAX_EDGE_PX, TALQING_IMAGE_TOPIC, } from "./images.js";
|
|
14
|
+
export { registerTalqingRpcHandlers, TalqingRpcError } from "./rpc.js";
|
|
15
|
+
export { TALQING_AUDIO_CAPTURE_DEFAULTS, TALQING_SCREEN_SHARE_CAPTURE, TALQING_SCREEN_SHARE_PUBLISH, } from "./media.js";
|
|
16
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AAGtD,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,aAAa,CAAC;AAGrB,OAAO,EAAE,0BAA0B,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAQvE,OAAO,EACL,8BAA8B,EAC9B,4BAA4B,EAC5B,4BAA4B,GAC7B,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/json.ts"],"names":[],"mappings":"AAcA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,SAAS,EAAE,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AACnF,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC"}
|
package/dist/esm/json.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/json.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ScreenShareCaptureOptions, TrackPublishOptions } from "livekit-client";
|
|
2
|
+
export declare const TALQING_AUDIO_CAPTURE_DEFAULTS: {
|
|
3
|
+
readonly echoCancellation: true;
|
|
4
|
+
readonly noiseSuppression: true;
|
|
5
|
+
readonly autoGainControl: true;
|
|
6
|
+
};
|
|
7
|
+
export declare const TALQING_SCREEN_SHARE_CAPTURE: ScreenShareCaptureOptions;
|
|
8
|
+
export declare const TALQING_SCREEN_SHARE_PUBLISH: TrackPublishOptions;
|
|
9
|
+
//# sourceMappingURL=media.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"media.d.ts","sourceRoot":"","sources":["../../src/media.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOrF,eAAO,MAAM,8BAA8B;;;;CAIjC,CAAC;AAiCX,eAAO,MAAM,4BAA4B,EAAE,yBAG1C,CAAC;AAEF,eAAO,MAAM,4BAA4B,EAAE,mBAG1C,CAAC"}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/* Echo cancellation is what stops the agent hearing its own speech through the
|
|
2
|
+
* caller's microphone and interrupting itself. It is a default rather than
|
|
3
|
+
* something each app remembers, because a call without it is broken in a way
|
|
4
|
+
* that sounds like a bad model. Pass `roomOptions.audioCaptureDefaults` to
|
|
5
|
+
* override. */
|
|
6
|
+
export const TALQING_AUDIO_CAPTURE_DEFAULTS = {
|
|
7
|
+
echoCancellation: true,
|
|
8
|
+
noiseSuppression: true,
|
|
9
|
+
autoGainControl: true,
|
|
10
|
+
};
|
|
11
|
+
/* Screen share, with the four settings that decide whether the agent can read
|
|
12
|
+
* what is on it.
|
|
13
|
+
*
|
|
14
|
+
* Only the first is obvious, and every one of them is upstream of anything we
|
|
15
|
+
* could do on our side:
|
|
16
|
+
*
|
|
17
|
+
* - `contentHint: "text"` tells the encoder to preserve spatial detail. The
|
|
18
|
+
* default for a screen share on an SVC codec is `motion`, which trades
|
|
19
|
+
* exactly that away — the wrong trade for reading code, and no JPEG quality
|
|
20
|
+
* setting downstream recovers it.
|
|
21
|
+
* - `videoCodec: "h264"` is what keeps that hint. `livekit-client` forces
|
|
22
|
+
* `contentHint = "motion"` on screen shares published as vp9 or av1,
|
|
23
|
+
* together with `scalabilityMode: "L1T3"`. **Publishing on an SVC codec will
|
|
24
|
+
* cost you legibility**, and it is the first thing to check if the agent
|
|
25
|
+
* starts misreading identifiers.
|
|
26
|
+
* - `screenShareEncoding.maxFramerate: 1` is the frame-rate cap that actually
|
|
27
|
+
* binds. `resolution.frameRate` reaches `getDisplayMedia` as a bare number,
|
|
28
|
+
* which WebRTC reads as `ideal` — a wish, not a limit. The default publish
|
|
29
|
+
* ceiling is 15 fps, and every delivered frame is copied into the agent's
|
|
30
|
+
* process before it can be dropped, so this is the single biggest lever on
|
|
31
|
+
* what a call costs the worker. One frame a second is ample: the agent is
|
|
32
|
+
* handed the newest frame at the end of each turn, and turns are seconds
|
|
33
|
+
* apart.
|
|
34
|
+
* - `resolution` is set explicitly rather than inherited. The default is 1080p
|
|
35
|
+
* by `ideal`, which is a strong hint and not a guarantee.
|
|
36
|
+
*
|
|
37
|
+
* The picker cannot be opened for the person — `getDisplayMedia` requires a
|
|
38
|
+
* transient user activation — so sharing must start from a click. The agent
|
|
39
|
+
* asking out loud is what prompts it; there is no version of this where the
|
|
40
|
+
* agent turns sharing on.
|
|
41
|
+
*/
|
|
42
|
+
export const TALQING_SCREEN_SHARE_CAPTURE = {
|
|
43
|
+
resolution: { width: 1920, height: 1080, frameRate: 1 },
|
|
44
|
+
contentHint: "text",
|
|
45
|
+
};
|
|
46
|
+
export const TALQING_SCREEN_SHARE_PUBLISH = {
|
|
47
|
+
videoCodec: "h264",
|
|
48
|
+
screenShareEncoding: { maxBitrate: 1000000, maxFramerate: 1 },
|
|
49
|
+
};
|
|
50
|
+
//# sourceMappingURL=media.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"media.js","sourceRoot":"","sources":["../../src/media.ts"],"names":[],"mappings":"AAEA;;;;eAIe;AACf,MAAM,CAAC,MAAM,8BAA8B,GAAG;IAC5C,gBAAgB,EAAE,IAAI;IACtB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,IAAI;CACb,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAA8B;IACrE,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE;IACvD,WAAW,EAAE,MAAM;CACpB,CAAC;AAEF,MAAM,CAAC,MAAM,4BAA4B,GAAwB;IAC/D,UAAU,EAAE,MAAM;IAClB,mBAAmB,EAAE,EAAE,UAAU,EAAE,OAAS,EAAE,YAAY,EAAE,CAAC,EAAE;CAChE,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type Room } from "livekit-client";
|
|
2
|
+
import type { JsonValue } from "./json.js";
|
|
3
|
+
export interface TalqingRpcInvocation {
|
|
4
|
+
payload: string;
|
|
5
|
+
responseTimeout?: number;
|
|
6
|
+
callerIdentity?: string;
|
|
7
|
+
}
|
|
8
|
+
export type TalqingRpcHandler = (data: TalqingRpcInvocation) => string | Promise<string>;
|
|
9
|
+
export type TalqingRpcHandlers = Record<string, TalqingRpcHandler>;
|
|
10
|
+
export interface TalqingFrontendRpcEnvelope {
|
|
11
|
+
method: string;
|
|
12
|
+
payload?: JsonValue;
|
|
13
|
+
}
|
|
14
|
+
/** Throw this from a handler to choose the code the agent sees. */
|
|
15
|
+
export declare class TalqingRpcError extends Error {
|
|
16
|
+
readonly code: number;
|
|
17
|
+
readonly data?: string;
|
|
18
|
+
constructor(code: number, message: string, data?: string);
|
|
19
|
+
}
|
|
20
|
+
/** Register the frontend-RPC envelope handler on a room. Returns the unregister.
|
|
21
|
+
*
|
|
22
|
+
* A `"*"` key catches every method with no handler of its own, and receives the
|
|
23
|
+
* WHOLE envelope as its payload string so it can tell what it caught. */
|
|
24
|
+
export declare function registerTalqingRpcHandlers(room: Room, handlers: TalqingRpcHandlers): () => void;
|
|
25
|
+
//# sourceMappingURL=rpc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/rpc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,IAAI,EAA0B,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAU3C,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,iBAAiB,GAAG,CAC9B,IAAI,EAAE,oBAAoB,KACvB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAC9B,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;AAEnE,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,SAAS,CAAC;CACrB;AAED,mEAAmE;AACnE,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAEX,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM;CAMzD;AAED;;;0EAG0E;AAC1E,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,kBAAkB,GAC3B,MAAM,IAAI,CAiCZ"}
|
package/dist/esm/rpc.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { RpcError } from "livekit-client";
|
|
2
|
+
/** Throw this from a handler to choose the code the agent sees. */
|
|
3
|
+
export class TalqingRpcError extends Error {
|
|
4
|
+
constructor(code, message, data) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "TalqingRpcError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
this.data = data;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
/** Register the frontend-RPC envelope handler on a room. Returns the unregister.
|
|
12
|
+
*
|
|
13
|
+
* A `"*"` key catches every method with no handler of its own, and receives the
|
|
14
|
+
* WHOLE envelope as its payload string so it can tell what it caught. */
|
|
15
|
+
export function registerTalqingRpcHandlers(room, handlers) {
|
|
16
|
+
room.registerRpcMethod("talqing.frontend_rpc", async (data) => {
|
|
17
|
+
try {
|
|
18
|
+
const envelope = JSON.parse(data.payload || "{}");
|
|
19
|
+
const isWildcard = handlers[envelope.method] === undefined;
|
|
20
|
+
const handler = handlers[envelope.method] ?? handlers["*"];
|
|
21
|
+
if (!handler) {
|
|
22
|
+
throw new TalqingRpcError(1404, `No frontend RPC handler registered for ${envelope.method}`);
|
|
23
|
+
}
|
|
24
|
+
return await handler({
|
|
25
|
+
payload: isWildcard
|
|
26
|
+
? JSON.stringify(envelope)
|
|
27
|
+
: envelope.payload === undefined
|
|
28
|
+
? "{}"
|
|
29
|
+
: JSON.stringify(envelope.payload),
|
|
30
|
+
responseTimeout: data.responseTimeout,
|
|
31
|
+
callerIdentity: data.callerIdentity,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
if (error instanceof TalqingRpcError) {
|
|
36
|
+
throw new RpcError(error.code, error.message, error.data);
|
|
37
|
+
}
|
|
38
|
+
throw new RpcError(1500, error instanceof Error ? error.message : "RPC handler failed");
|
|
39
|
+
}
|
|
40
|
+
});
|
|
41
|
+
return () => room.unregisterRpcMethod("talqing.frontend_rpc");
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=rpc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc.js","sourceRoot":"","sources":["../../src/rpc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAqC,MAAM,gBAAgB,CAAC;AA2B7E,mEAAmE;AACnE,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAIxC,YAAY,IAAY,EAAE,OAAe,EAAE,IAAa;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAED;;;0EAG0E;AAC1E,MAAM,UAAU,0BAA0B,CACxC,IAAU,EACV,QAA4B;IAE5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,IAAuB,EAAE,EAAE;QAC/E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAA+B,CAAC;YAChF,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC;YAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,eAAe,CACvB,IAAI,EACJ,0CAA0C,QAAQ,CAAC,MAAM,EAAE,CAC5D,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,OAAO,CAAC;gBACnB,OAAO,EAAE,UAAU;oBACjB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;oBAC1B,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS;wBAC9B,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACtC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;gBACrC,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,IAAI,QAAQ,CAChB,IAAI,EACJ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type TokenSourceFixed } from "livekit-client";
|
|
2
|
+
/** The two fields a browser needs to join a room, and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* Named as `POST /v1/calls/token` names them, so a server that mints one can
|
|
5
|
+
* forward the response through to the browser without renaming anything. */
|
|
6
|
+
export interface TalqingCallCredentials {
|
|
7
|
+
server_url: string;
|
|
8
|
+
participant_token: string;
|
|
9
|
+
}
|
|
10
|
+
/** Turn a one-shot mint into the token source LiveKit's session takes.
|
|
11
|
+
*
|
|
12
|
+
* `fetchToken` is called at most once per source, no matter how often LiveKit
|
|
13
|
+
* asks. That is load-bearing rather than an optimization: minting is not free,
|
|
14
|
+
* because `POST /v1/calls/token` files the session row the call is later read
|
|
15
|
+
* back from. The framework asks three times per call — to warm the connection,
|
|
16
|
+
* to connect, and once more as the call ends to drop what it assumes is a
|
|
17
|
+
* cached token — and only the middle one is a call anybody placed. Minting for
|
|
18
|
+
* the third filed a session nobody ever joined, which sat in the workspace's
|
|
19
|
+
* call list as queued for good.
|
|
20
|
+
*
|
|
21
|
+
* `TokenSource.literal` is the LiveKit variant that leaves caching to us, and
|
|
22
|
+
* this is us doing it.
|
|
23
|
+
*
|
|
24
|
+
* Freshness comes from a new source per call, never from re-minting inside
|
|
25
|
+
* one: the token names the room, and this call has exactly one. Mount the
|
|
26
|
+
* session per call and end it on unmount, as LiveKit's own session lifecycle
|
|
27
|
+
* does, and that holds by construction. */
|
|
28
|
+
export declare function createTalqingTokenSource(fetchToken: () => Promise<TalqingCallCredentials>): TokenSourceFixed;
|
|
29
|
+
//# sourceMappingURL=token.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../src/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEpE;;;6EAG6E;AAC7E,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;4CAiB4C;AAC5C,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,MAAM,OAAO,CAAC,sBAAsB,CAAC,GAChD,gBAAgB,CAiBlB"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { TokenSource } from "livekit-client";
|
|
2
|
+
/** Turn a one-shot mint into the token source LiveKit's session takes.
|
|
3
|
+
*
|
|
4
|
+
* `fetchToken` is called at most once per source, no matter how often LiveKit
|
|
5
|
+
* asks. That is load-bearing rather than an optimization: minting is not free,
|
|
6
|
+
* because `POST /v1/calls/token` files the session row the call is later read
|
|
7
|
+
* back from. The framework asks three times per call — to warm the connection,
|
|
8
|
+
* to connect, and once more as the call ends to drop what it assumes is a
|
|
9
|
+
* cached token — and only the middle one is a call anybody placed. Minting for
|
|
10
|
+
* the third filed a session nobody ever joined, which sat in the workspace's
|
|
11
|
+
* call list as queued for good.
|
|
12
|
+
*
|
|
13
|
+
* `TokenSource.literal` is the LiveKit variant that leaves caching to us, and
|
|
14
|
+
* this is us doing it.
|
|
15
|
+
*
|
|
16
|
+
* Freshness comes from a new source per call, never from re-minting inside
|
|
17
|
+
* one: the token names the room, and this call has exactly one. Mount the
|
|
18
|
+
* session per call and end it on unmount, as LiveKit's own session lifecycle
|
|
19
|
+
* does, and that holds by construction. */
|
|
20
|
+
export function createTalqingTokenSource(fetchToken) {
|
|
21
|
+
let minted = null;
|
|
22
|
+
return TokenSource.literal(() => {
|
|
23
|
+
// A mint that failed is not this call's token, and the surface will offer
|
|
24
|
+
// the call again — so keep the rejection out of the cache.
|
|
25
|
+
minted ?? (minted = fetchToken()
|
|
26
|
+
.then((credentials) => ({
|
|
27
|
+
serverUrl: credentials.server_url,
|
|
28
|
+
participantToken: credentials.participant_token,
|
|
29
|
+
}))
|
|
30
|
+
.catch((error) => {
|
|
31
|
+
minted = null;
|
|
32
|
+
throw error;
|
|
33
|
+
}));
|
|
34
|
+
return minted;
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=token.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../src/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAyB,MAAM,gBAAgB,CAAC;AAWpE;;;;;;;;;;;;;;;;;4CAiB4C;AAC5C,MAAM,UAAU,wBAAwB,CACtC,UAAiD;IAEjD,IAAI,MAAM,GAAoE,IAAI,CAAC;IAEnF,OAAO,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE;QAC9B,0EAA0E;QAC1E,2DAA2D;QAC3D,MAAM,KAAN,MAAM,GAAK,UAAU,EAAE;aACpB,IAAI,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;YACtB,SAAS,EAAE,WAAW,CAAC,UAAU;YACjC,gBAAgB,EAAE,WAAW,CAAC,iBAAiB;SAChD,CAAC,CAAC;aACF,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACxB,MAAM,GAAG,IAAI,CAAC;YACd,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,EAAC;QACL,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC"}
|