@talqing/client 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +121 -0
- package/dist/cjs/images.d.ts +62 -0
- package/dist/cjs/images.d.ts.map +1 -0
- package/dist/cjs/images.js +137 -0
- package/dist/cjs/images.js.map +1 -0
- package/dist/cjs/index.d.ts +10 -0
- package/dist/cjs/index.d.ts.map +1 -0
- package/dist/cjs/index.js +37 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/json.d.ts +8 -0
- package/dist/cjs/json.d.ts.map +1 -0
- package/dist/cjs/json.js +3 -0
- package/dist/cjs/json.js.map +1 -0
- package/dist/cjs/media.d.ts +9 -0
- package/dist/cjs/media.d.ts.map +1 -0
- package/dist/cjs/media.js +53 -0
- package/dist/cjs/media.js.map +1 -0
- package/dist/cjs/package.json +3 -0
- package/dist/cjs/rpc.d.ts +25 -0
- package/dist/cjs/rpc.d.ts.map +1 -0
- package/dist/cjs/rpc.js +48 -0
- package/dist/cjs/rpc.js.map +1 -0
- package/dist/cjs/token.d.ts +29 -0
- package/dist/cjs/token.d.ts.map +1 -0
- package/dist/cjs/token.js +40 -0
- package/dist/cjs/token.js.map +1 -0
- package/dist/cjs/userdata.d.ts +22 -0
- package/dist/cjs/userdata.d.ts.map +1 -0
- package/dist/cjs/userdata.js +51 -0
- package/dist/cjs/userdata.js.map +1 -0
- package/dist/esm/images.d.ts +62 -0
- package/dist/esm/images.d.ts.map +1 -0
- package/dist/esm/images.js +129 -0
- package/dist/esm/images.js.map +1 -0
- package/dist/esm/index.d.ts +10 -0
- package/dist/esm/index.d.ts.map +1 -0
- package/dist/esm/index.js +16 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/json.d.ts +8 -0
- package/dist/esm/json.d.ts.map +1 -0
- package/dist/esm/json.js +2 -0
- package/dist/esm/json.js.map +1 -0
- package/dist/esm/media.d.ts +9 -0
- package/dist/esm/media.d.ts.map +1 -0
- package/dist/esm/media.js +50 -0
- package/dist/esm/media.js.map +1 -0
- package/dist/esm/package.json +3 -0
- package/dist/esm/rpc.d.ts +25 -0
- package/dist/esm/rpc.d.ts.map +1 -0
- package/dist/esm/rpc.js +43 -0
- package/dist/esm/rpc.js.map +1 -0
- package/dist/esm/token.d.ts +29 -0
- package/dist/esm/token.d.ts.map +1 -0
- package/dist/esm/token.js +37 -0
- package/dist/esm/token.js.map +1 -0
- package/dist/esm/userdata.d.ts +22 -0
- package/dist/esm/userdata.d.ts.map +1 -0
- package/dist/esm/userdata.js +46 -0
- package/dist/esm/userdata.js.map +1 -0
- package/package.json +62 -0
- package/src/images.ts +160 -0
- package/src/index.ts +48 -0
- package/src/json.ts +17 -0
- package/src/media.ts +53 -0
- package/src/rpc.ts +81 -0
- package/src/token.ts +49 -0
- package/src/userdata.ts +57 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Talqing
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# @talqing/client
|
|
2
|
+
|
|
3
|
+
The Talqing call protocol, over a LiveKit `Room`. Framework-free.
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
npm install @talqing/client livekit-client
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Everything here is something only Talqing can write: the mint-once token source,
|
|
10
|
+
the two userdata RPC methods, the `talqing.images` byte-stream topic and its
|
|
11
|
+
acknowledgement, the `talqing.frontend_rpc` envelope, and the capture settings
|
|
12
|
+
that decide whether the agent can read a shared screen. **Nothing here renames
|
|
13
|
+
LiveKit** — for a room, a track or a participant, use `livekit-client` directly.
|
|
14
|
+
|
|
15
|
+
**If you are writing React, install
|
|
16
|
+
[`@talqing/react`](https://www.npmjs.com/package/@talqing/react) instead.** It
|
|
17
|
+
re-exports all of this and adds the hooks.
|
|
18
|
+
|
|
19
|
+
## Joining a call
|
|
20
|
+
|
|
21
|
+
Your **server** mints the token with `POST /v1/calls/token` — see
|
|
22
|
+
[`@talqing/sdk`](https://www.npmjs.com/package/@talqing/sdk) — and the browser
|
|
23
|
+
receives two strings. A Talqing credential that can mint a call token can also
|
|
24
|
+
create and delete agents, read every transcript and spend the workspace's
|
|
25
|
+
credits, so it never belongs in browser code.
|
|
26
|
+
|
|
27
|
+
```ts
|
|
28
|
+
import { Room } from "livekit-client";
|
|
29
|
+
import { createTalqingTokenSource, TALQING_AUDIO_CAPTURE_DEFAULTS } from "@talqing/client";
|
|
30
|
+
|
|
31
|
+
const room = new Room({ audioCaptureDefaults: TALQING_AUDIO_CAPTURE_DEFAULTS });
|
|
32
|
+
|
|
33
|
+
const tokenSource = createTalqingTokenSource(async () => {
|
|
34
|
+
const response = await fetch("/api/talqing-token", { method: "POST" });
|
|
35
|
+
return await response.json(); // { server_url, participant_token }
|
|
36
|
+
});
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The source mints **once**, however often it is asked. That is load-bearing:
|
|
40
|
+
minting files the session row the call is later read back from, and LiveKit asks
|
|
41
|
+
three times per call — to warm the connection, to connect, and once more as the
|
|
42
|
+
call ends. Freshness comes from a new source per call, never from re-minting
|
|
43
|
+
inside one.
|
|
44
|
+
|
|
45
|
+
Leave `TALQING_AUDIO_CAPTURE_DEFAULTS` on. Echo cancellation is what stops the
|
|
46
|
+
agent hearing its own speech through the microphone and interrupting itself,
|
|
47
|
+
which sounds exactly like a bad model.
|
|
48
|
+
|
|
49
|
+
## Userdata
|
|
50
|
+
|
|
51
|
+
What the agent knows about the person on the call, readable as
|
|
52
|
+
`{{userdata.field}}` from its prompt and its tools.
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { getTalqingUserdata, setTalqingUserdata } from "@talqing/client";
|
|
56
|
+
|
|
57
|
+
await setTalqingUserdata(room, { cart_total: 4200 }); // → { ok, error? }
|
|
58
|
+
const { tier } = await getTalqingUserdata(room, ["tier"]);
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Both are RPCs to the worker running the call, with a 3-second timeout by
|
|
62
|
+
default. `set` resolves `{ ok: false }` when the agent has not joined — a write
|
|
63
|
+
that did not land is a state you can render — while `get` **throws**, because a
|
|
64
|
+
read that returned nothing is not.
|
|
65
|
+
|
|
66
|
+
## Images
|
|
67
|
+
|
|
68
|
+
```ts
|
|
69
|
+
import { onTalqingImageResult, sendTalqingImage } from "@talqing/client";
|
|
70
|
+
|
|
71
|
+
const stop = onTalqingImageResult(room, ({ streamId, ok, error }) => render(streamId, ok, error));
|
|
72
|
+
const image = await sendTalqingImage(room, file); // validates, downscales, sends
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
`sendTalqingImage` resolves once the bytes are on their way; the verdict arrives
|
|
76
|
+
on the acknowledgement. There is one **even on the happy path**, because the
|
|
77
|
+
agent deliberately stays silent when a photo lands — the caller is
|
|
78
|
+
mid-conversation and about to say what it is for — so silence cannot double as
|
|
79
|
+
"it arrived". Nothing acknowledging within 30 seconds means the agent went away
|
|
80
|
+
mid-upload, which is a third outcome and not a failure you can describe.
|
|
81
|
+
|
|
82
|
+
`talqingImageRejection(file)`, `talqingDownscaleImage(file)` and
|
|
83
|
+
`talqingImageDataUrl(file)` are the three steps on their own. The last produces
|
|
84
|
+
the `data:` URL a **text** message carries.
|
|
85
|
+
|
|
86
|
+
## Frontend RPC
|
|
87
|
+
|
|
88
|
+
A tool's `frontend_rpc` operation calls into your page. The agent sends one
|
|
89
|
+
envelope method and this dispatches on `envelope.method`, so you register once
|
|
90
|
+
rather than per tool.
|
|
91
|
+
|
|
92
|
+
```ts
|
|
93
|
+
import { registerTalqingRpcHandlers, TalqingRpcError } from "@talqing/client";
|
|
94
|
+
|
|
95
|
+
const stop = registerTalqingRpcHandlers(room, {
|
|
96
|
+
open_page: async ({ payload }) => {
|
|
97
|
+
const { href } = JSON.parse(payload) as { href: string };
|
|
98
|
+
navigate(href);
|
|
99
|
+
return JSON.stringify({ ok: true });
|
|
100
|
+
},
|
|
101
|
+
cancel_order: async () => {
|
|
102
|
+
throw new TalqingRpcError(1403, "the customer declined");
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
A `"*"` key catches everything unmatched and receives the whole envelope. An
|
|
108
|
+
unknown method with no wildcard answers RPC code `1404`; anything else you throw
|
|
109
|
+
becomes `1500` with its message.
|
|
110
|
+
|
|
111
|
+
## Screen share
|
|
112
|
+
|
|
113
|
+
`TALQING_SCREEN_SHARE_CAPTURE` and `TALQING_SCREEN_SHARE_PUBLISH` are the four
|
|
114
|
+
settings that decide whether the agent can read what is on screen — h264 at one
|
|
115
|
+
frame a second, 1920×1080, `contentHint: "text"`. Publishing on an SVC codec
|
|
116
|
+
costs you legibility, and it is the first thing to check if the agent starts
|
|
117
|
+
misreading identifiers. Pass them to `room.localParticipant.setScreenShareEnabled`,
|
|
118
|
+
and call it from a click: the browser's picker needs a user gesture, so an agent
|
|
119
|
+
can ask for a screen but can never take one.
|
|
120
|
+
|
|
121
|
+
Full reference: <https://docs.talqing.com/sdks/react>.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import type { Room } from "livekit-client";
|
|
2
|
+
export declare const TALQING_IMAGE_TOPIC = "talqing.images";
|
|
3
|
+
/** Decoded size the server refuses above. The downscale below keeps a phone
|
|
4
|
+
* photo far under it; this is the belt for a file that is not a photo. */
|
|
5
|
+
export declare const TALQING_IMAGE_MAX_BYTES: number;
|
|
6
|
+
/** Longest side the server stores. Downscaling to it before sending is a
|
|
7
|
+
* courtesy to the data channel, which is ordered and reliable and will
|
|
8
|
+
* happily spend a minute pushing 10 MB past a call's audio. */
|
|
9
|
+
export declare const TALQING_IMAGE_MAX_EDGE_PX = 1568;
|
|
10
|
+
/** How long to wait for the agent's ack before saying we do not know. Without
|
|
11
|
+
* it a thumbnail sits on "sending" for the rest of the call when the worker
|
|
12
|
+
* dies mid-upload. */
|
|
13
|
+
export declare const TALQING_IMAGE_ACK_TIMEOUT_MS = 30000;
|
|
14
|
+
/** Refuse what the browser can see is wrong, instantly and with no round trip.
|
|
15
|
+
* Returns the sentence to show, or null when the file is fine. The server
|
|
16
|
+
* re-validates everything regardless — this exists so the common mistakes cost
|
|
17
|
+
* nothing to catch. */
|
|
18
|
+
export declare function talqingImageRejection(file: File): string | null;
|
|
19
|
+
/** Downscale to the longest edge the server keeps, re-encoding as JPEG or PNG.
|
|
20
|
+
*
|
|
21
|
+
* Also what normalizes whatever the OS handed the file input — this is how a
|
|
22
|
+
* Safari HEIC becomes a JPEG. Throws if the image cannot be decoded, which is
|
|
23
|
+
* the third dropzone check. */
|
|
24
|
+
export declare function talqingDownscaleImage(file: File): Promise<Blob>;
|
|
25
|
+
/** Validate and downscale a file into the `data:` URL a text message carries.
|
|
26
|
+
*
|
|
27
|
+
* `createTextMessage` takes `images: [{data_url, filename}]`; this produces the
|
|
28
|
+
* `data_url`. It doubles as the thumbnail's `src`, so there is no object URL to
|
|
29
|
+
* create and revoke. Throws the sentence to show if the file is refused. */
|
|
30
|
+
export declare function talqingImageDataUrl(file: File): Promise<string>;
|
|
31
|
+
export type TalqingImageStatus = "sending" | "sent" | "failed" | "unknown";
|
|
32
|
+
/** One image the browser sent on a call, and what became of it. */
|
|
33
|
+
export interface TalqingSentImage {
|
|
34
|
+
/** The stream id `sendFile` returned, which the ack comes back under. */
|
|
35
|
+
streamId: string;
|
|
36
|
+
name: string;
|
|
37
|
+
/** An object URL for the downscaled image, for rendering the thumbnail.
|
|
38
|
+
* Revoke it when you are done with it — `useTalqingImages` does. */
|
|
39
|
+
previewUrl: string;
|
|
40
|
+
/** When it was handed to the room, so a transcript can put it in the sequence
|
|
41
|
+
* it was sent in rather than at the end. */
|
|
42
|
+
sentAt: number;
|
|
43
|
+
status: TalqingImageStatus;
|
|
44
|
+
/** Set when `status` is "failed" or "unknown" — a sentence, shown as-is. */
|
|
45
|
+
error?: string;
|
|
46
|
+
}
|
|
47
|
+
/** Validate, downscale and send one image to the agent on this call.
|
|
48
|
+
*
|
|
49
|
+
* Resolves once the bytes are on their way, with the entry to render. The
|
|
50
|
+
* outcome arrives later, on the ack — see `onTalqingImageResult`. Throws the
|
|
51
|
+
* sentence to show if the file is refused or the agent is not there yet. */
|
|
52
|
+
export declare function sendTalqingImage(room: Room, file: File): Promise<TalqingSentImage>;
|
|
53
|
+
/** Listen for the agent's verdict on each image sent. Returns the unregister.
|
|
54
|
+
*
|
|
55
|
+
* There is one ack per image even on the happy path, because the agent stays
|
|
56
|
+
* silent when a photo lands — so silence cannot double as "it arrived". */
|
|
57
|
+
export declare function onTalqingImageResult(room: Room, onResult: (result: {
|
|
58
|
+
streamId: string;
|
|
59
|
+
ok: boolean;
|
|
60
|
+
error?: string;
|
|
61
|
+
}) => void): () => void;
|
|
62
|
+
//# sourceMappingURL=images.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images.d.ts","sourceRoot":"","sources":["../../src/images.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAqB,MAAM,gBAAgB,CAAC;AAgB9D,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AACpD;2EAC2E;AAC3E,eAAO,MAAM,uBAAuB,QAAmB,CAAC;AACxD;;gEAEgE;AAChE,eAAO,MAAM,yBAAyB,OAAO,CAAC;AAE9C;;uBAEuB;AACvB,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD;;;wBAGwB;AACxB,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,GAAG,IAAI,CAM/D;AAED;;;;gCAIgC;AAChC,wBAAsB,qBAAqB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBrE;AAED;;;;6EAI6E;AAC7E,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAUrE;AAED,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE3E,mEAAmE;AACnE,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb;yEACqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB;iDAC6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,kBAAkB,CAAC;IAC3B,4EAA4E;IAC5E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;6EAI6E;AAC7E,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAwBxF;AAED;;;4EAG4E;AAC5E,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,CAAC,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,KAAK,IAAI,GAC5E,MAAM,IAAI,CAeZ"}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TALQING_IMAGE_ACK_TIMEOUT_MS = exports.TALQING_IMAGE_MAX_EDGE_PX = exports.TALQING_IMAGE_MAX_BYTES = exports.TALQING_IMAGE_TOPIC = void 0;
|
|
4
|
+
exports.talqingImageRejection = talqingImageRejection;
|
|
5
|
+
exports.talqingDownscaleImage = talqingDownscaleImage;
|
|
6
|
+
exports.talqingImageDataUrl = talqingImageDataUrl;
|
|
7
|
+
exports.sendTalqingImage = sendTalqingImage;
|
|
8
|
+
exports.onTalqingImageResult = onTalqingImageResult;
|
|
9
|
+
const userdata_js_1 = require("./userdata.js");
|
|
10
|
+
/* ── images ───────────────────────────────────────────────────────────────
|
|
11
|
+
*
|
|
12
|
+
* A caller attaches a photo on a web voice or video call and the agent sees
|
|
13
|
+
* it. The bytes go over a LiveKit byte stream on the `talqing.images` topic,
|
|
14
|
+
* addressed to the agent; the agent answers on an RPC back, and the id
|
|
15
|
+
* `sendFile` returns is the id that ack carries, so there is nothing to
|
|
16
|
+
* correlate by hand.
|
|
17
|
+
*
|
|
18
|
+
* The agent deliberately does NOT speak when a photo lands — the caller is
|
|
19
|
+
* mid-conversation and about to say what it is for. The image is in the
|
|
20
|
+
* model's view for their next turn.
|
|
21
|
+
*/
|
|
22
|
+
exports.TALQING_IMAGE_TOPIC = "talqing.images";
|
|
23
|
+
/** Decoded size the server refuses above. The downscale below keeps a phone
|
|
24
|
+
* photo far under it; this is the belt for a file that is not a photo. */
|
|
25
|
+
exports.TALQING_IMAGE_MAX_BYTES = 10 * 1024 * 1024;
|
|
26
|
+
/** Longest side the server stores. Downscaling to it before sending is a
|
|
27
|
+
* courtesy to the data channel, which is ordered and reliable and will
|
|
28
|
+
* happily spend a minute pushing 10 MB past a call's audio. */
|
|
29
|
+
exports.TALQING_IMAGE_MAX_EDGE_PX = 1568;
|
|
30
|
+
const TALQING_IMAGE_TYPES = ["image/jpeg", "image/png", "image/webp"];
|
|
31
|
+
/** How long to wait for the agent's ack before saying we do not know. Without
|
|
32
|
+
* it a thumbnail sits on "sending" for the rest of the call when the worker
|
|
33
|
+
* dies mid-upload. */
|
|
34
|
+
exports.TALQING_IMAGE_ACK_TIMEOUT_MS = 30000;
|
|
35
|
+
/** Refuse what the browser can see is wrong, instantly and with no round trip.
|
|
36
|
+
* Returns the sentence to show, or null when the file is fine. The server
|
|
37
|
+
* re-validates everything regardless — this exists so the common mistakes cost
|
|
38
|
+
* nothing to catch. */
|
|
39
|
+
function talqingImageRejection(file) {
|
|
40
|
+
if (!TALQING_IMAGE_TYPES.includes(file.type)) {
|
|
41
|
+
return "that file is not a JPEG, PNG or WebP";
|
|
42
|
+
}
|
|
43
|
+
if (file.size > exports.TALQING_IMAGE_MAX_BYTES)
|
|
44
|
+
return "images are 10 MB or smaller";
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
/** Downscale to the longest edge the server keeps, re-encoding as JPEG or PNG.
|
|
48
|
+
*
|
|
49
|
+
* Also what normalizes whatever the OS handed the file input — this is how a
|
|
50
|
+
* Safari HEIC becomes a JPEG. Throws if the image cannot be decoded, which is
|
|
51
|
+
* the third dropzone check. */
|
|
52
|
+
async function talqingDownscaleImage(file) {
|
|
53
|
+
const bitmap = await createImageBitmap(file);
|
|
54
|
+
try {
|
|
55
|
+
const longest = Math.max(bitmap.width, bitmap.height);
|
|
56
|
+
const scale = longest > exports.TALQING_IMAGE_MAX_EDGE_PX ? exports.TALQING_IMAGE_MAX_EDGE_PX / longest : 1;
|
|
57
|
+
const canvas = document.createElement("canvas");
|
|
58
|
+
canvas.width = Math.max(1, Math.round(bitmap.width * scale));
|
|
59
|
+
canvas.height = Math.max(1, Math.round(bitmap.height * scale));
|
|
60
|
+
const context = canvas.getContext("2d");
|
|
61
|
+
if (!context)
|
|
62
|
+
throw new Error("this browser cannot resize images");
|
|
63
|
+
context.drawImage(bitmap, 0, 0, canvas.width, canvas.height);
|
|
64
|
+
// PNG keeps transparency; everything else becomes a JPEG, including the
|
|
65
|
+
// HEIC the OS decoded for us.
|
|
66
|
+
const type = file.type === "image/png" ? "image/png" : "image/jpeg";
|
|
67
|
+
const blob = await new Promise((resolve) => canvas.toBlob(resolve, type, 0.85));
|
|
68
|
+
if (!blob)
|
|
69
|
+
throw new Error("that image could not be read");
|
|
70
|
+
return blob;
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
bitmap.close();
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** Validate and downscale a file into the `data:` URL a text message carries.
|
|
77
|
+
*
|
|
78
|
+
* `createTextMessage` takes `images: [{data_url, filename}]`; this produces the
|
|
79
|
+
* `data_url`. It doubles as the thumbnail's `src`, so there is no object URL to
|
|
80
|
+
* create and revoke. Throws the sentence to show if the file is refused. */
|
|
81
|
+
async function talqingImageDataUrl(file) {
|
|
82
|
+
const rejection = talqingImageRejection(file);
|
|
83
|
+
if (rejection)
|
|
84
|
+
throw new Error(rejection);
|
|
85
|
+
const blob = await talqingDownscaleImage(file);
|
|
86
|
+
return await new Promise((resolve, reject) => {
|
|
87
|
+
const reader = new FileReader();
|
|
88
|
+
reader.onerror = () => reject(new Error("that image could not be read"));
|
|
89
|
+
reader.onload = () => resolve(String(reader.result));
|
|
90
|
+
reader.readAsDataURL(blob);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
/** Validate, downscale and send one image to the agent on this call.
|
|
94
|
+
*
|
|
95
|
+
* Resolves once the bytes are on their way, with the entry to render. The
|
|
96
|
+
* outcome arrives later, on the ack — see `onTalqingImageResult`. Throws the
|
|
97
|
+
* sentence to show if the file is refused or the agent is not there yet. */
|
|
98
|
+
async function sendTalqingImage(room, file) {
|
|
99
|
+
const agent = (0, userdata_js_1.talqingAgentParticipant)(room);
|
|
100
|
+
if (!agent)
|
|
101
|
+
throw new Error("the agent is not on the call yet");
|
|
102
|
+
const rejection = talqingImageRejection(file);
|
|
103
|
+
if (rejection)
|
|
104
|
+
throw new Error(rejection);
|
|
105
|
+
const blob = await talqingDownscaleImage(file);
|
|
106
|
+
// Addressed to the agent, never broadcast: on video the avatar worker is
|
|
107
|
+
// a participant too, and a broadcast sends the bytes twice.
|
|
108
|
+
const { id } = await room.localParticipant.sendFile(new File([blob], file.name, { type: blob.type }), {
|
|
109
|
+
topic: exports.TALQING_IMAGE_TOPIC,
|
|
110
|
+
mimeType: blob.type,
|
|
111
|
+
destinationIdentities: [agent.identity],
|
|
112
|
+
});
|
|
113
|
+
return {
|
|
114
|
+
streamId: id,
|
|
115
|
+
name: file.name,
|
|
116
|
+
previewUrl: URL.createObjectURL(blob),
|
|
117
|
+
sentAt: Date.now(),
|
|
118
|
+
status: "sending",
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
/** Listen for the agent's verdict on each image sent. Returns the unregister.
|
|
122
|
+
*
|
|
123
|
+
* There is one ack per image even on the happy path, because the agent stays
|
|
124
|
+
* silent when a photo lands — so silence cannot double as "it arrived". */
|
|
125
|
+
function onTalqingImageResult(room, onResult) {
|
|
126
|
+
room.registerRpcMethod("talqing.image_result", async (data) => {
|
|
127
|
+
const { stream_id, ok, error } = JSON.parse(data.payload || "{}");
|
|
128
|
+
onResult({
|
|
129
|
+
streamId: stream_id,
|
|
130
|
+
ok,
|
|
131
|
+
error: ok ? undefined : error || "that image was not accepted",
|
|
132
|
+
});
|
|
133
|
+
return "{}";
|
|
134
|
+
});
|
|
135
|
+
return () => room.unregisterRpcMethod("talqing.image_result");
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=images.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"images.js","sourceRoot":"","sources":["../../src/images.ts"],"names":[],"mappings":";;;AAkCA,sDAMC;AAOD,sDAsBC;AAOD,kDAUC;AAyBD,4CAwBC;AAMD,oDAkBC;AA9JD,+CAAwD;AAExD;;;;;;;;;;;GAWG;AAEU,QAAA,mBAAmB,GAAG,gBAAgB,CAAC;AACpD;2EAC2E;AAC9D,QAAA,uBAAuB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AACxD;;gEAEgE;AACnD,QAAA,yBAAyB,GAAG,IAAI,CAAC;AAC9C,MAAM,mBAAmB,GAAG,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAU,CAAC;AAC/E;;uBAEuB;AACV,QAAA,4BAA4B,GAAG,KAAM,CAAC;AAEnD;;;wBAGwB;AACxB,SAAgB,qBAAqB,CAAC,IAAU;IAC9C,IAAI,CAAE,mBAAyC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACpE,OAAO,sCAAsC,CAAC;IAChD,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,GAAG,+BAAuB;QAAE,OAAO,6BAA6B,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;gCAIgC;AACzB,KAAK,UAAU,qBAAqB,CAAC,IAAU;IACpD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,OAAO,GAAG,iCAAyB,CAAC,CAAC,CAAC,iCAAyB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAChD,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnE,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7D,wEAAwE;QACxE,8BAA8B;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAc,CAAC,OAAO,EAAE,EAAE,CACtD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CACnC,CAAC;QACF,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;;;6EAI6E;AACtE,KAAK,UAAU,mBAAmB,CAAC,IAAU;IAClD,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAC1C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,MAAM,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnD,MAAM,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAChC,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACzE,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACrD,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;AACL,CAAC;AAoBD;;;;6EAI6E;AACtE,KAAK,UAAU,gBAAgB,CAAC,IAAU,EAAE,IAAU;IAC3D,MAAM,KAAK,GAAG,IAAA,qCAAuB,EAAC,IAAI,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAE1C,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC/C,yEAAyE;IACzE,4DAA4D;IAC5D,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CACjD,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,EAChD;QACE,KAAK,EAAE,2BAAmB;QAC1B,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,qBAAqB,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC;KACxC,CACF,CAAC;IACF,OAAO;QACL,QAAQ,EAAE,EAAE;QACZ,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,UAAU,EAAE,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC;QACrC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE;QAClB,MAAM,EAAE,SAAS;KAClB,CAAC;AACJ,CAAC;AAED;;;4EAG4E;AAC5E,SAAgB,oBAAoB,CAClC,IAAU,EACV,QAA6E;IAE7E,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,IAAuB,EAAE,EAAE;QAC/E,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAI/D,CAAC;QACF,QAAQ,CAAC;YACP,QAAQ,EAAE,SAAS;YACnB,EAAE;YACF,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,6BAA6B;SAC/D,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export { createTalqingTokenSource } from "./token.js";
|
|
2
|
+
export type { TalqingCallCredentials } from "./token.js";
|
|
3
|
+
export { getTalqingUserdata, setTalqingUserdata, talqingAgentParticipant, } from "./userdata.js";
|
|
4
|
+
export { onTalqingImageResult, sendTalqingImage, talqingDownscaleImage, talqingImageDataUrl, talqingImageRejection, TALQING_IMAGE_ACK_TIMEOUT_MS, TALQING_IMAGE_MAX_BYTES, TALQING_IMAGE_MAX_EDGE_PX, TALQING_IMAGE_TOPIC, } from "./images.js";
|
|
5
|
+
export type { TalqingImageStatus, TalqingSentImage } from "./images.js";
|
|
6
|
+
export { registerTalqingRpcHandlers, TalqingRpcError } from "./rpc.js";
|
|
7
|
+
export type { TalqingFrontendRpcEnvelope, TalqingRpcHandler, TalqingRpcHandlers, TalqingRpcInvocation, } from "./rpc.js";
|
|
8
|
+
export { TALQING_AUDIO_CAPTURE_DEFAULTS, TALQING_SCREEN_SHARE_CAPTURE, TALQING_SCREEN_SHARE_PUBLISH, } from "./media.js";
|
|
9
|
+
export type { JsonObject, JsonPrimitive, JsonValue } from "./json.js";
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAC;AACtD,YAAY,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAEzD,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,eAAe,CAAC;AAEvB,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACrB,mBAAmB,EACnB,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,EACvB,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,aAAa,CAAC;AACrB,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAExE,OAAO,EAAE,0BAA0B,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACvE,YAAY,EACV,0BAA0B,EAC1B,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,UAAU,CAAC;AAElB,OAAO,EACL,8BAA8B,EAC9B,4BAA4B,EAC5B,4BAA4B,GAC7B,MAAM,YAAY,CAAC;AAEpB,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/* The Talqing call protocol, over a LiveKit `Room`. Framework-free.
|
|
3
|
+
*
|
|
4
|
+
* Everything here is something only Talqing can write: the mint-once token
|
|
5
|
+
* source, the two userdata RPC methods, the `talqing.images` byte-stream topic
|
|
6
|
+
* and its ack, the `talqing.frontend_rpc` envelope, and the measured capture
|
|
7
|
+
* settings. Nothing here renames LiveKit — for a room, a track or a
|
|
8
|
+
* participant, use `livekit-client` directly.
|
|
9
|
+
*
|
|
10
|
+
* React bindings for all of it are `@talqing/react`.
|
|
11
|
+
*/
|
|
12
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
13
|
+
exports.TALQING_SCREEN_SHARE_PUBLISH = exports.TALQING_SCREEN_SHARE_CAPTURE = exports.TALQING_AUDIO_CAPTURE_DEFAULTS = exports.TalqingRpcError = exports.registerTalqingRpcHandlers = exports.TALQING_IMAGE_TOPIC = exports.TALQING_IMAGE_MAX_EDGE_PX = exports.TALQING_IMAGE_MAX_BYTES = exports.TALQING_IMAGE_ACK_TIMEOUT_MS = exports.talqingImageRejection = exports.talqingImageDataUrl = exports.talqingDownscaleImage = exports.sendTalqingImage = exports.onTalqingImageResult = exports.talqingAgentParticipant = exports.setTalqingUserdata = exports.getTalqingUserdata = exports.createTalqingTokenSource = void 0;
|
|
14
|
+
var token_js_1 = require("./token.js");
|
|
15
|
+
Object.defineProperty(exports, "createTalqingTokenSource", { enumerable: true, get: function () { return token_js_1.createTalqingTokenSource; } });
|
|
16
|
+
var userdata_js_1 = require("./userdata.js");
|
|
17
|
+
Object.defineProperty(exports, "getTalqingUserdata", { enumerable: true, get: function () { return userdata_js_1.getTalqingUserdata; } });
|
|
18
|
+
Object.defineProperty(exports, "setTalqingUserdata", { enumerable: true, get: function () { return userdata_js_1.setTalqingUserdata; } });
|
|
19
|
+
Object.defineProperty(exports, "talqingAgentParticipant", { enumerable: true, get: function () { return userdata_js_1.talqingAgentParticipant; } });
|
|
20
|
+
var images_js_1 = require("./images.js");
|
|
21
|
+
Object.defineProperty(exports, "onTalqingImageResult", { enumerable: true, get: function () { return images_js_1.onTalqingImageResult; } });
|
|
22
|
+
Object.defineProperty(exports, "sendTalqingImage", { enumerable: true, get: function () { return images_js_1.sendTalqingImage; } });
|
|
23
|
+
Object.defineProperty(exports, "talqingDownscaleImage", { enumerable: true, get: function () { return images_js_1.talqingDownscaleImage; } });
|
|
24
|
+
Object.defineProperty(exports, "talqingImageDataUrl", { enumerable: true, get: function () { return images_js_1.talqingImageDataUrl; } });
|
|
25
|
+
Object.defineProperty(exports, "talqingImageRejection", { enumerable: true, get: function () { return images_js_1.talqingImageRejection; } });
|
|
26
|
+
Object.defineProperty(exports, "TALQING_IMAGE_ACK_TIMEOUT_MS", { enumerable: true, get: function () { return images_js_1.TALQING_IMAGE_ACK_TIMEOUT_MS; } });
|
|
27
|
+
Object.defineProperty(exports, "TALQING_IMAGE_MAX_BYTES", { enumerable: true, get: function () { return images_js_1.TALQING_IMAGE_MAX_BYTES; } });
|
|
28
|
+
Object.defineProperty(exports, "TALQING_IMAGE_MAX_EDGE_PX", { enumerable: true, get: function () { return images_js_1.TALQING_IMAGE_MAX_EDGE_PX; } });
|
|
29
|
+
Object.defineProperty(exports, "TALQING_IMAGE_TOPIC", { enumerable: true, get: function () { return images_js_1.TALQING_IMAGE_TOPIC; } });
|
|
30
|
+
var rpc_js_1 = require("./rpc.js");
|
|
31
|
+
Object.defineProperty(exports, "registerTalqingRpcHandlers", { enumerable: true, get: function () { return rpc_js_1.registerTalqingRpcHandlers; } });
|
|
32
|
+
Object.defineProperty(exports, "TalqingRpcError", { enumerable: true, get: function () { return rpc_js_1.TalqingRpcError; } });
|
|
33
|
+
var media_js_1 = require("./media.js");
|
|
34
|
+
Object.defineProperty(exports, "TALQING_AUDIO_CAPTURE_DEFAULTS", { enumerable: true, get: function () { return media_js_1.TALQING_AUDIO_CAPTURE_DEFAULTS; } });
|
|
35
|
+
Object.defineProperty(exports, "TALQING_SCREEN_SHARE_CAPTURE", { enumerable: true, get: function () { return media_js_1.TALQING_SCREEN_SHARE_CAPTURE; } });
|
|
36
|
+
Object.defineProperty(exports, "TALQING_SCREEN_SHARE_PUBLISH", { enumerable: true, get: function () { return media_js_1.TALQING_SCREEN_SHARE_PUBLISH; } });
|
|
37
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;;GASG;;;AAEH,uCAAsD;AAA7C,oHAAA,wBAAwB,OAAA;AAGjC,6CAIuB;AAHrB,iHAAA,kBAAkB,OAAA;AAClB,iHAAA,kBAAkB,OAAA;AAClB,sHAAA,uBAAuB,OAAA;AAGzB,yCAUqB;AATnB,iHAAA,oBAAoB,OAAA;AACpB,6GAAA,gBAAgB,OAAA;AAChB,kHAAA,qBAAqB,OAAA;AACrB,gHAAA,mBAAmB,OAAA;AACnB,kHAAA,qBAAqB,OAAA;AACrB,yHAAA,4BAA4B,OAAA;AAC5B,oHAAA,uBAAuB,OAAA;AACvB,sHAAA,yBAAyB,OAAA;AACzB,gHAAA,mBAAmB,OAAA;AAIrB,mCAAuE;AAA9D,oHAAA,0BAA0B,OAAA;AAAE,yGAAA,eAAe,OAAA;AAQpD,uCAIoB;AAHlB,0HAAA,8BAA8B,OAAA;AAC9B,wHAAA,4BAA4B,OAAA;AAC5B,wHAAA,4BAA4B,OAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../src/json.ts"],"names":[],"mappings":"AAcA,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,SAAS,EAAE,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AACnF,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CAAE,CAAC"}
|
package/dist/cjs/json.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"json.js","sourceRoot":"","sources":["../../src/json.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ScreenShareCaptureOptions, TrackPublishOptions } from "livekit-client";
|
|
2
|
+
export declare const TALQING_AUDIO_CAPTURE_DEFAULTS: {
|
|
3
|
+
readonly echoCancellation: true;
|
|
4
|
+
readonly noiseSuppression: true;
|
|
5
|
+
readonly autoGainControl: true;
|
|
6
|
+
};
|
|
7
|
+
export declare const TALQING_SCREEN_SHARE_CAPTURE: ScreenShareCaptureOptions;
|
|
8
|
+
export declare const TALQING_SCREEN_SHARE_PUBLISH: TrackPublishOptions;
|
|
9
|
+
//# sourceMappingURL=media.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"media.d.ts","sourceRoot":"","sources":["../../src/media.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAOrF,eAAO,MAAM,8BAA8B;;;;CAIjC,CAAC;AAiCX,eAAO,MAAM,4BAA4B,EAAE,yBAG1C,CAAC;AAEF,eAAO,MAAM,4BAA4B,EAAE,mBAG1C,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TALQING_SCREEN_SHARE_PUBLISH = exports.TALQING_SCREEN_SHARE_CAPTURE = exports.TALQING_AUDIO_CAPTURE_DEFAULTS = void 0;
|
|
4
|
+
/* Echo cancellation is what stops the agent hearing its own speech through the
|
|
5
|
+
* caller's microphone and interrupting itself. It is a default rather than
|
|
6
|
+
* something each app remembers, because a call without it is broken in a way
|
|
7
|
+
* that sounds like a bad model. Pass `roomOptions.audioCaptureDefaults` to
|
|
8
|
+
* override. */
|
|
9
|
+
exports.TALQING_AUDIO_CAPTURE_DEFAULTS = {
|
|
10
|
+
echoCancellation: true,
|
|
11
|
+
noiseSuppression: true,
|
|
12
|
+
autoGainControl: true,
|
|
13
|
+
};
|
|
14
|
+
/* Screen share, with the four settings that decide whether the agent can read
|
|
15
|
+
* what is on it.
|
|
16
|
+
*
|
|
17
|
+
* Only the first is obvious, and every one of them is upstream of anything we
|
|
18
|
+
* could do on our side:
|
|
19
|
+
*
|
|
20
|
+
* - `contentHint: "text"` tells the encoder to preserve spatial detail. The
|
|
21
|
+
* default for a screen share on an SVC codec is `motion`, which trades
|
|
22
|
+
* exactly that away — the wrong trade for reading code, and no JPEG quality
|
|
23
|
+
* setting downstream recovers it.
|
|
24
|
+
* - `videoCodec: "h264"` is what keeps that hint. `livekit-client` forces
|
|
25
|
+
* `contentHint = "motion"` on screen shares published as vp9 or av1,
|
|
26
|
+
* together with `scalabilityMode: "L1T3"`. **Publishing on an SVC codec will
|
|
27
|
+
* cost you legibility**, and it is the first thing to check if the agent
|
|
28
|
+
* starts misreading identifiers.
|
|
29
|
+
* - `screenShareEncoding.maxFramerate: 1` is the frame-rate cap that actually
|
|
30
|
+
* binds. `resolution.frameRate` reaches `getDisplayMedia` as a bare number,
|
|
31
|
+
* which WebRTC reads as `ideal` — a wish, not a limit. The default publish
|
|
32
|
+
* ceiling is 15 fps, and every delivered frame is copied into the agent's
|
|
33
|
+
* process before it can be dropped, so this is the single biggest lever on
|
|
34
|
+
* what a call costs the worker. One frame a second is ample: the agent is
|
|
35
|
+
* handed the newest frame at the end of each turn, and turns are seconds
|
|
36
|
+
* apart.
|
|
37
|
+
* - `resolution` is set explicitly rather than inherited. The default is 1080p
|
|
38
|
+
* by `ideal`, which is a strong hint and not a guarantee.
|
|
39
|
+
*
|
|
40
|
+
* The picker cannot be opened for the person — `getDisplayMedia` requires a
|
|
41
|
+
* transient user activation — so sharing must start from a click. The agent
|
|
42
|
+
* asking out loud is what prompts it; there is no version of this where the
|
|
43
|
+
* agent turns sharing on.
|
|
44
|
+
*/
|
|
45
|
+
exports.TALQING_SCREEN_SHARE_CAPTURE = {
|
|
46
|
+
resolution: { width: 1920, height: 1080, frameRate: 1 },
|
|
47
|
+
contentHint: "text",
|
|
48
|
+
};
|
|
49
|
+
exports.TALQING_SCREEN_SHARE_PUBLISH = {
|
|
50
|
+
videoCodec: "h264",
|
|
51
|
+
screenShareEncoding: { maxBitrate: 1000000, maxFramerate: 1 },
|
|
52
|
+
};
|
|
53
|
+
//# sourceMappingURL=media.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"media.js","sourceRoot":"","sources":["../../src/media.ts"],"names":[],"mappings":";;;AAEA;;;;eAIe;AACF,QAAA,8BAA8B,GAAG;IAC5C,gBAAgB,EAAE,IAAI;IACtB,gBAAgB,EAAE,IAAI;IACtB,eAAe,EAAE,IAAI;CACb,CAAC;AAEX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACU,QAAA,4BAA4B,GAA8B;IACrE,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EAAE;IACvD,WAAW,EAAE,MAAM;CACpB,CAAC;AAEW,QAAA,4BAA4B,GAAwB;IAC/D,UAAU,EAAE,MAAM;IAClB,mBAAmB,EAAE,EAAE,UAAU,EAAE,OAAS,EAAE,YAAY,EAAE,CAAC,EAAE;CAChE,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type Room } from "livekit-client";
|
|
2
|
+
import type { JsonValue } from "./json.js";
|
|
3
|
+
export interface TalqingRpcInvocation {
|
|
4
|
+
payload: string;
|
|
5
|
+
responseTimeout?: number;
|
|
6
|
+
callerIdentity?: string;
|
|
7
|
+
}
|
|
8
|
+
export type TalqingRpcHandler = (data: TalqingRpcInvocation) => string | Promise<string>;
|
|
9
|
+
export type TalqingRpcHandlers = Record<string, TalqingRpcHandler>;
|
|
10
|
+
export interface TalqingFrontendRpcEnvelope {
|
|
11
|
+
method: string;
|
|
12
|
+
payload?: JsonValue;
|
|
13
|
+
}
|
|
14
|
+
/** Throw this from a handler to choose the code the agent sees. */
|
|
15
|
+
export declare class TalqingRpcError extends Error {
|
|
16
|
+
readonly code: number;
|
|
17
|
+
readonly data?: string;
|
|
18
|
+
constructor(code: number, message: string, data?: string);
|
|
19
|
+
}
|
|
20
|
+
/** Register the frontend-RPC envelope handler on a room. Returns the unregister.
|
|
21
|
+
*
|
|
22
|
+
* A `"*"` key catches every method with no handler of its own, and receives the
|
|
23
|
+
* WHOLE envelope as its payload string so it can tell what it caught. */
|
|
24
|
+
export declare function registerTalqingRpcHandlers(room: Room, handlers: TalqingRpcHandlers): () => void;
|
|
25
|
+
//# sourceMappingURL=rpc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc.d.ts","sourceRoot":"","sources":["../../src/rpc.ts"],"names":[],"mappings":"AAAA,OAAO,EAAY,KAAK,IAAI,EAA0B,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAU3C,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,iBAAiB,GAAG,CAC9B,IAAI,EAAE,oBAAoB,KACvB,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;AAC9B,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;AAEnE,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,SAAS,CAAC;CACrB;AAED,mEAAmE;AACnE,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAEX,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM;CAMzD;AAED;;;0EAG0E;AAC1E,wBAAgB,0BAA0B,CACxC,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,kBAAkB,GAC3B,MAAM,IAAI,CAiCZ"}
|
package/dist/cjs/rpc.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TalqingRpcError = void 0;
|
|
4
|
+
exports.registerTalqingRpcHandlers = registerTalqingRpcHandlers;
|
|
5
|
+
const livekit_client_1 = require("livekit-client");
|
|
6
|
+
/** Throw this from a handler to choose the code the agent sees. */
|
|
7
|
+
class TalqingRpcError extends Error {
|
|
8
|
+
constructor(code, message, data) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "TalqingRpcError";
|
|
11
|
+
this.code = code;
|
|
12
|
+
this.data = data;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
exports.TalqingRpcError = TalqingRpcError;
|
|
16
|
+
/** Register the frontend-RPC envelope handler on a room. Returns the unregister.
|
|
17
|
+
*
|
|
18
|
+
* A `"*"` key catches every method with no handler of its own, and receives the
|
|
19
|
+
* WHOLE envelope as its payload string so it can tell what it caught. */
|
|
20
|
+
function registerTalqingRpcHandlers(room, handlers) {
|
|
21
|
+
room.registerRpcMethod("talqing.frontend_rpc", async (data) => {
|
|
22
|
+
try {
|
|
23
|
+
const envelope = JSON.parse(data.payload || "{}");
|
|
24
|
+
const isWildcard = handlers[envelope.method] === undefined;
|
|
25
|
+
const handler = handlers[envelope.method] ?? handlers["*"];
|
|
26
|
+
if (!handler) {
|
|
27
|
+
throw new TalqingRpcError(1404, `No frontend RPC handler registered for ${envelope.method}`);
|
|
28
|
+
}
|
|
29
|
+
return await handler({
|
|
30
|
+
payload: isWildcard
|
|
31
|
+
? JSON.stringify(envelope)
|
|
32
|
+
: envelope.payload === undefined
|
|
33
|
+
? "{}"
|
|
34
|
+
: JSON.stringify(envelope.payload),
|
|
35
|
+
responseTimeout: data.responseTimeout,
|
|
36
|
+
callerIdentity: data.callerIdentity,
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
if (error instanceof TalqingRpcError) {
|
|
41
|
+
throw new livekit_client_1.RpcError(error.code, error.message, error.data);
|
|
42
|
+
}
|
|
43
|
+
throw new livekit_client_1.RpcError(1500, error instanceof Error ? error.message : "RPC handler failed");
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
return () => room.unregisterRpcMethod("talqing.frontend_rpc");
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=rpc.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rpc.js","sourceRoot":"","sources":["../../src/rpc.ts"],"names":[],"mappings":";;;AA4CA,gEAoCC;AAhFD,mDAA6E;AA2B7E,mEAAmE;AACnE,MAAa,eAAgB,SAAQ,KAAK;IAIxC,YAAY,IAAY,EAAE,OAAe,EAAE,IAAa;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;CACF;AAVD,0CAUC;AAED;;;0EAG0E;AAC1E,SAAgB,0BAA0B,CACxC,IAAU,EACV,QAA4B;IAE5B,IAAI,CAAC,iBAAiB,CAAC,sBAAsB,EAAE,KAAK,EAAE,IAAuB,EAAE,EAAE;QAC/E,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAA+B,CAAC;YAChF,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,SAAS,CAAC;YAC3D,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC;YAC3D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,eAAe,CACvB,IAAI,EACJ,0CAA0C,QAAQ,CAAC,MAAM,EAAE,CAC5D,CAAC;YACJ,CAAC;YACD,OAAO,MAAM,OAAO,CAAC;gBACnB,OAAO,EAAE,UAAU;oBACjB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;oBAC1B,CAAC,CAAC,QAAQ,CAAC,OAAO,KAAK,SAAS;wBAC9B,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,CAAC;gBACtC,eAAe,EAAE,IAAI,CAAC,eAAe;gBACrC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,eAAe,EAAE,CAAC;gBACrC,MAAM,IAAI,yBAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,IAAI,yBAAQ,CAChB,IAAI,EACJ,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,CAAC,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type TokenSourceFixed } from "livekit-client";
|
|
2
|
+
/** The two fields a browser needs to join a room, and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* Named as `POST /v1/calls/token` names them, so a server that mints one can
|
|
5
|
+
* forward the response through to the browser without renaming anything. */
|
|
6
|
+
export interface TalqingCallCredentials {
|
|
7
|
+
server_url: string;
|
|
8
|
+
participant_token: string;
|
|
9
|
+
}
|
|
10
|
+
/** Turn a one-shot mint into the token source LiveKit's session takes.
|
|
11
|
+
*
|
|
12
|
+
* `fetchToken` is called at most once per source, no matter how often LiveKit
|
|
13
|
+
* asks. That is load-bearing rather than an optimization: minting is not free,
|
|
14
|
+
* because `POST /v1/calls/token` files the session row the call is later read
|
|
15
|
+
* back from. The framework asks three times per call — to warm the connection,
|
|
16
|
+
* to connect, and once more as the call ends to drop what it assumes is a
|
|
17
|
+
* cached token — and only the middle one is a call anybody placed. Minting for
|
|
18
|
+
* the third filed a session nobody ever joined, which sat in the workspace's
|
|
19
|
+
* call list as queued for good.
|
|
20
|
+
*
|
|
21
|
+
* `TokenSource.literal` is the LiveKit variant that leaves caching to us, and
|
|
22
|
+
* this is us doing it.
|
|
23
|
+
*
|
|
24
|
+
* Freshness comes from a new source per call, never from re-minting inside
|
|
25
|
+
* one: the token names the room, and this call has exactly one. Mount the
|
|
26
|
+
* session per call and end it on unmount, as LiveKit's own session lifecycle
|
|
27
|
+
* does, and that holds by construction. */
|
|
28
|
+
export declare function createTalqingTokenSource(fetchToken: () => Promise<TalqingCallCredentials>): TokenSourceFixed;
|
|
29
|
+
//# sourceMappingURL=token.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../src/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEpE;;;6EAG6E;AAC7E,MAAM,WAAW,sBAAsB;IACrC,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;4CAiB4C;AAC5C,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,MAAM,OAAO,CAAC,sBAAsB,CAAC,GAChD,gBAAgB,CAiBlB"}
|