@npcify/web 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +77 -0
- package/dist/index.d.ts +301 -16
- package/dist/index.js +1506 -240
- package/examples/index.html +1 -0
- package/examples/main.js +22 -0
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,3 +1,36 @@
|
|
|
1
|
+
// ../ts-shared/src/motion-frames.ts
|
|
2
|
+
var MOTION_FRAMES_CAPABILITY = "motion_frames";
|
|
3
|
+
var motionFrameWidth = function(boneCount) {
|
|
4
|
+
return 3 + 4 * boneCount;
|
|
5
|
+
};
|
|
6
|
+
var MAX_MOTION_FRAMES_PER_EVENT = 3600;
|
|
7
|
+
var MAX_MOTION_BONES = 64;
|
|
8
|
+
var isCount = function(value, max2) {
|
|
9
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 && value <= max2;
|
|
10
|
+
};
|
|
11
|
+
var readMotionFrames = function(data, frameCount, boneCount) {
|
|
12
|
+
const values = frameCount * motionFrameWidth(boneCount);
|
|
13
|
+
if (data.byteLength !== values * 4) {
|
|
14
|
+
throw new Error("Motion frames do not match their frame count.");
|
|
15
|
+
}
|
|
16
|
+
const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
|
|
17
|
+
const frames = new Float32Array(values);
|
|
18
|
+
for (let index = 0; index < values; index += 1) {
|
|
19
|
+
frames[index] = view.getFloat32(index * 4, true);
|
|
20
|
+
}
|
|
21
|
+
return frames;
|
|
22
|
+
};
|
|
23
|
+
var isValidMotionFramesEvent = function(event) {
|
|
24
|
+
if (!event || typeof event !== "object") {
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
const value = event;
|
|
28
|
+
const bones = value.bones;
|
|
29
|
+
const frameCount = value.frameCount;
|
|
30
|
+
const frames = value.frames;
|
|
31
|
+
return value.type === "motion_frames" && typeof value.taskId === "string" && typeof value.stagingId === "string" && isCount(value.revision, Number.MAX_SAFE_INTEGER) && typeof value.fps === "number" && value.fps > 0 && value.fps <= 240 && isCount(value.startFrame, 24 * 60 * 60 * 240) && isCount(frameCount, MAX_MOTION_FRAMES_PER_EVENT) && Array.isArray(bones) && bones.length > 0 && bones.length <= MAX_MOTION_BONES && bones.every((bone) => typeof bone === "string") && typeof value.restHipsHeight === "number" && value.restHipsHeight > 0 && Number.isFinite(value.restHipsHeight) && frames instanceof Uint8Array && frames.byteLength === frameCount * motionFrameWidth(bones.length) * 4;
|
|
32
|
+
};
|
|
33
|
+
|
|
1
34
|
// ../ts-shared/src/llm.ts
|
|
2
35
|
var STAGING_PROTOCOL_VERSION = 1;
|
|
3
36
|
|
|
@@ -55,6 +88,20 @@ var AI_ROLES = {
|
|
|
55
88
|
models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
56
89
|
options: []
|
|
57
90
|
},
|
|
91
|
+
// Sight (docs/sight.md): one role for each provider that an owner can
|
|
92
|
+
// choose. Priced by VISION_PRICES of the server.
|
|
93
|
+
vision_gemini: {
|
|
94
|
+
name: "Gemini sight",
|
|
95
|
+
source: "gemini",
|
|
96
|
+
models: ["gemini-3.5-flash-lite"],
|
|
97
|
+
options: []
|
|
98
|
+
},
|
|
99
|
+
vision_openai: {
|
|
100
|
+
name: "OpenAI sight",
|
|
101
|
+
source: "openai",
|
|
102
|
+
models: ["gpt-6-luna"],
|
|
103
|
+
options: []
|
|
104
|
+
},
|
|
58
105
|
// Priced by OPENAI_PRICES.text of the generator.
|
|
59
106
|
generator_agent: {
|
|
60
107
|
name: "Generator agent",
|
|
@@ -209,6 +256,43 @@ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
|
|
|
209
256
|
}
|
|
210
257
|
});
|
|
211
258
|
|
|
259
|
+
// ../ts-shared/src/sight.ts
|
|
260
|
+
var SIGHT_EVENTS = Object.freeze({
|
|
261
|
+
/** Client to server: one JPEG frame, acknowledged with a `SightFrameAck`. */
|
|
262
|
+
frame: "conversations:sight",
|
|
263
|
+
/** Server to the conversation's room: one `SightObservationEvent`. */
|
|
264
|
+
observation: "sight_observation"
|
|
265
|
+
});
|
|
266
|
+
var SIGHT_LIMITS = Object.freeze({
|
|
267
|
+
/** The largest JPEG frame, in bytes (200 KiB). */
|
|
268
|
+
maxFrameBytes: 200 * 1024,
|
|
269
|
+
/** The longest edge of a frame or a crop, in pixels. */
|
|
270
|
+
maxEdgePx: 1280,
|
|
271
|
+
/** The shortest time between two frames of one conversation. */
|
|
272
|
+
minFrameIntervalMs: 1e3,
|
|
273
|
+
maxCrops: 4,
|
|
274
|
+
/** The largest JPEG crop, in bytes (100 KiB). */
|
|
275
|
+
maxCropBytes: 100 * 1024,
|
|
276
|
+
maxCropLabelLength: 40
|
|
277
|
+
});
|
|
278
|
+
var SIGHT_CAPTURE_SIZE = Object.freeze({
|
|
279
|
+
width: 640,
|
|
280
|
+
height: 360,
|
|
281
|
+
jpegQuality: 0.7
|
|
282
|
+
});
|
|
283
|
+
var SIGHT_PROVIDER_PROFILES = Object.freeze({
|
|
284
|
+
gemini: {
|
|
285
|
+
model: "gemini-3.5-flash-lite",
|
|
286
|
+
frameToObservationMs: 1280,
|
|
287
|
+
usdPerFrame: 29e-5
|
|
288
|
+
},
|
|
289
|
+
openai: {
|
|
290
|
+
model: "gpt-6-luna",
|
|
291
|
+
frameToObservationMs: 2140,
|
|
292
|
+
usdPerFrame: 74e-6
|
|
293
|
+
}
|
|
294
|
+
});
|
|
295
|
+
|
|
212
296
|
// ../ts-shared/src/product-usage.ts
|
|
213
297
|
var productUsageEvents = [
|
|
214
298
|
"page_viewed",
|
|
@@ -2602,6 +2686,8 @@ var PERFORMANCE_KINDS = {
|
|
|
2602
2686
|
storedTimeline: 7,
|
|
2603
2687
|
cachedOpening: 8
|
|
2604
2688
|
};
|
|
2689
|
+
var PERFORMANCE_MAJOR_VERSION = 1;
|
|
2690
|
+
var PERFORMANCE_MINOR_VERSION = 1;
|
|
2605
2691
|
var HEADER_SIZE = 24;
|
|
2606
2692
|
var MAGIC = 1346588750;
|
|
2607
2693
|
var crcTable = Uint32Array.from({ length: 256 }, (_, index) => {
|
|
@@ -2673,8 +2759,8 @@ var encodePerformance = function(kind, value, options = {}) {
|
|
|
2673
2759
|
const bytes = new Uint8Array(HEADER_SIZE + body.length);
|
|
2674
2760
|
const header = new DataView(bytes.buffer);
|
|
2675
2761
|
header.setUint32(0, MAGIC, true);
|
|
2676
|
-
header.setUint8(4,
|
|
2677
|
-
header.setUint8(5,
|
|
2762
|
+
header.setUint8(4, PERFORMANCE_MAJOR_VERSION);
|
|
2763
|
+
header.setUint8(5, PERFORMANCE_MINOR_VERSION);
|
|
2678
2764
|
header.setUint8(6, PERFORMANCE_KINDS[kind]);
|
|
2679
2765
|
header.setUint8(7, options.compress ? 1 : 0);
|
|
2680
2766
|
header.setUint16(8, HEADER_SIZE, true);
|
|
@@ -2711,7 +2797,7 @@ var decodePerformance = function(input, expectedKind) {
|
|
|
2711
2797
|
throw new Error("Truncated performance header.");
|
|
2712
2798
|
}
|
|
2713
2799
|
const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
2714
|
-
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !==
|
|
2800
|
+
if (header.getUint32(0, true) !== MAGIC || header.getUint8(4) !== PERFORMANCE_MAJOR_VERSION) {
|
|
2715
2801
|
throw new Error("Unsupported performance format or major version.");
|
|
2716
2802
|
}
|
|
2717
2803
|
if (header.getUint8(6) !== PERFORMANCE_KINDS[expectedKind]) {
|
|
@@ -3166,7 +3252,8 @@ var createConversation = async function({
|
|
|
3166
3252
|
),
|
|
3167
3253
|
credentials: apiToken ? "omit" : "include",
|
|
3168
3254
|
method: "POST",
|
|
3169
|
-
|
|
3255
|
+
// This SDK plays live motion frames (ADR 0011).
|
|
3256
|
+
body: JSON.stringify({ avatarId, capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3170
3257
|
});
|
|
3171
3258
|
if (!res.ok) {
|
|
3172
3259
|
throw Object.assign(new Error(await res.text()), { status: res.status });
|
|
@@ -3219,7 +3306,8 @@ var createStaging = async function({
|
|
|
3219
3306
|
content: message,
|
|
3220
3307
|
silent: silent === true,
|
|
3221
3308
|
incrementalPlayback: true,
|
|
3222
|
-
parentStagingId
|
|
3309
|
+
parentStagingId,
|
|
3310
|
+
capabilities: [MOTION_FRAMES_CAPABILITY]
|
|
3223
3311
|
})
|
|
3224
3312
|
});
|
|
3225
3313
|
} catch (error) {
|
|
@@ -3319,6 +3407,35 @@ var getConversationWorldline = async function(options) {
|
|
|
3319
3407
|
}
|
|
3320
3408
|
return await response.json();
|
|
3321
3409
|
};
|
|
3410
|
+
var warmUpLiveMotion = async function({
|
|
3411
|
+
conversationId,
|
|
3412
|
+
apiToken,
|
|
3413
|
+
baseUrl,
|
|
3414
|
+
renderSessionId
|
|
3415
|
+
}) {
|
|
3416
|
+
const url = new URL(`/conversations/${conversationId}/motion/warm-up`, baseUrl);
|
|
3417
|
+
try {
|
|
3418
|
+
const res = await fetch(url.toString(), {
|
|
3419
|
+
headers: withRenderSession(
|
|
3420
|
+
buildAuthHeaders(apiToken, {
|
|
3421
|
+
json: true,
|
|
3422
|
+
allowEmpty: true
|
|
3423
|
+
}),
|
|
3424
|
+
renderSessionId
|
|
3425
|
+
),
|
|
3426
|
+
credentials: apiToken ? "omit" : "include",
|
|
3427
|
+
method: "POST",
|
|
3428
|
+
body: JSON.stringify({ capabilities: [MOTION_FRAMES_CAPABILITY] })
|
|
3429
|
+
});
|
|
3430
|
+
if (!res.ok) {
|
|
3431
|
+
return false;
|
|
3432
|
+
}
|
|
3433
|
+
const payload = await res.json();
|
|
3434
|
+
return payload.warming === true;
|
|
3435
|
+
} catch {
|
|
3436
|
+
return false;
|
|
3437
|
+
}
|
|
3438
|
+
};
|
|
3322
3439
|
|
|
3323
3440
|
// ../../clients/web-client/src/live-conversation.ts
|
|
3324
3441
|
var LiveConversation = class {
|
|
@@ -6045,6 +6162,10 @@ var StreamPlaybackController = class {
|
|
|
6045
6162
|
get cursor() {
|
|
6046
6163
|
return this.currentCursorValue;
|
|
6047
6164
|
}
|
|
6165
|
+
/** The staging that plays or waits to play, or null. */
|
|
6166
|
+
get activeStagingId() {
|
|
6167
|
+
return this.activePlayback?.stagingId ?? null;
|
|
6168
|
+
}
|
|
6048
6169
|
onStreamEvent(callback) {
|
|
6049
6170
|
return bindCallbackSlot(
|
|
6050
6171
|
() => this.streamEventCallback,
|
|
@@ -6138,6 +6259,10 @@ var StreamPlaybackController = class {
|
|
|
6138
6259
|
this.dependencies.telemetry.recordFirstTextDelta();
|
|
6139
6260
|
return;
|
|
6140
6261
|
}
|
|
6262
|
+
if (event.type === "motion_frames") {
|
|
6263
|
+
this.dependencies.onMotionFrames?.(event);
|
|
6264
|
+
return;
|
|
6265
|
+
}
|
|
6141
6266
|
const player = this.dependencies.getStagingPlayer();
|
|
6142
6267
|
if (event.type === "speech_chunk") {
|
|
6143
6268
|
player.appendSpeechChunk(event.streamId, event.mimeType, event.data);
|
|
@@ -6251,6 +6376,9 @@ var StreamPlaybackController = class {
|
|
|
6251
6376
|
this.activePlayback = { taskId: params.taskId, stagingId: params.stagingId };
|
|
6252
6377
|
this.playedStagingIds.add(params.stagingId);
|
|
6253
6378
|
this.recordPlaybackEvent(params, "requested");
|
|
6379
|
+
if (params.timeline.motion) {
|
|
6380
|
+
this.dependencies.onMotionTrack?.(params.stagingId, params.timeline.motion);
|
|
6381
|
+
}
|
|
6254
6382
|
const existing = this.states.get(params.stagingId);
|
|
6255
6383
|
const state = {
|
|
6256
6384
|
taskId: params.taskId,
|
|
@@ -8260,6 +8388,261 @@ var CharacterSceneRig = class {
|
|
|
8260
8388
|
}
|
|
8261
8389
|
};
|
|
8262
8390
|
|
|
8391
|
+
// ../../clients/web-client/src/character/live-motion.ts
|
|
8392
|
+
import { Quaternion as Quaternion3, Vector3 as Vector37 } from "three";
|
|
8393
|
+
var PARENTS = {
|
|
8394
|
+
spine: "hips",
|
|
8395
|
+
chest: "spine",
|
|
8396
|
+
upperChest: "chest",
|
|
8397
|
+
neck: "upperChest",
|
|
8398
|
+
head: "neck",
|
|
8399
|
+
leftShoulder: "upperChest",
|
|
8400
|
+
leftUpperArm: "leftShoulder",
|
|
8401
|
+
leftLowerArm: "leftUpperArm",
|
|
8402
|
+
leftHand: "leftLowerArm",
|
|
8403
|
+
rightShoulder: "upperChest",
|
|
8404
|
+
rightUpperArm: "rightShoulder",
|
|
8405
|
+
rightLowerArm: "rightUpperArm",
|
|
8406
|
+
rightHand: "rightLowerArm",
|
|
8407
|
+
leftUpperLeg: "hips",
|
|
8408
|
+
leftLowerLeg: "leftUpperLeg",
|
|
8409
|
+
leftFoot: "leftLowerLeg",
|
|
8410
|
+
leftToes: "leftFoot",
|
|
8411
|
+
rightUpperLeg: "hips",
|
|
8412
|
+
rightLowerLeg: "rightUpperLeg",
|
|
8413
|
+
rightFoot: "rightLowerLeg",
|
|
8414
|
+
rightToes: "rightFoot"
|
|
8415
|
+
};
|
|
8416
|
+
var MAX_TRACKS = 4;
|
|
8417
|
+
var approach = function(value, target, delta, seconds) {
|
|
8418
|
+
if (seconds <= 0) {
|
|
8419
|
+
return target;
|
|
8420
|
+
}
|
|
8421
|
+
const step = delta / seconds;
|
|
8422
|
+
return value < target ? Math.min(target, value + step) : Math.max(target, value - step);
|
|
8423
|
+
};
|
|
8424
|
+
var LiveMotionPlayer = class {
|
|
8425
|
+
vrm = null;
|
|
8426
|
+
tracks = /* @__PURE__ */ new Map();
|
|
8427
|
+
rootMode;
|
|
8428
|
+
blendIn;
|
|
8429
|
+
blendOut;
|
|
8430
|
+
weight = 0;
|
|
8431
|
+
/** The last applied pose, to blend out of it after the frames end. */
|
|
8432
|
+
lastPose = null;
|
|
8433
|
+
scratch = {
|
|
8434
|
+
from: new Quaternion3(),
|
|
8435
|
+
to: new Quaternion3(),
|
|
8436
|
+
carry: /* @__PURE__ */ new Map(),
|
|
8437
|
+
position: new Vector37()
|
|
8438
|
+
};
|
|
8439
|
+
constructor(options = {}) {
|
|
8440
|
+
this.rootMode = options.rootMode ?? "anchored";
|
|
8441
|
+
this.blendIn = options.blendInSeconds ?? 0.3;
|
|
8442
|
+
this.blendOut = options.blendOutSeconds ?? 0.5;
|
|
8443
|
+
}
|
|
8444
|
+
setVrm(vrm) {
|
|
8445
|
+
this.vrm = vrm;
|
|
8446
|
+
this.weight = 0;
|
|
8447
|
+
this.lastPose = null;
|
|
8448
|
+
}
|
|
8449
|
+
/** Adds the frames of a `motion_frames` event; invalid events are dropped. */
|
|
8450
|
+
push(received) {
|
|
8451
|
+
const frames = received.frames;
|
|
8452
|
+
const event = frames instanceof ArrayBuffer ? { ...received, frames: new Uint8Array(frames) } : received;
|
|
8453
|
+
if (!isValidMotionFramesEvent(event)) {
|
|
8454
|
+
return false;
|
|
8455
|
+
}
|
|
8456
|
+
const values = readMotionFrames(
|
|
8457
|
+
event.frames,
|
|
8458
|
+
event.frameCount,
|
|
8459
|
+
event.bones.length
|
|
8460
|
+
);
|
|
8461
|
+
const track = this.track(event.stagingId, {
|
|
8462
|
+
fps: event.fps,
|
|
8463
|
+
bones: event.bones,
|
|
8464
|
+
restHipsHeight: event.restHipsHeight
|
|
8465
|
+
});
|
|
8466
|
+
for (let frame = 0; frame < event.frameCount; frame += 1) {
|
|
8467
|
+
const index = event.startFrame + frame;
|
|
8468
|
+
const existing = track.frames.get(index);
|
|
8469
|
+
if (existing && existing.revision > event.revision) {
|
|
8470
|
+
continue;
|
|
8471
|
+
}
|
|
8472
|
+
track.frames.set(index, {
|
|
8473
|
+
revision: event.revision,
|
|
8474
|
+
values: values.subarray(frame * track.width, (frame + 1) * track.width)
|
|
8475
|
+
});
|
|
8476
|
+
}
|
|
8477
|
+
return true;
|
|
8478
|
+
}
|
|
8479
|
+
/** Loads the stored frames of a staging (a replay). */
|
|
8480
|
+
async load(stagingId, track, fetchBytes = async (url) => new Uint8Array(await (await fetch(url)).arrayBuffer())) {
|
|
8481
|
+
if (this.tracks.has(stagingId)) {
|
|
8482
|
+
return;
|
|
8483
|
+
}
|
|
8484
|
+
const data = track.frames.type === "bytes" ? track.frames.data : await fetchBytes(track.frames.url);
|
|
8485
|
+
const values = readMotionFrames(data, track.frameCount, track.bones.length);
|
|
8486
|
+
const stored = this.track(stagingId, track);
|
|
8487
|
+
for (let frame = 0; frame < track.frameCount; frame += 1) {
|
|
8488
|
+
stored.frames.set(frame, {
|
|
8489
|
+
revision: Number.MAX_SAFE_INTEGER,
|
|
8490
|
+
values: values.subarray(
|
|
8491
|
+
frame * stored.width,
|
|
8492
|
+
(frame + 1) * stored.width
|
|
8493
|
+
)
|
|
8494
|
+
});
|
|
8495
|
+
}
|
|
8496
|
+
}
|
|
8497
|
+
has(stagingId) {
|
|
8498
|
+
return (this.tracks.get(stagingId)?.frames.size ?? 0) > 0;
|
|
8499
|
+
}
|
|
8500
|
+
forget(stagingId) {
|
|
8501
|
+
this.tracks.delete(stagingId);
|
|
8502
|
+
}
|
|
8503
|
+
/** The current live weight, 0 (idle) to 1 (live frames). */
|
|
8504
|
+
get liveWeight() {
|
|
8505
|
+
return this.weight;
|
|
8506
|
+
}
|
|
8507
|
+
/**
|
|
8508
|
+
* Applies the live pose of `stagingId` at `time` seconds of its staging
|
|
8509
|
+
* clock; `time` null means that no staging plays now.
|
|
8510
|
+
*/
|
|
8511
|
+
update(delta, stagingId, time) {
|
|
8512
|
+
const vrm = this.vrm;
|
|
8513
|
+
if (!vrm?.humanoid) {
|
|
8514
|
+
return;
|
|
8515
|
+
}
|
|
8516
|
+
const track = stagingId ? this.tracks.get(stagingId) : void 0;
|
|
8517
|
+
let pose = null;
|
|
8518
|
+
if (track && time !== null && time >= 0) {
|
|
8519
|
+
pose = this.sample(track, time);
|
|
8520
|
+
}
|
|
8521
|
+
this.weight = approach(
|
|
8522
|
+
this.weight,
|
|
8523
|
+
pose ? 1 : 0,
|
|
8524
|
+
Math.max(0, delta),
|
|
8525
|
+
pose ? this.blendIn : this.blendOut
|
|
8526
|
+
);
|
|
8527
|
+
if (pose) {
|
|
8528
|
+
this.lastPose = pose;
|
|
8529
|
+
}
|
|
8530
|
+
const applied = pose ?? this.lastPose;
|
|
8531
|
+
if (!applied || this.weight <= 0) {
|
|
8532
|
+
if (this.weight <= 0) {
|
|
8533
|
+
this.lastPose = null;
|
|
8534
|
+
}
|
|
8535
|
+
return;
|
|
8536
|
+
}
|
|
8537
|
+
this.apply(vrm, applied.track, applied.values, this.weight);
|
|
8538
|
+
}
|
|
8539
|
+
track(stagingId, meta) {
|
|
8540
|
+
let track = this.tracks.get(stagingId);
|
|
8541
|
+
if (!track) {
|
|
8542
|
+
track = {
|
|
8543
|
+
fps: meta.fps,
|
|
8544
|
+
bones: [...meta.bones],
|
|
8545
|
+
restHipsHeight: meta.restHipsHeight,
|
|
8546
|
+
width: motionFrameWidth(meta.bones.length),
|
|
8547
|
+
frames: /* @__PURE__ */ new Map(),
|
|
8548
|
+
origin: null
|
|
8549
|
+
};
|
|
8550
|
+
this.tracks.set(stagingId, track);
|
|
8551
|
+
while (this.tracks.size > MAX_TRACKS) {
|
|
8552
|
+
const oldest = this.tracks.keys().next().value;
|
|
8553
|
+
if (oldest === void 0 || oldest === stagingId) {
|
|
8554
|
+
break;
|
|
8555
|
+
}
|
|
8556
|
+
this.tracks.delete(oldest);
|
|
8557
|
+
}
|
|
8558
|
+
}
|
|
8559
|
+
return track;
|
|
8560
|
+
}
|
|
8561
|
+
/** The interpolated frame at `time`, or null when its frames are missing. */
|
|
8562
|
+
sample(track, time) {
|
|
8563
|
+
const position = time * track.fps;
|
|
8564
|
+
const first = Math.floor(position);
|
|
8565
|
+
const a = track.frames.get(first);
|
|
8566
|
+
if (!a) {
|
|
8567
|
+
return null;
|
|
8568
|
+
}
|
|
8569
|
+
track.origin ??= { x: a.values[0] ?? 0, z: a.values[2] ?? 0 };
|
|
8570
|
+
const b = track.frames.get(first + 1);
|
|
8571
|
+
const amount = position - first;
|
|
8572
|
+
if (!b || amount <= 0) {
|
|
8573
|
+
return { track, values: a.values };
|
|
8574
|
+
}
|
|
8575
|
+
const values = new Float32Array(track.width);
|
|
8576
|
+
for (let index = 0; index < 3; index += 1) {
|
|
8577
|
+
values[index] = (a.values[index] ?? 0) * (1 - amount) + (b.values[index] ?? 0) * amount;
|
|
8578
|
+
}
|
|
8579
|
+
const from = this.scratch.from;
|
|
8580
|
+
const to = this.scratch.to;
|
|
8581
|
+
for (let offset = 3; offset < track.width; offset += 4) {
|
|
8582
|
+
from.fromArray(a.values, offset);
|
|
8583
|
+
to.fromArray(b.values, offset);
|
|
8584
|
+
from.slerp(to, amount);
|
|
8585
|
+
values[offset] = from.x;
|
|
8586
|
+
values[offset + 1] = from.y;
|
|
8587
|
+
values[offset + 2] = from.z;
|
|
8588
|
+
values[offset + 3] = from.w;
|
|
8589
|
+
}
|
|
8590
|
+
return { track, values };
|
|
8591
|
+
}
|
|
8592
|
+
apply(vrm, track, values, weight) {
|
|
8593
|
+
const humanoid = vrm.humanoid;
|
|
8594
|
+
const flip = vrm.meta.metaVersion === "0" && !isNpcifyNormalizedVrm(vrm);
|
|
8595
|
+
const carry = this.scratch.carry;
|
|
8596
|
+
carry.clear();
|
|
8597
|
+
track.bones.forEach((bone, slot) => {
|
|
8598
|
+
const offset = 3 + slot * 4;
|
|
8599
|
+
const rotation = new Quaternion3(
|
|
8600
|
+
values[offset] ?? 0,
|
|
8601
|
+
values[offset + 1] ?? 0,
|
|
8602
|
+
values[offset + 2] ?? 0,
|
|
8603
|
+
values[offset + 3] ?? 1
|
|
8604
|
+
);
|
|
8605
|
+
if (flip) {
|
|
8606
|
+
rotation.x = -rotation.x;
|
|
8607
|
+
rotation.z = -rotation.z;
|
|
8608
|
+
}
|
|
8609
|
+
const inherited = carry.get(bone);
|
|
8610
|
+
if (inherited) {
|
|
8611
|
+
rotation.premultiply(inherited);
|
|
8612
|
+
}
|
|
8613
|
+
const node = humanoid.getNormalizedBoneNode(bone);
|
|
8614
|
+
if (!node) {
|
|
8615
|
+
for (const [child, parent] of Object.entries(PARENTS)) {
|
|
8616
|
+
if (parent === bone) {
|
|
8617
|
+
carry.set(child, rotation.clone());
|
|
8618
|
+
}
|
|
8619
|
+
}
|
|
8620
|
+
return;
|
|
8621
|
+
}
|
|
8622
|
+
node.quaternion.slerp(rotation, weight);
|
|
8623
|
+
});
|
|
8624
|
+
const hips = humanoid.getNormalizedBoneNode("hips");
|
|
8625
|
+
const rest = humanoid.normalizedRestPose.hips?.position;
|
|
8626
|
+
if (!hips || !rest) {
|
|
8627
|
+
return;
|
|
8628
|
+
}
|
|
8629
|
+
const scale = (rest[1] ?? 1) / (track.restHipsHeight || 1);
|
|
8630
|
+
let x = values[0] ?? 0;
|
|
8631
|
+
const y = values[1] ?? 0;
|
|
8632
|
+
let z = values[2] ?? 0;
|
|
8633
|
+
if (this.rootMode === "anchored" && track.origin) {
|
|
8634
|
+
x -= track.origin.x;
|
|
8635
|
+
z -= track.origin.z;
|
|
8636
|
+
}
|
|
8637
|
+
const target = this.scratch.position.set(
|
|
8638
|
+
(flip ? -x : x) * scale + (this.rootMode === "anchored" ? rest[0] ?? 0 : 0),
|
|
8639
|
+
y * scale,
|
|
8640
|
+
(flip ? -z : z) * scale + (this.rootMode === "anchored" ? rest[2] ?? 0 : 0)
|
|
8641
|
+
);
|
|
8642
|
+
hips.position.lerp(target, weight);
|
|
8643
|
+
}
|
|
8644
|
+
};
|
|
8645
|
+
|
|
8263
8646
|
// ../../clients/web-client/src/character/incremental-staging.ts
|
|
8264
8647
|
var dialogueKey = function(block) {
|
|
8265
8648
|
return JSON.stringify(block, (key, value) => {
|
|
@@ -8534,9 +8917,17 @@ var StagingPlayer = class {
|
|
|
8534
8917
|
activeExpression = null;
|
|
8535
8918
|
activeExpressionMouthMode = null;
|
|
8536
8919
|
activeAnimation = null;
|
|
8920
|
+
currentTrackTime = null;
|
|
8537
8921
|
get paused() {
|
|
8538
8922
|
return this.clock.paused;
|
|
8539
8923
|
}
|
|
8924
|
+
/**
|
|
8925
|
+
* Seconds of the staging clock that the tracks show now, or null when no
|
|
8926
|
+
* staging plays (also while playback waits for the next dialogue block).
|
|
8927
|
+
*/
|
|
8928
|
+
get trackTime() {
|
|
8929
|
+
return this.currentTrackTime;
|
|
8930
|
+
}
|
|
8540
8931
|
pause() {
|
|
8541
8932
|
this.pauseRevision += 1;
|
|
8542
8933
|
this.clock.pause();
|
|
@@ -8937,6 +9328,7 @@ var StagingPlayer = class {
|
|
|
8937
9328
|
}
|
|
8938
9329
|
}
|
|
8939
9330
|
applyTracks(time, speechActive, timeline) {
|
|
9331
|
+
this.currentTrackTime = time;
|
|
8940
9332
|
const expression = activeClipAt(timeline.expressions, time);
|
|
8941
9333
|
const mouthMode = speechActive ? "viseme" : "expression";
|
|
8942
9334
|
if (expression !== this.activeExpression || expression && mouthMode !== this.activeExpressionMouthMode) {
|
|
@@ -8970,6 +9362,7 @@ var StagingPlayer = class {
|
|
|
8970
9362
|
}
|
|
8971
9363
|
}
|
|
8972
9364
|
clearPerformance() {
|
|
9365
|
+
this.currentTrackTime = null;
|
|
8973
9366
|
this.activeExpression = null;
|
|
8974
9367
|
this.activeExpressionMouthMode = null;
|
|
8975
9368
|
this.activeAnimation = null;
|
|
@@ -9408,6 +9801,7 @@ var CharacterRuntime = class {
|
|
|
9408
9801
|
performanceRecorder;
|
|
9409
9802
|
telemetry;
|
|
9410
9803
|
streamPlayback;
|
|
9804
|
+
liveMotion;
|
|
9411
9805
|
defaultParent;
|
|
9412
9806
|
currentGazeTarget = null;
|
|
9413
9807
|
/** The current gaze target came from `lookAt()`. */
|
|
@@ -9456,12 +9850,21 @@ var CharacterRuntime = class {
|
|
|
9456
9850
|
baseUrl: this.baseUrl,
|
|
9457
9851
|
apiToken: this.apiToken
|
|
9458
9852
|
});
|
|
9853
|
+
this.liveMotion = new LiveMotionPlayer(options.liveMotion);
|
|
9459
9854
|
this.streamPlayback = new StreamPlaybackController({
|
|
9460
9855
|
getStagingPlayer: this.getStagingPlayer,
|
|
9461
9856
|
telemetry: this.telemetry,
|
|
9462
9857
|
handleCallbackError: this.handleCallbackError,
|
|
9463
9858
|
reportError: this.reportError,
|
|
9464
|
-
recordEvent: (event) => this.performanceRecorder.recordEvent(event)
|
|
9859
|
+
recordEvent: (event) => this.performanceRecorder.recordEvent(event),
|
|
9860
|
+
onMotionFrames: (event) => {
|
|
9861
|
+
this.liveMotion.push(event);
|
|
9862
|
+
},
|
|
9863
|
+
onMotionTrack: (stagingId, track) => {
|
|
9864
|
+
void this.liveMotion.load(stagingId, track).catch((error) => {
|
|
9865
|
+
this.handleCallbackError("liveMotion", error);
|
|
9866
|
+
});
|
|
9867
|
+
}
|
|
9465
9868
|
});
|
|
9466
9869
|
if (options.gazeTarget !== void 0) {
|
|
9467
9870
|
this.setGazeTarget(options.gazeTarget);
|
|
@@ -9498,6 +9901,7 @@ var CharacterRuntime = class {
|
|
|
9498
9901
|
set vrm(value) {
|
|
9499
9902
|
this.sceneRig.setVrm(value);
|
|
9500
9903
|
this.facialMixer.setVrm(value);
|
|
9904
|
+
this.liveMotion.setVrm(value);
|
|
9501
9905
|
this.applyGaze();
|
|
9502
9906
|
if (value && this.defaultParent && !value.scene?.parent) {
|
|
9503
9907
|
this.sceneRig.attachTo(this.defaultParent);
|
|
@@ -9774,6 +10178,11 @@ var CharacterRuntime = class {
|
|
|
9774
10178
|
return;
|
|
9775
10179
|
}
|
|
9776
10180
|
this.animationController?.update(delta);
|
|
10181
|
+
this.liveMotion.update(
|
|
10182
|
+
delta,
|
|
10183
|
+
this.streamPlayback.activeStagingId,
|
|
10184
|
+
this.stagingPlayer?.trackTime ?? null
|
|
10185
|
+
);
|
|
9777
10186
|
this.visemePlayer.update(delta);
|
|
9778
10187
|
this.facialMixer.update(delta);
|
|
9779
10188
|
this.vrm?.update(delta);
|
|
@@ -9830,6 +10239,7 @@ var CharacterRuntime = class {
|
|
|
9830
10239
|
this.animations.clear();
|
|
9831
10240
|
this.sceneRig.setVrm(null);
|
|
9832
10241
|
this.facialMixer.setVrm(null);
|
|
10242
|
+
this.liveMotion.setVrm(null);
|
|
9833
10243
|
}
|
|
9834
10244
|
disposeRuntimeAssets() {
|
|
9835
10245
|
this.sceneRig.disposeVrm();
|
|
@@ -9852,184 +10262,1012 @@ var CharacterRuntime = class {
|
|
|
9852
10262
|
|
|
9853
10263
|
// ../../clients/web-client/src/conversation-socket.ts
|
|
9854
10264
|
import { io } from "socket.io-client";
|
|
9855
|
-
|
|
9856
|
-
|
|
9857
|
-
|
|
9858
|
-
constructor(
|
|
9859
|
-
|
|
10265
|
+
|
|
10266
|
+
// ../../clients/web-client/src/sight/sight-error.ts
|
|
10267
|
+
var SightError = class extends Error {
|
|
10268
|
+
constructor(code, message) {
|
|
10269
|
+
super(message ?? code);
|
|
10270
|
+
this.code = code;
|
|
10271
|
+
this.name = "NpcifySightError";
|
|
9860
10272
|
}
|
|
9861
|
-
|
|
9862
|
-
|
|
9863
|
-
|
|
10273
|
+
};
|
|
10274
|
+
var SIGHT_STOP_CODES = /* @__PURE__ */ new Set([
|
|
10275
|
+
"SIGHT_DISABLED",
|
|
10276
|
+
"INSUFFICIENT_POINTS",
|
|
10277
|
+
"CONVERSATION_ACCESS_DENIED"
|
|
10278
|
+
]);
|
|
10279
|
+
var SIGHT_OFF_MESSAGE = "Sight is off for this character. Its owner turns it on in the Sight tab.";
|
|
10280
|
+
var STOP_MESSAGES = {
|
|
10281
|
+
SIGHT_DISABLED: SIGHT_OFF_MESSAGE,
|
|
10282
|
+
INSUFFICIENT_POINTS: "The owner of the character has no points to pay for sight.",
|
|
10283
|
+
CONVERSATION_ACCESS_DENIED: "This connection has no access to the conversation."
|
|
10284
|
+
};
|
|
10285
|
+
var sightStopError = function(code) {
|
|
10286
|
+
return new SightError(code, STOP_MESSAGES[code] ?? code);
|
|
10287
|
+
};
|
|
10288
|
+
var toSightError = function(error, fallback) {
|
|
10289
|
+
if (error instanceof SightError) {
|
|
10290
|
+
return error;
|
|
9864
10291
|
}
|
|
9865
|
-
|
|
9866
|
-
|
|
9867
|
-
|
|
9868
|
-
|
|
9869
|
-
|
|
10292
|
+
const code = error?.code;
|
|
10293
|
+
return new SightError(
|
|
10294
|
+
typeof code === "string" && code ? code : fallback,
|
|
10295
|
+
error instanceof Error ? error.message : void 0
|
|
10296
|
+
);
|
|
10297
|
+
};
|
|
10298
|
+
|
|
10299
|
+
// ../../clients/web-client/src/sight/frame-source.ts
|
|
10300
|
+
var LUMA_GRID = Object.freeze({ width: 32, height: 18 });
|
|
10301
|
+
var SMALLER_JPEG_QUALITY = 0.5;
|
|
10302
|
+
var HAVE_CURRENT_DATA = 2;
|
|
10303
|
+
var fitCaptureSize = function(width, height, limit = SIGHT_CAPTURE_SIZE) {
|
|
10304
|
+
const scale = Math.min(1, limit.width / width, limit.height / height);
|
|
10305
|
+
return {
|
|
10306
|
+
width: Math.max(1, Math.round(width * scale)),
|
|
10307
|
+
height: Math.max(1, Math.round(height * scale))
|
|
10308
|
+
};
|
|
10309
|
+
};
|
|
10310
|
+
var lumaOf = function(rgba) {
|
|
10311
|
+
const luma = new Uint8Array(Math.floor(rgba.length / 4));
|
|
10312
|
+
for (let index = 0; index < luma.length; index += 1) {
|
|
10313
|
+
const offset = index * 4;
|
|
10314
|
+
luma[index] = Math.round(
|
|
10315
|
+
0.299 * (rgba[offset] ?? 0) + 0.587 * (rgba[offset + 1] ?? 0) + 0.114 * (rgba[offset + 2] ?? 0)
|
|
10316
|
+
);
|
|
10317
|
+
}
|
|
10318
|
+
return luma;
|
|
10319
|
+
};
|
|
10320
|
+
var meanLumaChange = function(previous, next) {
|
|
10321
|
+
const length = Math.min(previous.length, next.length);
|
|
10322
|
+
if (length === 0) {
|
|
10323
|
+
return 0;
|
|
10324
|
+
}
|
|
10325
|
+
let sum = 0;
|
|
10326
|
+
for (let index = 0; index < length; index += 1) {
|
|
10327
|
+
sum += Math.abs((next[index] ?? 0) - (previous[index] ?? 0));
|
|
10328
|
+
}
|
|
10329
|
+
return sum / length;
|
|
10330
|
+
};
|
|
10331
|
+
var encodeJpeg = function(canvas, quality) {
|
|
10332
|
+
return new Promise((resolve) => {
|
|
10333
|
+
canvas.toBlob(resolve, "image/jpeg", quality);
|
|
10334
|
+
});
|
|
10335
|
+
};
|
|
10336
|
+
var VideoFrameSource = class {
|
|
10337
|
+
constructor(stream, owned) {
|
|
10338
|
+
this.stream = stream;
|
|
10339
|
+
this.owned = owned;
|
|
10340
|
+
const settings = stream.getVideoTracks()[0]?.getSettings?.();
|
|
10341
|
+
this.kind = settings?.displaySurface ? "screen" : "camera";
|
|
10342
|
+
const video = document.createElement("video");
|
|
10343
|
+
video.muted = true;
|
|
10344
|
+
video.playsInline = true;
|
|
10345
|
+
video.setAttribute("playsinline", "");
|
|
10346
|
+
video.srcObject = stream;
|
|
10347
|
+
void video.play().catch(() => void 0);
|
|
10348
|
+
this.video = video;
|
|
10349
|
+
this.frameCanvas = document.createElement("canvas");
|
|
10350
|
+
const grid = document.createElement("canvas");
|
|
10351
|
+
grid.width = LUMA_GRID.width;
|
|
10352
|
+
grid.height = LUMA_GRID.height;
|
|
10353
|
+
this.gridContext = grid.getContext("2d", { willReadFrequently: true });
|
|
10354
|
+
if (this.gridContext) {
|
|
10355
|
+
this.gridContext.imageSmoothingQuality = "high";
|
|
10356
|
+
}
|
|
10357
|
+
}
|
|
10358
|
+
kind;
|
|
10359
|
+
video;
|
|
10360
|
+
frameCanvas;
|
|
10361
|
+
frameContext = null;
|
|
10362
|
+
gridContext;
|
|
10363
|
+
stopped = false;
|
|
10364
|
+
get live() {
|
|
10365
|
+
return !this.stopped && this.stream.getVideoTracks().some((track) => track.readyState === "live");
|
|
10366
|
+
}
|
|
10367
|
+
async capture() {
|
|
10368
|
+
const picture = this.pictureSize();
|
|
10369
|
+
if (!picture) {
|
|
10370
|
+
return null;
|
|
9870
10371
|
}
|
|
9871
|
-
|
|
10372
|
+
const size = fitCaptureSize(picture.width, picture.height);
|
|
10373
|
+
const canvas = this.frameCanvas;
|
|
10374
|
+
if (canvas.width !== size.width || canvas.height !== size.height) {
|
|
10375
|
+
canvas.width = size.width;
|
|
10376
|
+
canvas.height = size.height;
|
|
10377
|
+
}
|
|
10378
|
+
this.frameContext ??= canvas.getContext("2d");
|
|
10379
|
+
const context2 = this.frameContext;
|
|
10380
|
+
if (!context2) {
|
|
10381
|
+
return null;
|
|
10382
|
+
}
|
|
10383
|
+
context2.imageSmoothingQuality = "high";
|
|
10384
|
+
context2.drawImage(this.video, 0, 0, size.width, size.height);
|
|
10385
|
+
let blob = await encodeJpeg(canvas, SIGHT_CAPTURE_SIZE.jpegQuality);
|
|
10386
|
+
if (blob && blob.size > SIGHT_LIMITS.maxFrameBytes) {
|
|
10387
|
+
blob = await encodeJpeg(canvas, SMALLER_JPEG_QUALITY);
|
|
10388
|
+
}
|
|
10389
|
+
if (!blob || blob.size > SIGHT_LIMITS.maxFrameBytes) {
|
|
10390
|
+
return null;
|
|
10391
|
+
}
|
|
10392
|
+
return { jpeg: await blob.arrayBuffer(), ...size };
|
|
9872
10393
|
}
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
this.
|
|
10394
|
+
luminance() {
|
|
10395
|
+
const context2 = this.gridContext;
|
|
10396
|
+
if (!context2 || !this.pictureSize()) {
|
|
10397
|
+
return null;
|
|
10398
|
+
}
|
|
10399
|
+
context2.drawImage(this.video, 0, 0, LUMA_GRID.width, LUMA_GRID.height);
|
|
10400
|
+
return lumaOf(
|
|
10401
|
+
context2.getImageData(0, 0, LUMA_GRID.width, LUMA_GRID.height).data
|
|
10402
|
+
);
|
|
9876
10403
|
}
|
|
9877
|
-
|
|
9878
|
-
if (
|
|
10404
|
+
stop() {
|
|
10405
|
+
if (this.stopped) {
|
|
9879
10406
|
return;
|
|
9880
10407
|
}
|
|
9881
|
-
|
|
9882
|
-
|
|
9883
|
-
|
|
10408
|
+
this.stopped = true;
|
|
10409
|
+
this.video.pause();
|
|
10410
|
+
this.video.srcObject = null;
|
|
10411
|
+
if (this.owned) {
|
|
10412
|
+
this.stream.getTracks().forEach((track) => track.stop());
|
|
9884
10413
|
}
|
|
9885
|
-
|
|
9886
|
-
|
|
9887
|
-
|
|
9888
|
-
|
|
9889
|
-
|
|
9890
|
-
return window.location.origin;
|
|
9891
|
-
}
|
|
9892
|
-
})();
|
|
9893
|
-
this.socket = io(socketBaseUrl, {
|
|
9894
|
-
transports: ["websocket"],
|
|
9895
|
-
withCredentials: true,
|
|
9896
|
-
auth: {
|
|
9897
|
-
apiToken: this.options.apiToken
|
|
9898
|
-
}
|
|
9899
|
-
});
|
|
9900
|
-
this.socket.on("connect", () => {
|
|
9901
|
-
const activeConversationId = this.options.getConversationId()?.trim();
|
|
9902
|
-
if (activeConversationId) {
|
|
9903
|
-
this.socket?.emit("conversations:join", {
|
|
9904
|
-
conversationId: activeConversationId
|
|
9905
|
-
});
|
|
9906
|
-
}
|
|
9907
|
-
});
|
|
9908
|
-
this.socket.on("conversations:stream", (payload) => {
|
|
9909
|
-
let candidate;
|
|
9910
|
-
try {
|
|
9911
|
-
candidate = decodePerformance(
|
|
9912
|
-
payload,
|
|
9913
|
-
"stream"
|
|
9914
|
-
);
|
|
9915
|
-
} catch (error) {
|
|
9916
|
-
this.options.handleCallbackError("stream.decode", error);
|
|
9917
|
-
return;
|
|
9918
|
-
}
|
|
9919
|
-
if (typeof candidate?.conversationId !== "string") {
|
|
9920
|
-
return;
|
|
9921
|
-
}
|
|
9922
|
-
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
9923
|
-
return;
|
|
9924
|
-
}
|
|
9925
|
-
const event = candidate.event;
|
|
9926
|
-
if (!event || typeof event !== "object") {
|
|
9927
|
-
return;
|
|
9928
|
-
}
|
|
9929
|
-
this.options.handleStreamEvent(event);
|
|
9930
|
-
});
|
|
9931
|
-
this.socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
9932
|
-
const event = payload;
|
|
9933
|
-
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
9934
|
-
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
9935
|
-
}
|
|
9936
|
-
});
|
|
9937
|
-
this.socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
9938
|
-
let candidate;
|
|
9939
|
-
try {
|
|
9940
|
-
candidate = decodePerformance(
|
|
9941
|
-
payload,
|
|
9942
|
-
"turn"
|
|
9943
|
-
);
|
|
9944
|
-
} catch (error) {
|
|
9945
|
-
this.options.handleCallbackError("speech.turn.decode", error);
|
|
9946
|
-
return;
|
|
9947
|
-
}
|
|
9948
|
-
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
9949
|
-
return;
|
|
9950
|
-
}
|
|
9951
|
-
this.options.handleSpeechEvent?.({
|
|
9952
|
-
type: "turn",
|
|
9953
|
-
requestId: candidate.requestId,
|
|
9954
|
-
turn: candidate.turn
|
|
9955
|
-
});
|
|
9956
|
-
});
|
|
9957
|
-
this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
9958
|
-
const event = payload;
|
|
9959
|
-
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
9960
|
-
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
9961
|
-
}
|
|
9962
|
-
});
|
|
9963
|
-
this.socket.on("disconnect", () => {
|
|
9964
|
-
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
9965
|
-
});
|
|
9966
|
-
this.socket.on("conversations:turn", (payload) => {
|
|
9967
|
-
let candidate;
|
|
9968
|
-
try {
|
|
9969
|
-
candidate = decodePerformance(
|
|
9970
|
-
payload,
|
|
9971
|
-
"turn"
|
|
9972
|
-
);
|
|
9973
|
-
} catch (error) {
|
|
9974
|
-
this.options.handleCallbackError("turn.decode", error);
|
|
9975
|
-
return;
|
|
9976
|
-
}
|
|
9977
|
-
if (typeof candidate?.conversationId !== "string") {
|
|
9978
|
-
return;
|
|
9979
|
-
}
|
|
9980
|
-
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
9981
|
-
return;
|
|
9982
|
-
}
|
|
9983
|
-
const turn = candidate.turn;
|
|
9984
|
-
if (!turn || typeof turn !== "object") {
|
|
9985
|
-
return;
|
|
9986
|
-
}
|
|
9987
|
-
void this.options.handleTurn(turn).catch((error) => {
|
|
9988
|
-
this.options.handleCallbackError("onTurn", error);
|
|
9989
|
-
});
|
|
9990
|
-
});
|
|
9991
|
-
await new Promise((resolve) => {
|
|
9992
|
-
if (this.socket?.connected) {
|
|
9993
|
-
resolve();
|
|
9994
|
-
return;
|
|
9995
|
-
}
|
|
9996
|
-
this.socket?.once("connect", () => resolve());
|
|
9997
|
-
this.socket?.once("connect_error", () => resolve());
|
|
9998
|
-
});
|
|
10414
|
+
}
|
|
10415
|
+
pictureSize() {
|
|
10416
|
+
const { video } = this;
|
|
10417
|
+
if (this.stopped || video.readyState < HAVE_CURRENT_DATA || !video.videoWidth || !video.videoHeight) {
|
|
10418
|
+
return null;
|
|
9999
10419
|
}
|
|
10000
|
-
|
|
10001
|
-
|
|
10002
|
-
|
|
10003
|
-
|
|
10004
|
-
|
|
10005
|
-
|
|
10006
|
-
|
|
10007
|
-
|
|
10008
|
-
|
|
10009
|
-
|
|
10010
|
-
|
|
10011
|
-
|
|
10012
|
-
|
|
10420
|
+
return { width: video.videoWidth, height: video.videoHeight };
|
|
10421
|
+
}
|
|
10422
|
+
};
|
|
10423
|
+
var openFrameSource = async function(source) {
|
|
10424
|
+
if (source !== "camera") {
|
|
10425
|
+
if (typeof source?.getVideoTracks !== "function") {
|
|
10426
|
+
throw new SightError(
|
|
10427
|
+
"CAMERA_UNAVAILABLE",
|
|
10428
|
+
'The sight source must be "camera" or a MediaStream.'
|
|
10429
|
+
);
|
|
10430
|
+
}
|
|
10431
|
+
if (source.getVideoTracks().length === 0) {
|
|
10432
|
+
throw new SightError(
|
|
10433
|
+
"CAMERA_UNAVAILABLE",
|
|
10434
|
+
"The stream has no video track."
|
|
10013
10435
|
);
|
|
10436
|
+
}
|
|
10437
|
+
return new VideoFrameSource(source, false);
|
|
10438
|
+
}
|
|
10439
|
+
const devices = typeof navigator === "undefined" ? void 0 : navigator.mediaDevices;
|
|
10440
|
+
if (!devices?.getUserMedia) {
|
|
10441
|
+
throw new SightError(
|
|
10442
|
+
"CAMERA_UNAVAILABLE",
|
|
10443
|
+
"This browser cannot open the camera."
|
|
10444
|
+
);
|
|
10445
|
+
}
|
|
10446
|
+
let stream;
|
|
10447
|
+
try {
|
|
10448
|
+
stream = await devices.getUserMedia({
|
|
10449
|
+
video: {
|
|
10450
|
+
width: { ideal: SIGHT_CAPTURE_SIZE.width },
|
|
10451
|
+
height: { ideal: SIGHT_CAPTURE_SIZE.height }
|
|
10452
|
+
},
|
|
10453
|
+
audio: false
|
|
10014
10454
|
});
|
|
10455
|
+
} catch (error) {
|
|
10456
|
+
const name = error?.name;
|
|
10457
|
+
throw name === "NotAllowedError" || name === "SecurityError" ? new SightError("CAMERA_PERMISSION_DENIED", "Camera access was denied.") : new SightError("CAMERA_UNAVAILABLE", "No camera is available.");
|
|
10015
10458
|
}
|
|
10459
|
+
return new VideoFrameSource(stream, true);
|
|
10016
10460
|
};
|
|
10017
10461
|
|
|
10018
|
-
// ../../clients/web-client/src/
|
|
10019
|
-
var
|
|
10020
|
-
var
|
|
10021
|
-
|
|
10462
|
+
// ../../clients/web-client/src/sight/sight-input.ts
|
|
10463
|
+
var SIGHT_CHECK_MS = 250;
|
|
10464
|
+
var SIGHT_INTERVAL_MS = 5e3;
|
|
10465
|
+
var SIGHT_MOTION_THRESHOLD = 8;
|
|
10466
|
+
var ACK_TIMEOUT_MS = 2e4;
|
|
10467
|
+
var systemClock = {
|
|
10468
|
+
now: () => Date.now(),
|
|
10469
|
+
setInterval: (callback, ms) => setInterval(callback, ms),
|
|
10470
|
+
clearInterval: (handle) => clearInterval(handle)
|
|
10471
|
+
};
|
|
10472
|
+
var settle = async function(run) {
|
|
10473
|
+
return run();
|
|
10474
|
+
};
|
|
10475
|
+
var documentHidden = function() {
|
|
10476
|
+
return typeof document !== "undefined" && document.hidden === true;
|
|
10477
|
+
};
|
|
10478
|
+
var intervalOf = function(value) {
|
|
10479
|
+
return typeof value === "number" && value > 0 ? Math.max(SIGHT_LIMITS.minFrameIntervalMs, value) : SIGHT_INTERVAL_MS;
|
|
10480
|
+
};
|
|
10481
|
+
var thresholdOf = function(motion) {
|
|
10482
|
+
if (motion === false) {
|
|
10483
|
+
return null;
|
|
10484
|
+
}
|
|
10485
|
+
const threshold = motion?.threshold;
|
|
10486
|
+
return typeof threshold === "number" && Number.isFinite(threshold) ? Math.max(0, threshold) : SIGHT_MOTION_THRESHOLD;
|
|
10022
10487
|
};
|
|
10023
|
-
var
|
|
10024
|
-
const
|
|
10025
|
-
const
|
|
10026
|
-
|
|
10027
|
-
|
|
10028
|
-
|
|
10029
|
-
|
|
10030
|
-
|
|
10031
|
-
|
|
10032
|
-
|
|
10488
|
+
var isSightObservationEvent = function(value) {
|
|
10489
|
+
const event = value;
|
|
10490
|
+
const observation = event?.observation;
|
|
10491
|
+
return typeof event?.conversationId === "string" && typeof event.observationId === "string" && typeof event.summary === "string" && typeof observation === "object" && observation !== null && typeof observation.people === "number" && Array.isArray(observation.persons) && Array.isArray(observation.text);
|
|
10492
|
+
};
|
|
10493
|
+
var SightInput = class {
|
|
10494
|
+
constructor(deps) {
|
|
10495
|
+
this.deps = deps;
|
|
10496
|
+
this.clock = deps.clock ?? systemClock;
|
|
10497
|
+
this.hidden = deps.hidden ?? documentHidden;
|
|
10498
|
+
}
|
|
10499
|
+
clock;
|
|
10500
|
+
hidden;
|
|
10501
|
+
session = null;
|
|
10502
|
+
/** Why sight cannot start; null when it can. */
|
|
10503
|
+
unavailable = "Sight needs the character info: load the character first.";
|
|
10504
|
+
current = "unavailable";
|
|
10505
|
+
stateCallback = null;
|
|
10506
|
+
observationCallback = null;
|
|
10507
|
+
get state() {
|
|
10508
|
+
return this.current;
|
|
10509
|
+
}
|
|
10510
|
+
/**
|
|
10511
|
+
* The character info was read. Sight is available only when the owner
|
|
10512
|
+
* turned it on, and never in a baked conversation. Sight that becomes
|
|
10513
|
+
* unavailable stops.
|
|
10514
|
+
*/
|
|
10515
|
+
configure(settings) {
|
|
10516
|
+
this.unavailable = settings.baked ? "Sight is not available in a baked (offline) conversation." : settings.enabled ? null : SIGHT_OFF_MESSAGE;
|
|
10517
|
+
if (this.unavailable) {
|
|
10518
|
+
this.stop();
|
|
10519
|
+
} else {
|
|
10520
|
+
this.publish();
|
|
10521
|
+
}
|
|
10522
|
+
}
|
|
10523
|
+
/**
|
|
10524
|
+
* Opens the source and joins the conversation. Resolves when frames can
|
|
10525
|
+
* go; the first frame goes when the video has a picture.
|
|
10526
|
+
*/
|
|
10527
|
+
async start(options = {}) {
|
|
10528
|
+
if (this.unavailable) {
|
|
10529
|
+
throw new SightError("SIGHT_DISABLED", this.unavailable);
|
|
10530
|
+
}
|
|
10531
|
+
if (this.session) {
|
|
10532
|
+
throw new SightError(
|
|
10533
|
+
"SIGHT_ALREADY_STARTED",
|
|
10534
|
+
"Sight has already started."
|
|
10535
|
+
);
|
|
10536
|
+
}
|
|
10537
|
+
const conversationId = this.deps.conversationId();
|
|
10538
|
+
if (!conversationId) {
|
|
10539
|
+
throw new SightError(
|
|
10540
|
+
"CONVERSATION_REQUIRED",
|
|
10541
|
+
"Create a conversation before sight starts."
|
|
10542
|
+
);
|
|
10543
|
+
}
|
|
10544
|
+
const session = {
|
|
10545
|
+
intervalMs: intervalOf(options.intervalMs),
|
|
10546
|
+
threshold: thresholdOf(options.motion),
|
|
10547
|
+
onError: options.onError,
|
|
10548
|
+
source: null,
|
|
10549
|
+
timer: null,
|
|
10550
|
+
grid: null,
|
|
10551
|
+
lastSentAt: Number.NEGATIVE_INFINITY,
|
|
10552
|
+
motion: false,
|
|
10553
|
+
inFlight: false,
|
|
10554
|
+
rejoin: false
|
|
10555
|
+
};
|
|
10556
|
+
this.session = session;
|
|
10557
|
+
this.publish();
|
|
10558
|
+
const open = this.deps.openSource ?? openFrameSource;
|
|
10559
|
+
const [joined, opened] = await Promise.allSettled([
|
|
10560
|
+
settle(() => this.deps.socket().ensureConnected(conversationId)),
|
|
10561
|
+
settle(() => open(options.source ?? "camera"))
|
|
10562
|
+
]);
|
|
10563
|
+
const source = opened.status === "fulfilled" ? opened.value : null;
|
|
10564
|
+
if (session !== this.session) {
|
|
10565
|
+
source?.stop();
|
|
10566
|
+
return;
|
|
10567
|
+
}
|
|
10568
|
+
const failure = opened.status === "rejected" ? toSightError(opened.reason, "CAMERA_UNAVAILABLE") : joined.status === "rejected" ? toSightError(joined.reason, "CONVERSATION_DISCONNECTED") : null;
|
|
10569
|
+
if (failure || !source) {
|
|
10570
|
+
source?.stop();
|
|
10571
|
+
this.session = null;
|
|
10572
|
+
this.publish();
|
|
10573
|
+
throw failure ?? new SightError("CAMERA_UNAVAILABLE");
|
|
10574
|
+
}
|
|
10575
|
+
session.source = source;
|
|
10576
|
+
session.timer = this.clock.setInterval(
|
|
10577
|
+
() => this.check(session),
|
|
10578
|
+
SIGHT_CHECK_MS
|
|
10579
|
+
);
|
|
10580
|
+
this.publish();
|
|
10581
|
+
this.check(session);
|
|
10582
|
+
}
|
|
10583
|
+
/** Stops the frames and releases the source. Does nothing when sight is off. */
|
|
10584
|
+
stop() {
|
|
10585
|
+
if (this.session) {
|
|
10586
|
+
this.end(this.session);
|
|
10587
|
+
}
|
|
10588
|
+
this.publish();
|
|
10589
|
+
}
|
|
10590
|
+
/** An observation of the current conversation (the socket checked the ID). */
|
|
10591
|
+
handleObservation(event) {
|
|
10592
|
+
if (this.observationCallback) {
|
|
10593
|
+
invokeSafeCallback(
|
|
10594
|
+
"onSight",
|
|
10595
|
+
this.deps.handleCallbackError,
|
|
10596
|
+
() => this.observationCallback?.(event)
|
|
10597
|
+
);
|
|
10598
|
+
}
|
|
10599
|
+
}
|
|
10600
|
+
onObservation(callback) {
|
|
10601
|
+
return bindCallbackSlot(
|
|
10602
|
+
() => this.observationCallback,
|
|
10603
|
+
(next) => this.observationCallback = next,
|
|
10604
|
+
callback
|
|
10605
|
+
);
|
|
10606
|
+
}
|
|
10607
|
+
onState(callback) {
|
|
10608
|
+
return bindCallbackSlot(
|
|
10609
|
+
() => this.stateCallback,
|
|
10610
|
+
(next) => this.stateCallback = next,
|
|
10611
|
+
callback
|
|
10612
|
+
);
|
|
10613
|
+
}
|
|
10614
|
+
/** One check: the end of the source, a hidden page, motion, then a due frame. */
|
|
10615
|
+
check(session) {
|
|
10616
|
+
const source = session.source;
|
|
10617
|
+
if (session !== this.session || !source) {
|
|
10618
|
+
return;
|
|
10619
|
+
}
|
|
10620
|
+
if (!source.live) {
|
|
10621
|
+
this.fail(
|
|
10622
|
+
session,
|
|
10623
|
+
new SightError("SIGHT_SOURCE_ENDED", "The video of sight ended.")
|
|
10624
|
+
);
|
|
10625
|
+
return;
|
|
10626
|
+
}
|
|
10627
|
+
if (this.hidden()) {
|
|
10628
|
+
session.grid = null;
|
|
10629
|
+
return;
|
|
10630
|
+
}
|
|
10631
|
+
if (session.threshold !== null) {
|
|
10632
|
+
const grid = source.luminance();
|
|
10633
|
+
if (grid && session.grid && meanLumaChange(session.grid, grid) > session.threshold) {
|
|
10634
|
+
session.motion = true;
|
|
10635
|
+
}
|
|
10636
|
+
session.grid = grid;
|
|
10637
|
+
}
|
|
10638
|
+
this.sendWhenDue(session);
|
|
10639
|
+
}
|
|
10640
|
+
/**
|
|
10641
|
+
* A frame is due at the interval, or after motion when one second passed
|
|
10642
|
+
* since the last frame. While a frame waits for its acknowledgement, a due
|
|
10643
|
+
* frame waits too: the acknowledgement sends it.
|
|
10644
|
+
*/
|
|
10645
|
+
sendWhenDue(session) {
|
|
10646
|
+
if (session !== this.session || session.inFlight || this.hidden()) {
|
|
10647
|
+
return;
|
|
10648
|
+
}
|
|
10649
|
+
const elapsed = this.clock.now() - session.lastSentAt;
|
|
10650
|
+
if (elapsed >= session.intervalMs || session.motion && elapsed >= SIGHT_LIMITS.minFrameIntervalMs) {
|
|
10651
|
+
void this.send(session);
|
|
10652
|
+
}
|
|
10653
|
+
}
|
|
10654
|
+
async send(session) {
|
|
10655
|
+
const source = session.source;
|
|
10656
|
+
if (!source) {
|
|
10657
|
+
return;
|
|
10658
|
+
}
|
|
10659
|
+
session.inFlight = true;
|
|
10660
|
+
let sent = false;
|
|
10661
|
+
try {
|
|
10662
|
+
const capturedAt = this.clock.now();
|
|
10663
|
+
let frame;
|
|
10664
|
+
try {
|
|
10665
|
+
frame = await source.capture();
|
|
10666
|
+
} catch (error) {
|
|
10667
|
+
this.fail(
|
|
10668
|
+
session,
|
|
10669
|
+
new SightError(
|
|
10670
|
+
"CAMERA_UNAVAILABLE",
|
|
10671
|
+
error instanceof Error ? error.message : "The video gave no picture."
|
|
10672
|
+
)
|
|
10673
|
+
);
|
|
10674
|
+
return;
|
|
10675
|
+
}
|
|
10676
|
+
if (!frame || session !== this.session) {
|
|
10677
|
+
return;
|
|
10678
|
+
}
|
|
10679
|
+
const conversationId = this.deps.conversationId();
|
|
10680
|
+
if (!conversationId) {
|
|
10681
|
+
this.stop();
|
|
10682
|
+
return;
|
|
10683
|
+
}
|
|
10684
|
+
session.lastSentAt = this.clock.now();
|
|
10685
|
+
session.motion = false;
|
|
10686
|
+
sent = true;
|
|
10687
|
+
const socket = this.deps.socket();
|
|
10688
|
+
if (session.rejoin) {
|
|
10689
|
+
await socket.ensureConnected(conversationId);
|
|
10690
|
+
session.rejoin = false;
|
|
10691
|
+
}
|
|
10692
|
+
const request = {
|
|
10693
|
+
conversationId,
|
|
10694
|
+
capturedAt,
|
|
10695
|
+
width: frame.width,
|
|
10696
|
+
height: frame.height,
|
|
10697
|
+
source: source.kind,
|
|
10698
|
+
jpeg: frame.jpeg
|
|
10699
|
+
};
|
|
10700
|
+
const ack = await socket.request(
|
|
10701
|
+
SIGHT_EVENTS.frame,
|
|
10702
|
+
request,
|
|
10703
|
+
ACK_TIMEOUT_MS
|
|
10704
|
+
);
|
|
10705
|
+
const code = ack && !ack.ok ? ack.error : null;
|
|
10706
|
+
if (code && SIGHT_STOP_CODES.has(code)) {
|
|
10707
|
+
this.fail(session, sightStopError(code));
|
|
10708
|
+
}
|
|
10709
|
+
} catch (error) {
|
|
10710
|
+
this.failed(session, error);
|
|
10711
|
+
} finally {
|
|
10712
|
+
session.inFlight = false;
|
|
10713
|
+
if (sent) {
|
|
10714
|
+
this.sendWhenDue(session);
|
|
10715
|
+
}
|
|
10716
|
+
}
|
|
10717
|
+
}
|
|
10718
|
+
/**
|
|
10719
|
+
* A frame that did not reach the server or got no acknowledgement. A
|
|
10720
|
+
* refused conversation stops sight. A timeout skips the frame; a lost
|
|
10721
|
+
* connection also joins the conversation again before the next frame.
|
|
10722
|
+
*/
|
|
10723
|
+
failed(session, error) {
|
|
10724
|
+
const code = error?.code;
|
|
10725
|
+
if (typeof code === "string" && SIGHT_STOP_CODES.has(code)) {
|
|
10726
|
+
this.fail(session, sightStopError(code));
|
|
10727
|
+
return;
|
|
10728
|
+
}
|
|
10729
|
+
if (code !== "CONVERSATION_TIMEOUT") {
|
|
10730
|
+
session.rejoin = true;
|
|
10731
|
+
}
|
|
10732
|
+
}
|
|
10733
|
+
/** Stops sight because of `error`, and reports it. */
|
|
10734
|
+
fail(session, error) {
|
|
10735
|
+
if (session !== this.session) {
|
|
10736
|
+
return;
|
|
10737
|
+
}
|
|
10738
|
+
this.end(session);
|
|
10739
|
+
if (error.code === "SIGHT_DISABLED") {
|
|
10740
|
+
this.unavailable = SIGHT_OFF_MESSAGE;
|
|
10741
|
+
}
|
|
10742
|
+
this.publish(error);
|
|
10743
|
+
const onError = session.onError;
|
|
10744
|
+
if (onError) {
|
|
10745
|
+
invokeSafeCallback(
|
|
10746
|
+
"startSight onError",
|
|
10747
|
+
this.deps.handleCallbackError,
|
|
10748
|
+
() => onError(error)
|
|
10749
|
+
);
|
|
10750
|
+
}
|
|
10751
|
+
}
|
|
10752
|
+
end(session) {
|
|
10753
|
+
if (this.session !== session) {
|
|
10754
|
+
return;
|
|
10755
|
+
}
|
|
10756
|
+
this.session = null;
|
|
10757
|
+
if (session.timer !== null) {
|
|
10758
|
+
this.clock.clearInterval(session.timer);
|
|
10759
|
+
session.timer = null;
|
|
10760
|
+
}
|
|
10761
|
+
session.source?.stop();
|
|
10762
|
+
}
|
|
10763
|
+
publish(error) {
|
|
10764
|
+
const session = this.session;
|
|
10765
|
+
const next = session ? session.source ? "on" : "starting" : this.unavailable ? "unavailable" : "off";
|
|
10766
|
+
if (next === this.current && !error) {
|
|
10767
|
+
return;
|
|
10768
|
+
}
|
|
10769
|
+
this.current = next;
|
|
10770
|
+
if (this.stateCallback) {
|
|
10771
|
+
invokeSafeCallback(
|
|
10772
|
+
"onSightState",
|
|
10773
|
+
this.deps.handleCallbackError,
|
|
10774
|
+
() => this.stateCallback?.(next, error)
|
|
10775
|
+
);
|
|
10776
|
+
}
|
|
10777
|
+
}
|
|
10778
|
+
};
|
|
10779
|
+
|
|
10780
|
+
// ../../clients/web-client/src/conversation-socket.ts
|
|
10781
|
+
var ConversationConnectionError = class extends Error {
|
|
10782
|
+
constructor(code, message, reason = null) {
|
|
10783
|
+
super(message);
|
|
10784
|
+
this.code = code;
|
|
10785
|
+
this.reason = reason;
|
|
10786
|
+
this.name = "NpcifyConversationConnectionError";
|
|
10787
|
+
}
|
|
10788
|
+
};
|
|
10789
|
+
var JOIN_EVENT = "conversations:join";
|
|
10790
|
+
var JOIN_TIMEOUT_MS = 1e4;
|
|
10791
|
+
var ACCESS_DENIED = "CONVERSATION_ACCESS_DENIED";
|
|
10792
|
+
var JOIN_FAILED = "CONVERSATION_JOIN_FAILED";
|
|
10793
|
+
var CLIENT_CLOSE = "io client disconnect";
|
|
10794
|
+
var SOCKET_IO_TIMEOUT = "operation has timed out";
|
|
10795
|
+
var joinMessage = function(code) {
|
|
10796
|
+
if (code === ACCESS_DENIED) {
|
|
10797
|
+
return "Conversation streaming access was denied.";
|
|
10798
|
+
}
|
|
10799
|
+
if (code === JOIN_FAILED) {
|
|
10800
|
+
return "The server could not join the conversation. Try again later.";
|
|
10801
|
+
}
|
|
10802
|
+
return `The server did not join the conversation (${code}). Try again later.`;
|
|
10803
|
+
};
|
|
10804
|
+
var closeMessage = function(reason, detail) {
|
|
10805
|
+
if (reason === "io server disconnect") {
|
|
10806
|
+
return "The server closed the conversation connection.";
|
|
10807
|
+
}
|
|
10808
|
+
if (reason === CLIENT_CLOSE) {
|
|
10809
|
+
return "The conversation connection closed.";
|
|
10810
|
+
}
|
|
10811
|
+
if (reason === "connect_error") {
|
|
10812
|
+
return `The server refused the conversation connection (${detail || "no reason"}).`;
|
|
10813
|
+
}
|
|
10814
|
+
return `The conversation connection was lost (${reason}).`;
|
|
10815
|
+
};
|
|
10816
|
+
var acknowledgementError = function(error, lost) {
|
|
10817
|
+
if (error.message === SOCKET_IO_TIMEOUT) {
|
|
10818
|
+
return new ConversationConnectionError(
|
|
10819
|
+
"CONVERSATION_TIMEOUT",
|
|
10820
|
+
"The server did not answer in time."
|
|
10821
|
+
);
|
|
10822
|
+
}
|
|
10823
|
+
return new ConversationConnectionError(
|
|
10824
|
+
"CONVERSATION_DISCONNECTED",
|
|
10825
|
+
closeMessage(lost ?? "transport close"),
|
|
10826
|
+
lost
|
|
10827
|
+
);
|
|
10828
|
+
};
|
|
10829
|
+
var refusedConversation = function(payload) {
|
|
10830
|
+
const exception = payload;
|
|
10831
|
+
const named = exception?.cause?.data?.conversationId;
|
|
10832
|
+
if (exception?.message !== ACCESS_DENIED || typeof named !== "string") {
|
|
10833
|
+
return null;
|
|
10834
|
+
}
|
|
10835
|
+
return named.trim() || null;
|
|
10836
|
+
};
|
|
10837
|
+
var ConversationSocket = class {
|
|
10838
|
+
options;
|
|
10839
|
+
connection = null;
|
|
10840
|
+
/**
|
|
10841
|
+
* The conversations that the server refused. A join with the same
|
|
10842
|
+
* credentials fails the same way, so the SDK does not try them again until
|
|
10843
|
+
* `disconnect`.
|
|
10844
|
+
*/
|
|
10845
|
+
refused = /* @__PURE__ */ new Set();
|
|
10846
|
+
constructor(options) {
|
|
10847
|
+
this.options = options;
|
|
10848
|
+
}
|
|
10849
|
+
/** Closes the connection and forgets the refused conversations. */
|
|
10850
|
+
disconnect() {
|
|
10851
|
+
this.refused.clear();
|
|
10852
|
+
if (this.connection) {
|
|
10853
|
+
this.close(this.connection, CLIENT_CLOSE);
|
|
10854
|
+
}
|
|
10855
|
+
}
|
|
10856
|
+
/** An event with an acknowledgement. Needs `ensureConnected` first. */
|
|
10857
|
+
async request(event, payload, timeoutMs = 1e4) {
|
|
10858
|
+
const connection = this.connection;
|
|
10859
|
+
if (!connection?.socket.connected) {
|
|
10860
|
+
throw new ConversationConnectionError(
|
|
10861
|
+
"CONVERSATION_DISCONNECTED",
|
|
10862
|
+
"The conversation connection is not open."
|
|
10863
|
+
);
|
|
10864
|
+
}
|
|
10865
|
+
return this.acknowledged(connection, event, payload, timeoutMs, null);
|
|
10866
|
+
}
|
|
10867
|
+
/** An event without an acknowledgement, in order with the other events. */
|
|
10868
|
+
send(event, payload) {
|
|
10869
|
+
this.connection?.socket.emit(event, payload);
|
|
10870
|
+
}
|
|
10871
|
+
/**
|
|
10872
|
+
* Joins the room of the conversation. A connection that the server closed
|
|
10873
|
+
* is not used again: the join opens a new one. A refused conversation fails
|
|
10874
|
+
* at once, without a connection.
|
|
10875
|
+
*/
|
|
10876
|
+
async ensureConnected(conversationId) {
|
|
10877
|
+
if (typeof window === "undefined") {
|
|
10878
|
+
return;
|
|
10879
|
+
}
|
|
10880
|
+
const normalized = conversationId.trim();
|
|
10881
|
+
if (!normalized) {
|
|
10882
|
+
return;
|
|
10883
|
+
}
|
|
10884
|
+
if (this.refused.has(normalized)) {
|
|
10885
|
+
throw new ConversationConnectionError(
|
|
10886
|
+
ACCESS_DENIED,
|
|
10887
|
+
joinMessage(ACCESS_DENIED)
|
|
10888
|
+
);
|
|
10889
|
+
}
|
|
10890
|
+
const connection = this.connection ?? this.open();
|
|
10891
|
+
await this.connected(connection);
|
|
10892
|
+
await this.acknowledged(
|
|
10893
|
+
connection,
|
|
10894
|
+
JOIN_EVENT,
|
|
10895
|
+
{ conversationId: normalized },
|
|
10896
|
+
this.options.joinTimeoutMs ?? JOIN_TIMEOUT_MS,
|
|
10897
|
+
normalized
|
|
10898
|
+
);
|
|
10899
|
+
}
|
|
10900
|
+
open() {
|
|
10901
|
+
const socketBaseUrl = (() => {
|
|
10902
|
+
try {
|
|
10903
|
+
return new URL(this.options.baseUrl).origin;
|
|
10904
|
+
} catch {
|
|
10905
|
+
return window.location.origin;
|
|
10906
|
+
}
|
|
10907
|
+
})();
|
|
10908
|
+
const socket = io(socketBaseUrl, {
|
|
10909
|
+
transports: ["websocket"],
|
|
10910
|
+
withCredentials: true,
|
|
10911
|
+
forceNew: true,
|
|
10912
|
+
auth: {
|
|
10913
|
+
apiToken: this.options.apiToken
|
|
10914
|
+
}
|
|
10915
|
+
});
|
|
10916
|
+
const connection = {
|
|
10917
|
+
socket,
|
|
10918
|
+
pending: /* @__PURE__ */ new Set(),
|
|
10919
|
+
connectedBefore: false,
|
|
10920
|
+
lost: null,
|
|
10921
|
+
closed: null,
|
|
10922
|
+
failure: null
|
|
10923
|
+
};
|
|
10924
|
+
this.connection = connection;
|
|
10925
|
+
this.watch(connection);
|
|
10926
|
+
this.receive(socket);
|
|
10927
|
+
return connection;
|
|
10928
|
+
}
|
|
10929
|
+
/** The lifecycle of one connection (docs/streaming_reply.md#web-sdk). */
|
|
10930
|
+
watch(connection) {
|
|
10931
|
+
const { socket } = connection;
|
|
10932
|
+
socket.on("connect", () => {
|
|
10933
|
+
if (!connection.connectedBefore) {
|
|
10934
|
+
connection.connectedBefore = true;
|
|
10935
|
+
return;
|
|
10936
|
+
}
|
|
10937
|
+
const activeConversationId = this.options.getConversationId()?.trim();
|
|
10938
|
+
if (activeConversationId && !this.refused.has(activeConversationId)) {
|
|
10939
|
+
socket.emit(JOIN_EVENT, { conversationId: activeConversationId });
|
|
10940
|
+
}
|
|
10941
|
+
});
|
|
10942
|
+
socket.on("exception", (payload) => {
|
|
10943
|
+
const conversationId = refusedConversation(payload);
|
|
10944
|
+
if (conversationId) {
|
|
10945
|
+
connection.failure ??= { code: ACCESS_DENIED, conversationId };
|
|
10946
|
+
this.refused.add(conversationId);
|
|
10947
|
+
}
|
|
10948
|
+
});
|
|
10949
|
+
socket.on("disconnect", (reason) => {
|
|
10950
|
+
if (!socket.active) {
|
|
10951
|
+
this.close(connection, reason);
|
|
10952
|
+
return;
|
|
10953
|
+
}
|
|
10954
|
+
connection.lost = reason;
|
|
10955
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10956
|
+
});
|
|
10957
|
+
socket.on("connect_error", (error) => {
|
|
10958
|
+
if (!socket.active) {
|
|
10959
|
+
this.close(connection, "connect_error", error.message);
|
|
10960
|
+
}
|
|
10961
|
+
});
|
|
10962
|
+
}
|
|
10963
|
+
/**
|
|
10964
|
+
* Closes the connection for good: socket.io does not open it again. The
|
|
10965
|
+
* waits fail now, and the next join opens a new connection.
|
|
10966
|
+
*/
|
|
10967
|
+
close(connection, reason, detail) {
|
|
10968
|
+
if (connection.closed) {
|
|
10969
|
+
return;
|
|
10970
|
+
}
|
|
10971
|
+
connection.closed = { reason, message: closeMessage(reason, detail) };
|
|
10972
|
+
if (this.connection === connection) {
|
|
10973
|
+
this.connection = null;
|
|
10974
|
+
}
|
|
10975
|
+
const { socket } = connection;
|
|
10976
|
+
socket.removeAllListeners();
|
|
10977
|
+
if (connection.connectedBefore) {
|
|
10978
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
10979
|
+
}
|
|
10980
|
+
const pending = [...connection.pending];
|
|
10981
|
+
connection.pending.clear();
|
|
10982
|
+
for (const entry of pending) {
|
|
10983
|
+
entry.fail(this.closeError(connection, entry.conversationId));
|
|
10984
|
+
}
|
|
10985
|
+
socket.disconnect();
|
|
10986
|
+
if (reason !== CLIENT_CLOSE) {
|
|
10987
|
+
this.options.handleConnectionClosed?.({
|
|
10988
|
+
reason,
|
|
10989
|
+
code: connection.failure?.code ?? null,
|
|
10990
|
+
conversationId: connection.failure?.conversationId ?? null
|
|
10991
|
+
});
|
|
10992
|
+
}
|
|
10993
|
+
}
|
|
10994
|
+
closeError(connection, conversationId) {
|
|
10995
|
+
if (conversationId !== null && this.refused.has(conversationId)) {
|
|
10996
|
+
return new ConversationConnectionError(
|
|
10997
|
+
ACCESS_DENIED,
|
|
10998
|
+
joinMessage(ACCESS_DENIED)
|
|
10999
|
+
);
|
|
11000
|
+
}
|
|
11001
|
+
return new ConversationConnectionError(
|
|
11002
|
+
"CONVERSATION_DISCONNECTED",
|
|
11003
|
+
connection.closed?.message ?? closeMessage(CLIENT_CLOSE),
|
|
11004
|
+
connection.closed?.reason ?? null
|
|
11005
|
+
);
|
|
11006
|
+
}
|
|
11007
|
+
/**
|
|
11008
|
+
* Waits for the first connection. A connection error that socket.io
|
|
11009
|
+
* recovers from also ends the wait: the join then waits in socket.io's
|
|
11010
|
+
* buffer, for the time of its acknowledgement at most.
|
|
11011
|
+
*/
|
|
11012
|
+
connected(connection) {
|
|
11013
|
+
const { socket } = connection;
|
|
11014
|
+
if (connection.closed) {
|
|
11015
|
+
return Promise.reject(this.closeError(connection, null));
|
|
11016
|
+
}
|
|
11017
|
+
if (socket.connected || connection.connectedBefore) {
|
|
11018
|
+
return Promise.resolve();
|
|
11019
|
+
}
|
|
11020
|
+
return new Promise((resolve, reject) => {
|
|
11021
|
+
const entry = { conversationId: null, fail: reject };
|
|
11022
|
+
const done = () => {
|
|
11023
|
+
if (connection.pending.delete(entry)) {
|
|
11024
|
+
socket.off("connect", done);
|
|
11025
|
+
socket.off("connect_error", done);
|
|
11026
|
+
resolve();
|
|
11027
|
+
}
|
|
11028
|
+
};
|
|
11029
|
+
connection.pending.add(entry);
|
|
11030
|
+
socket.on("connect", done);
|
|
11031
|
+
socket.on("connect_error", done);
|
|
11032
|
+
});
|
|
11033
|
+
}
|
|
11034
|
+
/**
|
|
11035
|
+
* Emits an event and waits for its acknowledgement. A join (`joining`)
|
|
11036
|
+
* that the server does not accept fails with the code of its
|
|
11037
|
+
* acknowledgement. A close of the connection fails the wait at once; a
|
|
11038
|
+
* timeout fails it with `CONVERSATION_TIMEOUT`.
|
|
11039
|
+
*/
|
|
11040
|
+
acknowledged(connection, event, payload, timeoutMs, joining) {
|
|
11041
|
+
return new Promise((resolve, reject) => {
|
|
11042
|
+
if (connection.closed) {
|
|
11043
|
+
reject(this.closeError(connection, joining));
|
|
11044
|
+
return;
|
|
11045
|
+
}
|
|
11046
|
+
const entry = { conversationId: joining, fail: reject };
|
|
11047
|
+
connection.pending.add(entry);
|
|
11048
|
+
connection.socket.timeout(timeoutMs).emit(event, payload, (error, result) => {
|
|
11049
|
+
if (!connection.pending.delete(entry)) {
|
|
11050
|
+
return;
|
|
11051
|
+
}
|
|
11052
|
+
const failure = error ? acknowledgementError(error, connection.lost) : joining === null ? null : this.joinFailure(connection, joining, result);
|
|
11053
|
+
if (failure) {
|
|
11054
|
+
reject(failure);
|
|
11055
|
+
} else {
|
|
11056
|
+
resolve(result);
|
|
11057
|
+
}
|
|
11058
|
+
});
|
|
11059
|
+
});
|
|
11060
|
+
}
|
|
11061
|
+
/** The error of a join that the server did not accept; null after a join. */
|
|
11062
|
+
joinFailure(connection, conversationId, result) {
|
|
11063
|
+
const ack = result;
|
|
11064
|
+
if (ack?.ok) {
|
|
11065
|
+
return null;
|
|
11066
|
+
}
|
|
11067
|
+
const named = typeof ack?.error === "string" ? ack.error.trim() : "";
|
|
11068
|
+
const code = named || JOIN_FAILED;
|
|
11069
|
+
connection.failure ??= { code, conversationId };
|
|
11070
|
+
if (code === ACCESS_DENIED) {
|
|
11071
|
+
this.refused.add(conversationId);
|
|
11072
|
+
}
|
|
11073
|
+
this.close(connection, "io server disconnect");
|
|
11074
|
+
return new ConversationConnectionError(code, joinMessage(code));
|
|
11075
|
+
}
|
|
11076
|
+
/** The events that the server sends to the conversation's room. */
|
|
11077
|
+
receive(socket) {
|
|
11078
|
+
socket.on("conversations:stream", (payload) => {
|
|
11079
|
+
let candidate;
|
|
11080
|
+
try {
|
|
11081
|
+
candidate = decodePerformance(
|
|
11082
|
+
payload,
|
|
11083
|
+
"stream"
|
|
11084
|
+
);
|
|
11085
|
+
} catch (error) {
|
|
11086
|
+
this.options.handleCallbackError("stream.decode", error);
|
|
11087
|
+
return;
|
|
11088
|
+
}
|
|
11089
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
11090
|
+
return;
|
|
11091
|
+
}
|
|
11092
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
11093
|
+
return;
|
|
11094
|
+
}
|
|
11095
|
+
const event = candidate.event;
|
|
11096
|
+
if (!event || typeof event !== "object") {
|
|
11097
|
+
return;
|
|
11098
|
+
}
|
|
11099
|
+
this.options.handleStreamEvent(event);
|
|
11100
|
+
});
|
|
11101
|
+
socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
11102
|
+
const event = payload;
|
|
11103
|
+
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
11104
|
+
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
11105
|
+
}
|
|
11106
|
+
});
|
|
11107
|
+
socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
11108
|
+
let candidate;
|
|
11109
|
+
try {
|
|
11110
|
+
candidate = decodePerformance(
|
|
11111
|
+
payload,
|
|
11112
|
+
"turn"
|
|
11113
|
+
);
|
|
11114
|
+
} catch (error) {
|
|
11115
|
+
this.options.handleCallbackError("speech.turn.decode", error);
|
|
11116
|
+
return;
|
|
11117
|
+
}
|
|
11118
|
+
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
11119
|
+
return;
|
|
11120
|
+
}
|
|
11121
|
+
this.options.handleSpeechEvent?.({
|
|
11122
|
+
type: "turn",
|
|
11123
|
+
requestId: candidate.requestId,
|
|
11124
|
+
turn: candidate.turn
|
|
11125
|
+
});
|
|
11126
|
+
});
|
|
11127
|
+
socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
11128
|
+
const event = payload;
|
|
11129
|
+
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
11130
|
+
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
11131
|
+
}
|
|
11132
|
+
});
|
|
11133
|
+
socket.on(SIGHT_EVENTS.observation, (payload) => {
|
|
11134
|
+
if (isSightObservationEvent(payload) && payload.conversationId === this.options.getConversationId()) {
|
|
11135
|
+
this.options.handleSightEvent?.(payload);
|
|
11136
|
+
}
|
|
11137
|
+
});
|
|
11138
|
+
socket.on("conversations:turn", (payload) => {
|
|
11139
|
+
let candidate;
|
|
11140
|
+
try {
|
|
11141
|
+
candidate = decodePerformance(
|
|
11142
|
+
payload,
|
|
11143
|
+
"turn"
|
|
11144
|
+
);
|
|
11145
|
+
} catch (error) {
|
|
11146
|
+
this.options.handleCallbackError("turn.decode", error);
|
|
11147
|
+
return;
|
|
11148
|
+
}
|
|
11149
|
+
if (typeof candidate?.conversationId !== "string") {
|
|
11150
|
+
return;
|
|
11151
|
+
}
|
|
11152
|
+
if (candidate.conversationId !== this.options.getConversationId()) {
|
|
11153
|
+
return;
|
|
11154
|
+
}
|
|
11155
|
+
const turn = candidate.turn;
|
|
11156
|
+
if (!turn || typeof turn !== "object") {
|
|
11157
|
+
return;
|
|
11158
|
+
}
|
|
11159
|
+
void this.options.handleTurn(turn).catch((error) => {
|
|
11160
|
+
this.options.handleCallbackError("onTurn", error);
|
|
11161
|
+
});
|
|
11162
|
+
});
|
|
11163
|
+
}
|
|
11164
|
+
};
|
|
11165
|
+
|
|
11166
|
+
// ../../clients/web-client/src/input-mode-rules.ts
|
|
11167
|
+
var InputModeRules = class {
|
|
11168
|
+
configured;
|
|
11169
|
+
resolved = null;
|
|
11170
|
+
/** The character's voice input switch, after the character loads. */
|
|
11171
|
+
speechInput = null;
|
|
11172
|
+
latestOptions = [];
|
|
11173
|
+
/**
|
|
11174
|
+
* Throws an `InputModeError` at once for a malformed configuration, and
|
|
11175
|
+
* for a baked asset with a mode other than `options`.
|
|
11176
|
+
*/
|
|
11177
|
+
constructor(inputModes, bakedAsset) {
|
|
11178
|
+
this.configured = parseInputModes(inputModes);
|
|
11179
|
+
if (this.configured && bakedAsset) {
|
|
11180
|
+
resolveInputModes(this.configured, {
|
|
11181
|
+
allowCustomReply: false,
|
|
11182
|
+
baked: true,
|
|
11183
|
+
speechInput: false
|
|
11184
|
+
});
|
|
11185
|
+
}
|
|
11186
|
+
}
|
|
11187
|
+
/**
|
|
11188
|
+
* The character loaded. Throws an `InputModeError` when the configuration
|
|
11189
|
+
* has a mode that the character does not permit.
|
|
11190
|
+
*/
|
|
11191
|
+
load(permissions) {
|
|
11192
|
+
this.speechInput = permissions.speechInput;
|
|
11193
|
+
this.resolved = resolveInputModes(this.configured, permissions);
|
|
11194
|
+
}
|
|
11195
|
+
/**
|
|
11196
|
+
* The input modes in use: the configured modes, or every mode that the
|
|
11197
|
+
* character permits. Before the character loads, the configuration or all
|
|
11198
|
+
* modes.
|
|
11199
|
+
*/
|
|
11200
|
+
get modes() {
|
|
11201
|
+
return this.resolved ?? this.configured ?? permittedInputModes({
|
|
11202
|
+
allowCustomReply: true,
|
|
11203
|
+
baked: false,
|
|
11204
|
+
speechInput: true
|
|
11205
|
+
});
|
|
11206
|
+
}
|
|
11207
|
+
/** The reply options of the latest result, for `assertReply`. */
|
|
11208
|
+
setOptions(options) {
|
|
11209
|
+
this.latestOptions = options;
|
|
11210
|
+
}
|
|
11211
|
+
/** Throws when the input modes do not include `mode`. */
|
|
11212
|
+
assertMode(mode, baked) {
|
|
11213
|
+
if (mode === "voice" && baked) {
|
|
11214
|
+
throw new InputModeError(
|
|
11215
|
+
"Voice input is not available in a baked (offline) conversation."
|
|
11216
|
+
);
|
|
11217
|
+
}
|
|
11218
|
+
if (mode === "voice" && this.speechInput === false) {
|
|
11219
|
+
throw new InputModeError(
|
|
11220
|
+
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
11221
|
+
);
|
|
11222
|
+
}
|
|
11223
|
+
const modes = this.modes;
|
|
11224
|
+
if (!modes.includes(mode)) {
|
|
11225
|
+
throw new InputModeError(
|
|
11226
|
+
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
11227
|
+
);
|
|
11228
|
+
}
|
|
11229
|
+
}
|
|
11230
|
+
/**
|
|
11231
|
+
* A typed reply needs the `text` mode; one of the current reply options
|
|
11232
|
+
* needs `options` or `text`. Without a configuration the server's rules
|
|
11233
|
+
* apply alone, as before.
|
|
11234
|
+
*/
|
|
11235
|
+
assertReply(message) {
|
|
11236
|
+
if (!this.configured) {
|
|
11237
|
+
return;
|
|
11238
|
+
}
|
|
11239
|
+
const modes = this.modes;
|
|
11240
|
+
if (modes.includes("text")) {
|
|
11241
|
+
return;
|
|
11242
|
+
}
|
|
11243
|
+
const reply = message.trim();
|
|
11244
|
+
const isOption = this.latestOptions.some(
|
|
11245
|
+
(option) => option.label.trim() === reply
|
|
11246
|
+
);
|
|
11247
|
+
if (isOption && modes.includes("options")) {
|
|
11248
|
+
return;
|
|
11249
|
+
}
|
|
11250
|
+
throw new InputModeError(
|
|
11251
|
+
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
11252
|
+
);
|
|
11253
|
+
}
|
|
11254
|
+
};
|
|
11255
|
+
|
|
11256
|
+
// ../../clients/web-client/src/voice/push-to-talk.ts
|
|
11257
|
+
var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
|
|
11258
|
+
var elementOf = function(target) {
|
|
11259
|
+
return target && typeof target.closest === "function" ? target : null;
|
|
11260
|
+
};
|
|
11261
|
+
var bindPushToTalk = function(target, options = {}) {
|
|
11262
|
+
const element = options.element ?? null;
|
|
11263
|
+
const key = options.key === void 0 ? "Space" : options.key;
|
|
11264
|
+
const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
|
|
11265
|
+
let active = null;
|
|
11266
|
+
let pointerId = null;
|
|
11267
|
+
let pressedAt = 0;
|
|
11268
|
+
const begin = function(source) {
|
|
11269
|
+
if (active || target.listeningState !== "idle") {
|
|
11270
|
+
return false;
|
|
10033
11271
|
}
|
|
10034
11272
|
if (options.enabled && !options.enabled()) {
|
|
10035
11273
|
return false;
|
|
@@ -10967,18 +12205,18 @@ var Character = class extends CharacterRuntime {
|
|
|
10967
12205
|
notifiedStagingIds = /* @__PURE__ */ new Map();
|
|
10968
12206
|
liveConversation;
|
|
10969
12207
|
baked = null;
|
|
12208
|
+
/** The conversation whose live motion was warmed up. */
|
|
12209
|
+
liveMotionPrepared = null;
|
|
10970
12210
|
authoring;
|
|
10971
12211
|
normalizeIdleOrientation;
|
|
10972
12212
|
bakedAsset;
|
|
10973
12213
|
animationRefreshId = 0;
|
|
10974
12214
|
voice;
|
|
10975
|
-
|
|
10976
|
-
|
|
10977
|
-
/** The character's voice input switch, after the character loads. */
|
|
10978
|
-
speechInput = null;
|
|
10979
|
-
latestOptions = [];
|
|
12215
|
+
sight;
|
|
12216
|
+
inputs;
|
|
10980
12217
|
listeningStateCallback = null;
|
|
10981
12218
|
transcriptCallback = null;
|
|
12219
|
+
connectionClosedCallback = null;
|
|
10982
12220
|
conversationId = null;
|
|
10983
12221
|
constructor(options) {
|
|
10984
12222
|
const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
|
|
@@ -10999,14 +12237,10 @@ var Character = class extends CharacterRuntime {
|
|
|
10999
12237
|
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
11000
12238
|
this.bakedAsset = options.bakedConversation;
|
|
11001
12239
|
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
11002
|
-
this.
|
|
11003
|
-
|
|
11004
|
-
|
|
11005
|
-
|
|
11006
|
-
baked: true,
|
|
11007
|
-
speechInput: false
|
|
11008
|
-
});
|
|
11009
|
-
}
|
|
12240
|
+
this.inputs = new InputModeRules(
|
|
12241
|
+
options.inputModes,
|
|
12242
|
+
Boolean(this.bakedAsset) && !this.authoring
|
|
12243
|
+
);
|
|
11010
12244
|
this.conversationSocket = new ConversationSocket({
|
|
11011
12245
|
baseUrl: this.baseUrl,
|
|
11012
12246
|
apiToken: this.apiToken,
|
|
@@ -11021,13 +12255,24 @@ var Character = class extends CharacterRuntime {
|
|
|
11021
12255
|
await this.handleStagingResult(turn, { source: "turn" });
|
|
11022
12256
|
},
|
|
11023
12257
|
handleCallbackError: this.handleCallbackError,
|
|
11024
|
-
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
|
|
12258
|
+
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event),
|
|
12259
|
+
handleSightEvent: (event) => this.sight.handleObservation(event),
|
|
12260
|
+
handleConnectionClosed: (event) => invokeSafeCallback(
|
|
12261
|
+
"onConnectionClosed",
|
|
12262
|
+
this.handleCallbackError,
|
|
12263
|
+
() => this.connectionClosedCallback?.(event)
|
|
12264
|
+
)
|
|
12265
|
+
});
|
|
12266
|
+
this.sight = new SightInput({
|
|
12267
|
+
socket: () => this.conversationSocket,
|
|
12268
|
+
conversationId: () => this.conversationId,
|
|
12269
|
+
handleCallbackError: this.handleCallbackError
|
|
11025
12270
|
});
|
|
11026
12271
|
this.voice = new VoiceInput({
|
|
11027
12272
|
socket: () => this.conversationSocket,
|
|
11028
12273
|
conversationId: () => this.conversationId,
|
|
11029
12274
|
renderSessionId: () => this.renderSessionId,
|
|
11030
|
-
assertVoiceAllowed: () => this.
|
|
12275
|
+
assertVoiceAllowed: () => this.inputs.assertMode("voice", this.baked !== null),
|
|
11031
12276
|
turnInFlight: () => this.liveConversation.inFlight,
|
|
11032
12277
|
beginTurn: () => {
|
|
11033
12278
|
if (this.conversationId) {
|
|
@@ -11091,12 +12336,12 @@ var Character = class extends CharacterRuntime {
|
|
|
11091
12336
|
const progressReporter = createAssetProgressReporter(callback);
|
|
11092
12337
|
const info = await this.getAssetInfo();
|
|
11093
12338
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11094
|
-
this.
|
|
11095
|
-
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
12339
|
+
this.inputs.load({
|
|
11096
12340
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11097
12341
|
baked,
|
|
11098
|
-
speechInput:
|
|
12342
|
+
speechInput: info.speechInputEnabled === true
|
|
11099
12343
|
});
|
|
12344
|
+
this.sight.configure({ enabled: info.sight?.enabled === true, baked });
|
|
11100
12345
|
this.baked = baked ? await loadBakedConversation({
|
|
11101
12346
|
baseUrl: this.baseUrl,
|
|
11102
12347
|
characterId: this.characterId,
|
|
@@ -11166,14 +12411,36 @@ var Character = class extends CharacterRuntime {
|
|
|
11166
12411
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
11167
12412
|
throw new Error("Baked conversation requires local rendering.");
|
|
11168
12413
|
}
|
|
11169
|
-
this.
|
|
11170
|
-
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
12414
|
+
this.inputs.load({
|
|
11171
12415
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11172
12416
|
baked: false,
|
|
11173
|
-
speechInput:
|
|
12417
|
+
speechInput: info.speechInputEnabled === true
|
|
12418
|
+
});
|
|
12419
|
+
this.sight.configure({
|
|
12420
|
+
enabled: info.sight?.enabled === true,
|
|
12421
|
+
baked: false
|
|
11174
12422
|
});
|
|
11175
12423
|
this.animationController = new RemoteAnimationController();
|
|
11176
12424
|
}
|
|
12425
|
+
/**
|
|
12426
|
+
* Warms up live motion for the current conversation, once, on the
|
|
12427
|
+
* player's intent: call it at the first focus or key in a reply field or
|
|
12428
|
+
* when a talk key goes down. It does nothing for a character without live
|
|
12429
|
+
* motion; the server limits warm-ups per player (ADR 0011).
|
|
12430
|
+
*/
|
|
12431
|
+
async prepareLiveMotion() {
|
|
12432
|
+
const conversationId = this.conversationId;
|
|
12433
|
+
if (this.baked || !conversationId || this.liveMotionPrepared === conversationId) {
|
|
12434
|
+
return false;
|
|
12435
|
+
}
|
|
12436
|
+
this.liveMotionPrepared = conversationId;
|
|
12437
|
+
return warmUpLiveMotion({
|
|
12438
|
+
conversationId,
|
|
12439
|
+
baseUrl: this.baseUrl,
|
|
12440
|
+
apiToken: this.apiToken,
|
|
12441
|
+
renderSessionId: this.renderSessionId
|
|
12442
|
+
});
|
|
12443
|
+
}
|
|
11177
12444
|
async createConversation(options) {
|
|
11178
12445
|
const response = this.baked ? {
|
|
11179
12446
|
conversation: {
|
|
@@ -11215,6 +12482,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11215
12482
|
}
|
|
11216
12483
|
async disconnect() {
|
|
11217
12484
|
this.voice.reset();
|
|
12485
|
+
this.sight.stop();
|
|
11218
12486
|
this.thinkingState = false;
|
|
11219
12487
|
if (this.animationController) {
|
|
11220
12488
|
this.streamPlayback.cleanup();
|
|
@@ -11250,7 +12518,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11250
12518
|
maybeMessageOrOptions,
|
|
11251
12519
|
maybeOptions
|
|
11252
12520
|
);
|
|
11253
|
-
this.
|
|
12521
|
+
this.inputs.assertReply(message);
|
|
11254
12522
|
if (this.voice.turnPending) {
|
|
11255
12523
|
throw new Error("A reply is already being generated. Please wait.");
|
|
11256
12524
|
}
|
|
@@ -11298,11 +12566,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11298
12566
|
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11299
12567
|
*/
|
|
11300
12568
|
get inputModes() {
|
|
11301
|
-
return this.
|
|
11302
|
-
allowCustomReply: true,
|
|
11303
|
-
baked: false,
|
|
11304
|
-
speechInput: true
|
|
11305
|
-
});
|
|
12569
|
+
return this.inputs.modes;
|
|
11306
12570
|
}
|
|
11307
12571
|
get listeningState() {
|
|
11308
12572
|
return this.voice.listeningState;
|
|
@@ -11346,6 +12610,48 @@ var Character = class extends CharacterRuntime {
|
|
|
11346
12610
|
callback
|
|
11347
12611
|
);
|
|
11348
12612
|
}
|
|
12613
|
+
/**
|
|
12614
|
+
* `'unavailable'` until the character info turns sight on (its owner's
|
|
12615
|
+
* Sight tab), and in a baked conversation; then `'off'`, `'starting'` or
|
|
12616
|
+
* `'on'`.
|
|
12617
|
+
*/
|
|
12618
|
+
get sightState() {
|
|
12619
|
+
return this.sight.state;
|
|
12620
|
+
}
|
|
12621
|
+
/**
|
|
12622
|
+
* Lets the character see the player's camera (docs/sight.md). Needs a
|
|
12623
|
+
* conversation; the first call asks for the camera permission. Rejects
|
|
12624
|
+
* with a `SightError`: `SIGHT_DISABLED` when sight is unavailable,
|
|
12625
|
+
* `CONVERSATION_REQUIRED`, `CAMERA_PERMISSION_DENIED`, ...
|
|
12626
|
+
*/
|
|
12627
|
+
startSight(options) {
|
|
12628
|
+
return this.sight.start(options);
|
|
12629
|
+
}
|
|
12630
|
+
/** Stops the frames. Closes the camera that `startSight` opened. */
|
|
12631
|
+
stopSight() {
|
|
12632
|
+
this.sight.stop();
|
|
12633
|
+
}
|
|
12634
|
+
/** What the character saw: each observation of the current conversation. */
|
|
12635
|
+
onSight(callback) {
|
|
12636
|
+
return this.sight.onObservation(callback);
|
|
12637
|
+
}
|
|
12638
|
+
/** The sight state. `error` is the `SightError` that stopped sight. */
|
|
12639
|
+
onSightState(callback) {
|
|
12640
|
+
return this.sight.onState(callback);
|
|
12641
|
+
}
|
|
12642
|
+
/**
|
|
12643
|
+
* The server closed the conversation connection: a refused or failed join,
|
|
12644
|
+
* or an API key that was rotated, edited or deleted. The next `connect`,
|
|
12645
|
+
* `sendMessage` or `startListening` opens a new connection; a refused
|
|
12646
|
+
* conversation fails at once with `CONVERSATION_ACCESS_DENIED`.
|
|
12647
|
+
*/
|
|
12648
|
+
onConnectionClosed(callback) {
|
|
12649
|
+
return bindCallbackSlot(
|
|
12650
|
+
() => this.connectionClosedCallback,
|
|
12651
|
+
(next) => this.connectionClosedCallback = next,
|
|
12652
|
+
callback
|
|
12653
|
+
);
|
|
12654
|
+
}
|
|
11349
12655
|
async getConversationWorldline(conversationId = this.conversationId) {
|
|
11350
12656
|
if (!conversationId) {
|
|
11351
12657
|
throw new Error("Conversation ID is required.");
|
|
@@ -11409,7 +12715,7 @@ var Character = class extends CharacterRuntime {
|
|
|
11409
12715
|
async handleStagingResult(payload, options) {
|
|
11410
12716
|
this.thinkingState = false;
|
|
11411
12717
|
const normalized = normalizeStagingResultPayload(payload);
|
|
11412
|
-
this.
|
|
12718
|
+
this.inputs.setOptions(normalized.options);
|
|
11413
12719
|
if (options.source === "turn") {
|
|
11414
12720
|
this.liveConversation.received(normalized);
|
|
11415
12721
|
}
|
|
@@ -11436,49 +12742,6 @@ var Character = class extends CharacterRuntime {
|
|
|
11436
12742
|
void this.streamPlayback.playPayload(normalized);
|
|
11437
12743
|
}
|
|
11438
12744
|
}
|
|
11439
|
-
/** Throws when the input modes do not include `mode`. */
|
|
11440
|
-
assertInputMode(mode) {
|
|
11441
|
-
if (mode === "voice" && this.baked) {
|
|
11442
|
-
throw new InputModeError(
|
|
11443
|
-
"Voice input is not available in a baked (offline) conversation."
|
|
11444
|
-
);
|
|
11445
|
-
}
|
|
11446
|
-
if (mode === "voice" && this.speechInput === false) {
|
|
11447
|
-
throw new InputModeError(
|
|
11448
|
-
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
11449
|
-
);
|
|
11450
|
-
}
|
|
11451
|
-
const modes = this.inputModes;
|
|
11452
|
-
if (!modes.includes(mode)) {
|
|
11453
|
-
throw new InputModeError(
|
|
11454
|
-
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
11455
|
-
);
|
|
11456
|
-
}
|
|
11457
|
-
}
|
|
11458
|
-
/**
|
|
11459
|
-
* A typed reply needs the `text` mode; one of the current reply options
|
|
11460
|
-
* needs `options` or `text`. Without a configuration the server's rules
|
|
11461
|
-
* apply alone, as before.
|
|
11462
|
-
*/
|
|
11463
|
-
assertReplyMode(message) {
|
|
11464
|
-
if (!this.configuredInputModes) {
|
|
11465
|
-
return;
|
|
11466
|
-
}
|
|
11467
|
-
const modes = this.inputModes;
|
|
11468
|
-
if (modes.includes("text")) {
|
|
11469
|
-
return;
|
|
11470
|
-
}
|
|
11471
|
-
const reply = message.trim();
|
|
11472
|
-
const isOption = this.latestOptions.some(
|
|
11473
|
-
(option) => option.label.trim() === reply
|
|
11474
|
-
);
|
|
11475
|
-
if (isOption && modes.includes("options")) {
|
|
11476
|
-
return;
|
|
11477
|
-
}
|
|
11478
|
-
throw new InputModeError(
|
|
11479
|
-
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
11480
|
-
);
|
|
11481
|
-
}
|
|
11482
12745
|
invokeThinkingCallback(payload) {
|
|
11483
12746
|
this.thinkingState = true;
|
|
11484
12747
|
if (this.thinkingCallback) {
|
|
@@ -12075,10 +13338,12 @@ export {
|
|
|
12075
13338
|
CharacterExpressions,
|
|
12076
13339
|
CharacterRuntime,
|
|
12077
13340
|
CharacterSceneRig,
|
|
13341
|
+
ConversationConnectionError,
|
|
12078
13342
|
FacialPerformanceLayer,
|
|
12079
13343
|
FacialPerformanceMixer,
|
|
12080
13344
|
INPUT_MODES,
|
|
12081
13345
|
InputModeError,
|
|
13346
|
+
LiveMotionPlayer,
|
|
12082
13347
|
MicrophoneError,
|
|
12083
13348
|
Npcify,
|
|
12084
13349
|
Character as NpcifyCharacter,
|
|
@@ -12086,6 +13351,7 @@ export {
|
|
|
12086
13351
|
PERFORMANCE_MEDIA_TYPE,
|
|
12087
13352
|
PerformanceRecordingSchemaVersion,
|
|
12088
13353
|
RemoteAnimationController,
|
|
13354
|
+
SightError,
|
|
12089
13355
|
SpeechInputError,
|
|
12090
13356
|
analyzePerformanceRecording,
|
|
12091
13357
|
bindPushToTalk,
|