@npcify/web 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +65 -3
- package/dist/index.d.ts +208 -4
- package/dist/index.js +1960 -106
- package/examples/index.html +1 -0
- package/examples/main.js +40 -12
- package/examples/three-game.js +13 -3
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -6,9 +6,186 @@ var SERVER_RENDER_PROTOCOL_VERSION = 3;
|
|
|
6
6
|
var MAXIMUM_PLAYBACK_DURATION_SECONDS = 10 * 60;
|
|
7
7
|
|
|
8
8
|
// ../ts-shared/src/tts.ts
|
|
9
|
-
var TTS_PROVIDERS = ["11labs"];
|
|
9
|
+
var TTS_PROVIDERS = ["11labs", "gemini"];
|
|
10
10
|
var DEFAULT_TTS_PROVIDER = TTS_PROVIDERS[0];
|
|
11
11
|
|
|
12
|
+
// ../ts-shared/src/ai-settings.ts
|
|
13
|
+
var GPT_IMAGE_MODELS = [
|
|
14
|
+
"gpt-image-2.5-flare",
|
|
15
|
+
"gpt-image-2.5-sunburst"
|
|
16
|
+
];
|
|
17
|
+
var GEMINI_SPEECH_MODEL_VENDORS = {
|
|
18
|
+
"gemini-3.5-transcribe-live": "gemini",
|
|
19
|
+
"gemini-3.5-transcribe": "gemini"
|
|
20
|
+
};
|
|
21
|
+
var AI_ROLES = {
|
|
22
|
+
character_draft: {
|
|
23
|
+
name: "Character draft",
|
|
24
|
+
source: "llm",
|
|
25
|
+
models: [],
|
|
26
|
+
options: ["temperature"]
|
|
27
|
+
},
|
|
28
|
+
motion_planner: {
|
|
29
|
+
name: "Motion planner",
|
|
30
|
+
source: "llm",
|
|
31
|
+
models: [],
|
|
32
|
+
options: []
|
|
33
|
+
},
|
|
34
|
+
// Not charged.
|
|
35
|
+
prompt_check: {
|
|
36
|
+
name: "Prompt check",
|
|
37
|
+
source: "openai",
|
|
38
|
+
models: ["gpt-6-luna"],
|
|
39
|
+
options: []
|
|
40
|
+
},
|
|
41
|
+
// Priced by OPENAI_IMAGE_PRICES of the server (flare and sunburst alike).
|
|
42
|
+
design_sheet_image: {
|
|
43
|
+
name: "Design sheet image",
|
|
44
|
+
source: "openai",
|
|
45
|
+
models: GPT_IMAGE_MODELS,
|
|
46
|
+
options: []
|
|
47
|
+
},
|
|
48
|
+
// Not charged per call.
|
|
49
|
+
tts: {
|
|
50
|
+
name: "ElevenLabs text to speech",
|
|
51
|
+
source: "elevenlabs",
|
|
52
|
+
models: ["eleven_v3_conversational", "eleven_v3"],
|
|
53
|
+
options: []
|
|
54
|
+
},
|
|
55
|
+
// Not charged per call.
|
|
56
|
+
gemini_tts: {
|
|
57
|
+
name: "Gemini text to speech",
|
|
58
|
+
source: "gemini",
|
|
59
|
+
models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
60
|
+
options: []
|
|
61
|
+
},
|
|
62
|
+
// Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
|
|
63
|
+
// Audio that arrives while the player speaks. The model selects the vendor.
|
|
64
|
+
speech_input_live: {
|
|
65
|
+
name: "Speech input (live)",
|
|
66
|
+
source: "openai",
|
|
67
|
+
models: [
|
|
68
|
+
"gpt-transcribe",
|
|
69
|
+
"gpt-live-transcribe",
|
|
70
|
+
"gemini-3.5-transcribe-live",
|
|
71
|
+
"gemini-3.5-transcribe"
|
|
72
|
+
],
|
|
73
|
+
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
74
|
+
options: []
|
|
75
|
+
},
|
|
76
|
+
// One complete recording (an upload of an engine SDK).
|
|
77
|
+
speech_input_file: {
|
|
78
|
+
name: "Speech input (file)",
|
|
79
|
+
source: "openai",
|
|
80
|
+
models: ["gpt-transcribe", "gemini-3.5-transcribe"],
|
|
81
|
+
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
82
|
+
options: []
|
|
83
|
+
},
|
|
84
|
+
// Priced by OPENAI_PRICES.text of the generator.
|
|
85
|
+
generator_agent: {
|
|
86
|
+
name: "Generator agent",
|
|
87
|
+
source: "openai",
|
|
88
|
+
models: ["gpt-6.1-sol"],
|
|
89
|
+
options: ["reasoningEffort"]
|
|
90
|
+
},
|
|
91
|
+
generator_reading: {
|
|
92
|
+
name: "Generator face reading",
|
|
93
|
+
source: "openai",
|
|
94
|
+
models: ["gpt-6.1-sol"],
|
|
95
|
+
options: ["reasoningEffort"]
|
|
96
|
+
},
|
|
97
|
+
// Priced by OPENAI_PRICES of the generator (flare and sunburst alike).
|
|
98
|
+
generator_image: {
|
|
99
|
+
name: "Generator images",
|
|
100
|
+
source: "openai",
|
|
101
|
+
models: GPT_IMAGE_MODELS,
|
|
102
|
+
options: []
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
var AI_ROLE_NAMES = Object.keys(AI_ROLES);
|
|
106
|
+
|
|
107
|
+
// ../ts-shared/src/speech-input.ts
|
|
108
|
+
var INPUT_MODES = ["options", "text", "voice"];
|
|
109
|
+
var InputModeError = class extends Error {
|
|
110
|
+
constructor(message) {
|
|
111
|
+
super(message);
|
|
112
|
+
this.name = "NpcifyInputModeError";
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
var permittedInputModes = function(permissions) {
|
|
116
|
+
if (permissions.baked || !permissions.allowCustomReply) {
|
|
117
|
+
return ["options"];
|
|
118
|
+
}
|
|
119
|
+
return ["options", "text", "voice"];
|
|
120
|
+
};
|
|
121
|
+
var isInputMode = function(value) {
|
|
122
|
+
return INPUT_MODES.includes(value);
|
|
123
|
+
};
|
|
124
|
+
var parseInputModes = function(value) {
|
|
125
|
+
if (value === void 0 || value === null) {
|
|
126
|
+
return void 0;
|
|
127
|
+
}
|
|
128
|
+
if (!Array.isArray(value)) {
|
|
129
|
+
throw new InputModeError(
|
|
130
|
+
'inputModes must be an array of "options", "text" and "voice".'
|
|
131
|
+
);
|
|
132
|
+
}
|
|
133
|
+
if (value.length === 0) {
|
|
134
|
+
throw new InputModeError("inputModes must have at least one mode.");
|
|
135
|
+
}
|
|
136
|
+
const modes = [];
|
|
137
|
+
for (const mode of value) {
|
|
138
|
+
if (!isInputMode(mode)) {
|
|
139
|
+
throw new InputModeError(
|
|
140
|
+
`inputModes has an unknown mode ${JSON.stringify(mode)}. Use "options", "text" or "voice".`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
if (modes.includes(mode)) {
|
|
144
|
+
throw new InputModeError(`inputModes has "${mode}" two times.`);
|
|
145
|
+
}
|
|
146
|
+
modes.push(mode);
|
|
147
|
+
}
|
|
148
|
+
return modes;
|
|
149
|
+
};
|
|
150
|
+
var resolveInputModes = function(configured, permissions) {
|
|
151
|
+
const permitted = permittedInputModes(permissions);
|
|
152
|
+
if (!configured) {
|
|
153
|
+
return permitted;
|
|
154
|
+
}
|
|
155
|
+
if (permissions.baked && !(configured.length === 1 && configured[0] === "options")) {
|
|
156
|
+
throw new InputModeError(
|
|
157
|
+
'A baked (offline) conversation accepts only reply options. Set inputModes to ["options"] or remove it.'
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
const refused = configured.filter((mode) => !permitted.includes(mode));
|
|
161
|
+
if (refused.length > 0) {
|
|
162
|
+
throw new InputModeError(
|
|
163
|
+
`This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
|
|
164
|
+
);
|
|
165
|
+
}
|
|
166
|
+
return [...configured];
|
|
167
|
+
};
|
|
168
|
+
var SPEECH_AUDIO_FORMAT = Object.freeze({
|
|
169
|
+
encoding: "pcm_s16le",
|
|
170
|
+
sampleRate: 16e3,
|
|
171
|
+
channels: 1
|
|
172
|
+
});
|
|
173
|
+
var SPEECH_BYTES_PER_SECOND = 32e3;
|
|
174
|
+
var SPEECH_MAX_DURATION_MS = 3e4;
|
|
175
|
+
var SPEECH_MAX_AUDIO_BYTES = SPEECH_MAX_DURATION_MS / 1e3 * SPEECH_BYTES_PER_SECOND;
|
|
176
|
+
var SPEECH_MAX_UPLOAD_BYTES = SPEECH_MAX_AUDIO_BYTES + 4096;
|
|
177
|
+
var SPEECH_FRAME_MS = 100;
|
|
178
|
+
var SPEECH_MIN_HOLD_MS = 300;
|
|
179
|
+
var SPEECH_EVENTS = Object.freeze({
|
|
180
|
+
start: "speech:start",
|
|
181
|
+
audio: "speech:audio",
|
|
182
|
+
stop: "speech:stop",
|
|
183
|
+
cancel: "speech:cancel",
|
|
184
|
+
partial: "speech:partial",
|
|
185
|
+
turn: "speech:turn",
|
|
186
|
+
turnFailed: "speech:turn_failed"
|
|
187
|
+
});
|
|
188
|
+
|
|
12
189
|
// ../ts-shared/src/product-usage.ts
|
|
13
190
|
var productUsageEvents = [
|
|
14
191
|
"page_viewed",
|
|
@@ -17,7 +194,11 @@ var productUsageEvents = [
|
|
|
17
194
|
"checkout_submitted",
|
|
18
195
|
"checkout_failed",
|
|
19
196
|
"checkout_closed",
|
|
20
|
-
"paywall_shown"
|
|
197
|
+
"paywall_shown",
|
|
198
|
+
"playback_blocked",
|
|
199
|
+
"playback_start_pressed",
|
|
200
|
+
"reply_options_shown",
|
|
201
|
+
"page_left"
|
|
21
202
|
];
|
|
22
203
|
|
|
23
204
|
// ../ts-shared/src/analytics.ts
|
|
@@ -3122,8 +3303,14 @@ var LiveConversation = class {
|
|
|
3122
3303
|
this.dependencies = dependencies;
|
|
3123
3304
|
}
|
|
3124
3305
|
sending = false;
|
|
3306
|
+
/** A typed turn is in flight. */
|
|
3307
|
+
get inFlight() {
|
|
3308
|
+
return this.sending;
|
|
3309
|
+
}
|
|
3125
3310
|
pending = null;
|
|
3311
|
+
revision = 0;
|
|
3126
3312
|
reset() {
|
|
3313
|
+
this.revision += 1;
|
|
3127
3314
|
this.pending = null;
|
|
3128
3315
|
}
|
|
3129
3316
|
received(payload) {
|
|
@@ -3136,6 +3323,7 @@ var LiveConversation = class {
|
|
|
3136
3323
|
throw new Error("A reply is already being generated. Please wait.");
|
|
3137
3324
|
}
|
|
3138
3325
|
this.sending = true;
|
|
3326
|
+
const revision = this.revision;
|
|
3139
3327
|
const { telemetry, socket } = this.dependencies;
|
|
3140
3328
|
try {
|
|
3141
3329
|
telemetry.begin(conversationId);
|
|
@@ -3143,10 +3331,10 @@ var LiveConversation = class {
|
|
|
3143
3331
|
const silent = options?.silent === true;
|
|
3144
3332
|
const parentStagingId = options?.parentStagingId;
|
|
3145
3333
|
const pending = this.pending;
|
|
3146
|
-
const request = pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
|
|
3334
|
+
const request = !options?.requestId && pending?.conversationId === conversationId && pending.message === message && pending.silent === silent && pending.parentStagingId === parentStagingId ? pending : {
|
|
3147
3335
|
conversationId,
|
|
3148
3336
|
message,
|
|
3149
|
-
requestId: globalThis.crypto.randomUUID(),
|
|
3337
|
+
requestId: options?.requestId ?? globalThis.crypto.randomUUID(),
|
|
3150
3338
|
silent,
|
|
3151
3339
|
parentStagingId
|
|
3152
3340
|
};
|
|
@@ -3180,6 +3368,9 @@ var LiveConversation = class {
|
|
|
3180
3368
|
if (this.pending?.requestId === request.requestId) {
|
|
3181
3369
|
this.pending = null;
|
|
3182
3370
|
}
|
|
3371
|
+
if (revision !== this.revision) {
|
|
3372
|
+
return payload;
|
|
3373
|
+
}
|
|
3183
3374
|
await this.dependencies.handleResult(
|
|
3184
3375
|
payload,
|
|
3185
3376
|
payload.taskId !== request.requestId,
|
|
@@ -4310,7 +4501,7 @@ var disposeSceneContext = function(ctx) {
|
|
|
4310
4501
|
|
|
4311
4502
|
// ../vrm-renderer/src/preview-controller.ts
|
|
4312
4503
|
import { VRMUtils as VRMUtils4 } from "@pixiv/three-vrm";
|
|
4313
|
-
import { Box3, GridHelper, Vector3 as
|
|
4504
|
+
import { Box3, GridHelper, Vector3 as Vector34 } from "three";
|
|
4314
4505
|
|
|
4315
4506
|
// ../vrm-renderer/src/loaders.ts
|
|
4316
4507
|
import { VRMLoaderPlugin as VRMLoaderPlugin2, VRMUtils as VRMUtils2 } from "@pixiv/three-vrm";
|
|
@@ -4332,7 +4523,33 @@ import { AnimationMixer, LoopOnce, LoopRepeat } from "three";
|
|
|
4332
4523
|
import {
|
|
4333
4524
|
createVRMAnimationClip as createPixivVRMAnimationClip
|
|
4334
4525
|
} from "@pixiv/three-vrm-animation";
|
|
4526
|
+
import {
|
|
4527
|
+
AnimationClip,
|
|
4528
|
+
Euler,
|
|
4529
|
+
Quaternion as Quaternion2,
|
|
4530
|
+
QuaternionKeyframeTrack,
|
|
4531
|
+
QuaternionLinearInterpolant,
|
|
4532
|
+
Vector3 as Vector33,
|
|
4533
|
+
VectorKeyframeTrack
|
|
4534
|
+
} from "three";
|
|
4335
4535
|
var NPCIFY_VRM_NORMALIZATION_ROOT_NAME = "NPCifyNormalizationRoot";
|
|
4536
|
+
var TORSO_BONES = [
|
|
4537
|
+
"hips",
|
|
4538
|
+
"spine",
|
|
4539
|
+
"chest",
|
|
4540
|
+
"upperChest"
|
|
4541
|
+
];
|
|
4542
|
+
var HEAD_BONES = ["neck", "head"];
|
|
4543
|
+
var AXIAL_BONES = [...TORSO_BONES, ...HEAD_BONES];
|
|
4544
|
+
var PARENT_OF_HEAD_BONES = [
|
|
4545
|
+
...TORSO_BONES,
|
|
4546
|
+
"neck"
|
|
4547
|
+
];
|
|
4548
|
+
var IDENTITY_QUATERNION = new Quaternion2();
|
|
4549
|
+
var UP_AXIS = new Vector33(0, 1, 0);
|
|
4550
|
+
var MAX_HEAD_PITCH_CORRECTION_RAD = 35 * Math.PI / 180;
|
|
4551
|
+
var NECK_CORRECTION_SHARE = 0.35;
|
|
4552
|
+
var EPSILON = 1e-5;
|
|
4336
4553
|
var canonicalAnimationTargets = /* @__PURE__ */ new WeakMap();
|
|
4337
4554
|
var isNpcifyNormalizedVrm = function(vrm) {
|
|
4338
4555
|
return vrm.scene.children.some(
|
|
@@ -4358,8 +4575,300 @@ var canonicalAnimationTarget = function(vrm) {
|
|
|
4358
4575
|
canonicalAnimationTargets.set(vrm, target);
|
|
4359
4576
|
return target;
|
|
4360
4577
|
};
|
|
4361
|
-
var
|
|
4362
|
-
|
|
4578
|
+
var resolveAxialBoneNodeNames = function(vrm) {
|
|
4579
|
+
const result = /* @__PURE__ */ new Map();
|
|
4580
|
+
const seenNodeNames = /* @__PURE__ */ new Set();
|
|
4581
|
+
for (const bone of AXIAL_BONES) {
|
|
4582
|
+
const nodeName = vrm.humanoid?.getNormalizedBoneNode?.(bone)?.name;
|
|
4583
|
+
if (nodeName && !seenNodeNames.has(nodeName)) {
|
|
4584
|
+
seenNodeNames.add(nodeName);
|
|
4585
|
+
result.set(bone, nodeName);
|
|
4586
|
+
}
|
|
4587
|
+
}
|
|
4588
|
+
return result;
|
|
4589
|
+
};
|
|
4590
|
+
var findQuaternionTrack = function(clip, nodeName) {
|
|
4591
|
+
if (!nodeName) {
|
|
4592
|
+
return void 0;
|
|
4593
|
+
}
|
|
4594
|
+
const trackName = `${nodeName}.quaternion`;
|
|
4595
|
+
return clip.tracks.find(
|
|
4596
|
+
(track) => track.name === trackName && track.times.length > 0 && track.values.length >= track.times.length * 4
|
|
4597
|
+
);
|
|
4598
|
+
};
|
|
4599
|
+
var sampleBoneQuaternion = function(interpolant, time, target) {
|
|
4600
|
+
if (!interpolant) {
|
|
4601
|
+
return target.identity();
|
|
4602
|
+
}
|
|
4603
|
+
const values = interpolant.evaluate(time);
|
|
4604
|
+
return target.set(values[0] ?? 0, values[1] ?? 0, values[2] ?? 0, values[3] ?? 1).normalize();
|
|
4605
|
+
};
|
|
4606
|
+
var evaluateChainOrientation = function(bones, interpolants, time, rootPreRotation, target, scratch) {
|
|
4607
|
+
target.copy(rootPreRotation);
|
|
4608
|
+
for (const bone of bones) {
|
|
4609
|
+
const interpolant = interpolants.get(bone);
|
|
4610
|
+
if (interpolant) {
|
|
4611
|
+
sampleBoneQuaternion(interpolant, time, scratch);
|
|
4612
|
+
target.multiply(scratch);
|
|
4613
|
+
}
|
|
4614
|
+
}
|
|
4615
|
+
return target;
|
|
4616
|
+
};
|
|
4617
|
+
var rotateQuaternionTrackInParentSpace = function(track, parentBones, interpolants, rootPreRotation, worldCorrection) {
|
|
4618
|
+
const values = new Float32Array(track.values.length);
|
|
4619
|
+
const parentQuat = new Quaternion2();
|
|
4620
|
+
const parentInvQuat = new Quaternion2();
|
|
4621
|
+
const localCorrection = new Quaternion2();
|
|
4622
|
+
const keyframeQuat = new Quaternion2();
|
|
4623
|
+
const prevOrigQuat = new Quaternion2();
|
|
4624
|
+
const prevNewQuat = new Quaternion2();
|
|
4625
|
+
const scratchQuat = new Quaternion2();
|
|
4626
|
+
const count = track.times.length;
|
|
4627
|
+
for (let i = 0; i < count; i += 1) {
|
|
4628
|
+
const time = track.times[i] ?? 0;
|
|
4629
|
+
const offset = i * 4;
|
|
4630
|
+
keyframeQuat.set(
|
|
4631
|
+
track.values[offset] ?? 0,
|
|
4632
|
+
track.values[offset + 1] ?? 0,
|
|
4633
|
+
track.values[offset + 2] ?? 0,
|
|
4634
|
+
track.values[offset + 3] ?? 1
|
|
4635
|
+
).normalize();
|
|
4636
|
+
const origDot = i > 0 ? prevOrigQuat.dot(keyframeQuat) : 1;
|
|
4637
|
+
prevOrigQuat.copy(keyframeQuat);
|
|
4638
|
+
if (parentBones.length === 0) {
|
|
4639
|
+
keyframeQuat.premultiply(worldCorrection).normalize();
|
|
4640
|
+
} else {
|
|
4641
|
+
evaluateChainOrientation(
|
|
4642
|
+
parentBones,
|
|
4643
|
+
interpolants,
|
|
4644
|
+
time,
|
|
4645
|
+
rootPreRotation,
|
|
4646
|
+
parentQuat,
|
|
4647
|
+
scratchQuat
|
|
4648
|
+
);
|
|
4649
|
+
parentInvQuat.copy(parentQuat).invert();
|
|
4650
|
+
localCorrection.copy(parentInvQuat).multiply(worldCorrection).multiply(parentQuat);
|
|
4651
|
+
keyframeQuat.premultiply(localCorrection).normalize();
|
|
4652
|
+
}
|
|
4653
|
+
if (i > 0 && origDot >= 0 && prevNewQuat.dot(keyframeQuat) < 0) {
|
|
4654
|
+
keyframeQuat.set(
|
|
4655
|
+
-keyframeQuat.x,
|
|
4656
|
+
-keyframeQuat.y,
|
|
4657
|
+
-keyframeQuat.z,
|
|
4658
|
+
-keyframeQuat.w
|
|
4659
|
+
);
|
|
4660
|
+
}
|
|
4661
|
+
prevNewQuat.copy(keyframeQuat);
|
|
4662
|
+
values[offset] = keyframeQuat.x;
|
|
4663
|
+
values[offset + 1] = keyframeQuat.y;
|
|
4664
|
+
values[offset + 2] = keyframeQuat.z;
|
|
4665
|
+
values[offset + 3] = keyframeQuat.w;
|
|
4666
|
+
}
|
|
4667
|
+
return new QuaternionKeyframeTrack(
|
|
4668
|
+
track.name,
|
|
4669
|
+
Array.from(track.times),
|
|
4670
|
+
Array.from(values)
|
|
4671
|
+
);
|
|
4672
|
+
};
|
|
4673
|
+
var rotateHipsPositionTrack = function(track, yawRadians) {
|
|
4674
|
+
const count = Math.floor(track.values.length / 3);
|
|
4675
|
+
if (count === 0) {
|
|
4676
|
+
return new VectorKeyframeTrack(
|
|
4677
|
+
track.name,
|
|
4678
|
+
Array.from(track.times),
|
|
4679
|
+
Array.from(track.values)
|
|
4680
|
+
);
|
|
4681
|
+
}
|
|
4682
|
+
let meanX = 0;
|
|
4683
|
+
let meanZ = 0;
|
|
4684
|
+
for (let i = 0; i < count; i += 1) {
|
|
4685
|
+
meanX += track.values[i * 3] ?? 0;
|
|
4686
|
+
meanZ += track.values[i * 3 + 2] ?? 0;
|
|
4687
|
+
}
|
|
4688
|
+
meanX /= count;
|
|
4689
|
+
meanZ /= count;
|
|
4690
|
+
const cosYaw = Math.cos(yawRadians);
|
|
4691
|
+
const sinYaw = Math.sin(yawRadians);
|
|
4692
|
+
const values = new Float32Array(track.values.length);
|
|
4693
|
+
for (let i = 0; i < count; i += 1) {
|
|
4694
|
+
const offset = i * 3;
|
|
4695
|
+
const dx = (track.values[offset] ?? 0) - meanX;
|
|
4696
|
+
const y = track.values[offset + 1] ?? 0;
|
|
4697
|
+
const dz = (track.values[offset + 2] ?? 0) - meanZ;
|
|
4698
|
+
values[offset] = meanX + dx * cosYaw + dz * sinYaw;
|
|
4699
|
+
values[offset + 1] = y;
|
|
4700
|
+
values[offset + 2] = meanZ - dx * sinYaw + dz * cosYaw;
|
|
4701
|
+
}
|
|
4702
|
+
return new VectorKeyframeTrack(
|
|
4703
|
+
track.name,
|
|
4704
|
+
Array.from(track.times),
|
|
4705
|
+
Array.from(values)
|
|
4706
|
+
);
|
|
4707
|
+
};
|
|
4708
|
+
var normalizeIdleAnimationClip = function(clip, vrm) {
|
|
4709
|
+
const boneNodeNames = resolveAxialBoneNodeNames(vrm);
|
|
4710
|
+
if (boneNodeNames.size === 0) {
|
|
4711
|
+
return clip;
|
|
4712
|
+
}
|
|
4713
|
+
const interpolants = /* @__PURE__ */ new Map();
|
|
4714
|
+
for (const bone of AXIAL_BONES) {
|
|
4715
|
+
const track = findQuaternionTrack(clip, boneNodeNames.get(bone));
|
|
4716
|
+
if (track) {
|
|
4717
|
+
interpolants.set(
|
|
4718
|
+
bone,
|
|
4719
|
+
new QuaternionLinearInterpolant(track.times, track.values, 4)
|
|
4720
|
+
);
|
|
4721
|
+
}
|
|
4722
|
+
}
|
|
4723
|
+
if (interpolants.size === 0) {
|
|
4724
|
+
return clip;
|
|
4725
|
+
}
|
|
4726
|
+
const duration = clip.duration > 0 ? clip.duration : 0;
|
|
4727
|
+
const sampleCount = duration > 0 ? Math.max(16, Math.min(120, Math.ceil(duration * 30))) : 1;
|
|
4728
|
+
const sampleTimes = [];
|
|
4729
|
+
for (let i = 0; i < sampleCount; i += 1) {
|
|
4730
|
+
sampleTimes.push(duration > 0 ? (i + 0.5) / sampleCount * duration : 0);
|
|
4731
|
+
}
|
|
4732
|
+
const chainQuat = new Quaternion2();
|
|
4733
|
+
const scratchQuat = new Quaternion2();
|
|
4734
|
+
const forwardVec = new Vector33();
|
|
4735
|
+
const rootTorsoBone = TORSO_BONES.find((bone) => interpolants.has(bone));
|
|
4736
|
+
const rootPreRotation = new Quaternion2();
|
|
4737
|
+
let torsoYawCorrection = 0;
|
|
4738
|
+
if (rootTorsoBone) {
|
|
4739
|
+
let sumX = 0;
|
|
4740
|
+
let sumZ = 0;
|
|
4741
|
+
for (const time of sampleTimes) {
|
|
4742
|
+
evaluateChainOrientation(
|
|
4743
|
+
TORSO_BONES,
|
|
4744
|
+
interpolants,
|
|
4745
|
+
time,
|
|
4746
|
+
IDENTITY_QUATERNION,
|
|
4747
|
+
chainQuat,
|
|
4748
|
+
scratchQuat
|
|
4749
|
+
);
|
|
4750
|
+
forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
|
|
4751
|
+
const horizLen = Math.hypot(forwardVec.x, forwardVec.z);
|
|
4752
|
+
if (horizLen > EPSILON) {
|
|
4753
|
+
sumX += forwardVec.x / horizLen;
|
|
4754
|
+
sumZ += forwardVec.z / horizLen;
|
|
4755
|
+
}
|
|
4756
|
+
}
|
|
4757
|
+
if (Math.hypot(sumX, sumZ) > EPSILON) {
|
|
4758
|
+
const meanTorsoYaw = Math.atan2(sumX, sumZ);
|
|
4759
|
+
if (Math.abs(meanTorsoYaw) > EPSILON) {
|
|
4760
|
+
torsoYawCorrection = -meanTorsoYaw;
|
|
4761
|
+
rootPreRotation.setFromAxisAngle(UP_AXIS, torsoYawCorrection);
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4764
|
+
}
|
|
4765
|
+
const hasNeckTrack = interpolants.has("neck");
|
|
4766
|
+
const hasHeadTrack = interpolants.has("head");
|
|
4767
|
+
let neckWorldCorrection = null;
|
|
4768
|
+
let headWorldCorrection = null;
|
|
4769
|
+
if (hasNeckTrack || hasHeadTrack) {
|
|
4770
|
+
const meanHeadForward = new Vector33();
|
|
4771
|
+
for (const time of sampleTimes) {
|
|
4772
|
+
evaluateChainOrientation(
|
|
4773
|
+
AXIAL_BONES,
|
|
4774
|
+
interpolants,
|
|
4775
|
+
time,
|
|
4776
|
+
rootPreRotation,
|
|
4777
|
+
chainQuat,
|
|
4778
|
+
scratchQuat
|
|
4779
|
+
);
|
|
4780
|
+
forwardVec.set(0, 0, 1).applyQuaternion(chainQuat);
|
|
4781
|
+
meanHeadForward.add(forwardVec);
|
|
4782
|
+
}
|
|
4783
|
+
if (meanHeadForward.lengthSq() > EPSILON * EPSILON) {
|
|
4784
|
+
meanHeadForward.normalize();
|
|
4785
|
+
const meanHeadYaw = Math.atan2(meanHeadForward.x, meanHeadForward.z);
|
|
4786
|
+
const rawHeadPitch = Math.atan2(
|
|
4787
|
+
-meanHeadForward.y,
|
|
4788
|
+
Math.hypot(meanHeadForward.x, meanHeadForward.z)
|
|
4789
|
+
);
|
|
4790
|
+
const clampedHeadPitch = Math.max(
|
|
4791
|
+
-MAX_HEAD_PITCH_CORRECTION_RAD,
|
|
4792
|
+
Math.min(MAX_HEAD_PITCH_CORRECTION_RAD, rawHeadPitch)
|
|
4793
|
+
);
|
|
4794
|
+
if (Math.abs(meanHeadYaw) > EPSILON || Math.abs(clampedHeadPitch) > EPSILON) {
|
|
4795
|
+
const totalWorldCorrection = new Quaternion2().setFromEuler(
|
|
4796
|
+
new Euler(-clampedHeadPitch, -meanHeadYaw, 0, "XYZ")
|
|
4797
|
+
);
|
|
4798
|
+
if (hasNeckTrack && hasHeadTrack) {
|
|
4799
|
+
neckWorldCorrection = new Quaternion2().setFromEuler(
|
|
4800
|
+
new Euler(
|
|
4801
|
+
-clampedHeadPitch * NECK_CORRECTION_SHARE,
|
|
4802
|
+
-meanHeadYaw * NECK_CORRECTION_SHARE,
|
|
4803
|
+
0,
|
|
4804
|
+
"XYZ"
|
|
4805
|
+
)
|
|
4806
|
+
);
|
|
4807
|
+
headWorldCorrection = neckWorldCorrection.clone().invert().multiply(totalWorldCorrection);
|
|
4808
|
+
} else if (hasNeckTrack) {
|
|
4809
|
+
neckWorldCorrection = totalWorldCorrection;
|
|
4810
|
+
} else {
|
|
4811
|
+
headWorldCorrection = totalWorldCorrection;
|
|
4812
|
+
}
|
|
4813
|
+
}
|
|
4814
|
+
}
|
|
4815
|
+
}
|
|
4816
|
+
const hasTorsoCorrection = Math.abs(torsoYawCorrection) > EPSILON;
|
|
4817
|
+
if (!hasTorsoCorrection && !neckWorldCorrection && !headWorldCorrection) {
|
|
4818
|
+
return clip;
|
|
4819
|
+
}
|
|
4820
|
+
const rootTorsoNodeName = rootTorsoBone ? boneNodeNames.get(rootTorsoBone) : void 0;
|
|
4821
|
+
const rootTorsoQuatTrackName = rootTorsoNodeName ? `${rootTorsoNodeName}.quaternion` : null;
|
|
4822
|
+
const hipsNodeName = boneNodeNames.get("hips");
|
|
4823
|
+
const hipsPosTrackName = hipsNodeName && rootTorsoBone === "hips" ? `${hipsNodeName}.position` : null;
|
|
4824
|
+
const neckNodeName = boneNodeNames.get("neck");
|
|
4825
|
+
const neckQuatTrackName = neckNodeName ? `${neckNodeName}.quaternion` : null;
|
|
4826
|
+
const headNodeName = boneNodeNames.get("head");
|
|
4827
|
+
const headQuatTrackName = headNodeName ? `${headNodeName}.quaternion` : null;
|
|
4828
|
+
const nextTracks = clip.tracks.map((track) => {
|
|
4829
|
+
if (hasTorsoCorrection && track.name === rootTorsoQuatTrackName) {
|
|
4830
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4831
|
+
track,
|
|
4832
|
+
[],
|
|
4833
|
+
interpolants,
|
|
4834
|
+
IDENTITY_QUATERNION,
|
|
4835
|
+
rootPreRotation
|
|
4836
|
+
);
|
|
4837
|
+
}
|
|
4838
|
+
if (hasTorsoCorrection && track.name === hipsPosTrackName) {
|
|
4839
|
+
return rotateHipsPositionTrack(track, torsoYawCorrection);
|
|
4840
|
+
}
|
|
4841
|
+
if (neckWorldCorrection && track.name === neckQuatTrackName) {
|
|
4842
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4843
|
+
track,
|
|
4844
|
+
TORSO_BONES,
|
|
4845
|
+
interpolants,
|
|
4846
|
+
rootPreRotation,
|
|
4847
|
+
neckWorldCorrection
|
|
4848
|
+
);
|
|
4849
|
+
}
|
|
4850
|
+
if (headWorldCorrection && track.name === headQuatTrackName) {
|
|
4851
|
+
return rotateQuaternionTrackInParentSpace(
|
|
4852
|
+
track,
|
|
4853
|
+
PARENT_OF_HEAD_BONES,
|
|
4854
|
+
interpolants,
|
|
4855
|
+
rootPreRotation,
|
|
4856
|
+
headWorldCorrection
|
|
4857
|
+
);
|
|
4858
|
+
}
|
|
4859
|
+
return track;
|
|
4860
|
+
});
|
|
4861
|
+
return new AnimationClip(clip.name, clip.duration, nextTracks, clip.blendMode);
|
|
4862
|
+
};
|
|
4863
|
+
var createVRMAnimationClip = function(animation, vrm, options) {
|
|
4864
|
+
const clip = createPixivVRMAnimationClip(
|
|
4865
|
+
animation,
|
|
4866
|
+
canonicalAnimationTarget(vrm)
|
|
4867
|
+
);
|
|
4868
|
+
if (options?.normalizeIdleOrientation) {
|
|
4869
|
+
return normalizeIdleAnimationClip(clip, vrm);
|
|
4870
|
+
}
|
|
4871
|
+
return clip;
|
|
4363
4872
|
};
|
|
4364
4873
|
|
|
4365
4874
|
// ../vrm-renderer/src/session.ts
|
|
@@ -4696,16 +5205,20 @@ var RemoteAnimationController = class {
|
|
|
4696
5205
|
}
|
|
4697
5206
|
};
|
|
4698
5207
|
var AnimationController = class {
|
|
4699
|
-
constructor(vrm, idleAnimationId) {
|
|
5208
|
+
constructor(vrm, idleAnimationId, options) {
|
|
4700
5209
|
this.vrm = vrm;
|
|
4701
5210
|
this.mixer = new AnimationMixer2(this.vrm.scene);
|
|
4702
5211
|
this.idleAnimationId = idleAnimationId ?? null;
|
|
5212
|
+
this.normalizeIdleOrientation = options?.normalizeIdleOrientation ?? false;
|
|
4703
5213
|
this.mixer.addEventListener("finished", this.handleActionFinished);
|
|
4704
5214
|
}
|
|
4705
5215
|
prepareAnimations;
|
|
4706
5216
|
mixer;
|
|
4707
5217
|
clips = /* @__PURE__ */ new Map();
|
|
5218
|
+
rawClips = /* @__PURE__ */ new Map();
|
|
5219
|
+
normalizedIdleClips = /* @__PURE__ */ new Map();
|
|
4708
5220
|
configs = /* @__PURE__ */ new Map();
|
|
5221
|
+
normalizeIdleOrientation;
|
|
4709
5222
|
previewMixer = null;
|
|
4710
5223
|
restorePreview = null;
|
|
4711
5224
|
activeAction = null;
|
|
@@ -4731,8 +5244,10 @@ var AnimationController = class {
|
|
|
4731
5244
|
return this.clips.has(id);
|
|
4732
5245
|
}
|
|
4733
5246
|
addAnimation(animationId, animation, config) {
|
|
4734
|
-
const
|
|
4735
|
-
this.
|
|
5247
|
+
const rawClip = createVRMAnimationClip(animation, this.vrm);
|
|
5248
|
+
this.rawClips.set(animationId, rawClip);
|
|
5249
|
+
this.normalizedIdleClips.delete(animationId);
|
|
5250
|
+
this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
|
|
4736
5251
|
this.configs.set(animationId, {
|
|
4737
5252
|
loop: config?.loop ?? defaultAnimationConfig.loop,
|
|
4738
5253
|
weight: config?.weight ?? defaultAnimationConfig.weight
|
|
@@ -4762,7 +5277,20 @@ var AnimationController = class {
|
|
|
4762
5277
|
return restore;
|
|
4763
5278
|
}
|
|
4764
5279
|
setIdleAnimation(animationId) {
|
|
5280
|
+
const previousIdleId = this.idleAnimationId;
|
|
4765
5281
|
this.idleAnimationId = animationId;
|
|
5282
|
+
if (previousIdleId && previousIdleId !== animationId) {
|
|
5283
|
+
const previousRaw = this.rawClips.get(previousIdleId);
|
|
5284
|
+
if (previousRaw) {
|
|
5285
|
+
this.clips.set(previousIdleId, previousRaw);
|
|
5286
|
+
}
|
|
5287
|
+
}
|
|
5288
|
+
if (animationId) {
|
|
5289
|
+
const rawClip = this.rawClips.get(animationId);
|
|
5290
|
+
if (rawClip) {
|
|
5291
|
+
this.clips.set(animationId, this.resolveClipForId(animationId, rawClip));
|
|
5292
|
+
}
|
|
5293
|
+
}
|
|
4766
5294
|
this.playIdle();
|
|
4767
5295
|
}
|
|
4768
5296
|
clearAnimations() {
|
|
@@ -4770,6 +5298,8 @@ var AnimationController = class {
|
|
|
4770
5298
|
this.mixer.stopAllAction();
|
|
4771
5299
|
this.mixer.uncacheRoot(this.vrm.scene);
|
|
4772
5300
|
this.clips.clear();
|
|
5301
|
+
this.rawClips.clear();
|
|
5302
|
+
this.normalizedIdleClips.clear();
|
|
4773
5303
|
this.configs.clear();
|
|
4774
5304
|
this.activeAction = null;
|
|
4775
5305
|
this.activeAnimationId = null;
|
|
@@ -4777,6 +5307,18 @@ var AnimationController = class {
|
|
|
4777
5307
|
this.idleAnimationId = null;
|
|
4778
5308
|
this.vrm.humanoid.resetNormalizedPose();
|
|
4779
5309
|
}
|
|
5310
|
+
resolveClipForId(animationId, rawClip) {
|
|
5311
|
+
if (!this.normalizeIdleOrientation || animationId !== this.idleAnimationId) {
|
|
5312
|
+
return rawClip;
|
|
5313
|
+
}
|
|
5314
|
+
const cached = this.normalizedIdleClips.get(animationId);
|
|
5315
|
+
if (cached) {
|
|
5316
|
+
return cached;
|
|
5317
|
+
}
|
|
5318
|
+
const normalized = normalizeIdleAnimationClip(rawClip, this.vrm);
|
|
5319
|
+
this.normalizedIdleClips.set(animationId, normalized);
|
|
5320
|
+
return normalized;
|
|
5321
|
+
}
|
|
4780
5322
|
playAnimation(animationId, fadeDuration = 0.25, loopOverride) {
|
|
4781
5323
|
this.restorePreview?.();
|
|
4782
5324
|
const clip = this.clips.get(animationId);
|
|
@@ -5545,6 +6087,7 @@ var StreamPlaybackController = class {
|
|
|
5545
6087
|
force: options?.force === true
|
|
5546
6088
|
});
|
|
5547
6089
|
}
|
|
6090
|
+
/** Returns the playback result for a timeline event. */
|
|
5548
6091
|
handleEvent(event) {
|
|
5549
6092
|
if (this.streamEventCallback) {
|
|
5550
6093
|
this.safeInvoke("onStreamEvent", () => this.streamEventCallback?.(event));
|
|
@@ -5588,14 +6131,13 @@ var StreamPlaybackController = class {
|
|
|
5588
6131
|
if (event.type === "staging" || event.type === "staging_prefix" || event.type === "staging_live") {
|
|
5589
6132
|
this.recordStreamEvent(event, "timeline");
|
|
5590
6133
|
this.dependencies.telemetry.recordFirstStaging();
|
|
5591
|
-
|
|
6134
|
+
return this.playTimeline({
|
|
5592
6135
|
taskId: event.taskId,
|
|
5593
6136
|
stagingId: event.stagingId,
|
|
5594
6137
|
timeline: event.timeline,
|
|
5595
6138
|
complete: event.complete,
|
|
5596
6139
|
revision: event.revision
|
|
5597
6140
|
});
|
|
5598
|
-
return;
|
|
5599
6141
|
}
|
|
5600
6142
|
if (event.type === "options") {
|
|
5601
6143
|
this.recordStreamEvent(event, "options");
|
|
@@ -5954,8 +6496,18 @@ var decodeBase64 = function(value) {
|
|
|
5954
6496
|
}
|
|
5955
6497
|
return bytes;
|
|
5956
6498
|
};
|
|
5957
|
-
var
|
|
5958
|
-
return
|
|
6499
|
+
var managedMediaSource = function() {
|
|
6500
|
+
return globalThis.ManagedMediaSource;
|
|
6501
|
+
};
|
|
6502
|
+
var needsManagedMediaSource = function() {
|
|
6503
|
+
return typeof MediaSource === "undefined" && Boolean(managedMediaSource());
|
|
6504
|
+
};
|
|
6505
|
+
var streamingMediaSource = function(mimeType) {
|
|
6506
|
+
if (typeof URL?.createObjectURL !== "function") {
|
|
6507
|
+
return null;
|
|
6508
|
+
}
|
|
6509
|
+
const source = typeof MediaSource === "undefined" ? managedMediaSource() : MediaSource;
|
|
6510
|
+
return source?.isTypeSupported(mimeType) ? source : null;
|
|
5959
6511
|
};
|
|
5960
6512
|
var StreamingAudioBuffer = class {
|
|
5961
6513
|
constructor(mimeType) {
|
|
@@ -5998,8 +6550,9 @@ var StreamingAudioBuffer = class {
|
|
|
5998
6550
|
if (this.objectUrl) {
|
|
5999
6551
|
return this.objectUrl;
|
|
6000
6552
|
}
|
|
6001
|
-
|
|
6002
|
-
|
|
6553
|
+
const Source2 = streamingMediaSource(this.mimeType);
|
|
6554
|
+
if (Source2) {
|
|
6555
|
+
this.mediaSource = new Source2();
|
|
6003
6556
|
this.objectUrl = URL.createObjectURL(this.mediaSource);
|
|
6004
6557
|
this.mediaSource.addEventListener("sourceopen", this.handleSourceOpen, {
|
|
6005
6558
|
once: true
|
|
@@ -6091,7 +6644,7 @@ var StreamingAudioBuffer = class {
|
|
|
6091
6644
|
}
|
|
6092
6645
|
};
|
|
6093
6646
|
finishFallbackIfReady() {
|
|
6094
|
-
if (!this.completed || this.objectUrl ||
|
|
6647
|
+
if (!this.completed || this.objectUrl || streamingMediaSource(this.mimeType) || this.failedError || this.disposed) {
|
|
6095
6648
|
return;
|
|
6096
6649
|
}
|
|
6097
6650
|
const blobParts = this.chunks.map((chunk) => Uint8Array.from(chunk).buffer);
|
|
@@ -6418,7 +6971,8 @@ var AudioPlayer = class {
|
|
|
6418
6971
|
this.visemePlayer.cleanup();
|
|
6419
6972
|
if (this.audioElement) {
|
|
6420
6973
|
this.audioElement.srcObject = null;
|
|
6421
|
-
this.audioElement.src
|
|
6974
|
+
this.audioElement.removeAttribute("src");
|
|
6975
|
+
this.audioElement.load();
|
|
6422
6976
|
}
|
|
6423
6977
|
this.streams.forEach((stream) => stream.dispose());
|
|
6424
6978
|
this.streams.clear();
|
|
@@ -6435,8 +6989,10 @@ var AudioPlayer = class {
|
|
|
6435
6989
|
this.streams.delete(streamId);
|
|
6436
6990
|
}
|
|
6437
6991
|
buildSilentWav = function() {
|
|
6438
|
-
const
|
|
6439
|
-
const
|
|
6992
|
+
const sampleRate = 44100;
|
|
6993
|
+
const bytesPerSample = 2;
|
|
6994
|
+
const dataSize = Math.round(sampleRate * 0.1) * bytesPerSample;
|
|
6995
|
+
const bytes = new Uint8Array(44 + dataSize);
|
|
6440
6996
|
const view = new DataView(bytes.buffer);
|
|
6441
6997
|
const writeText = function(offset, value) {
|
|
6442
6998
|
Array.from(value).forEach((character, index) => {
|
|
@@ -6444,19 +7000,18 @@ var AudioPlayer = class {
|
|
|
6444
7000
|
});
|
|
6445
7001
|
};
|
|
6446
7002
|
writeText(0, "RIFF");
|
|
6447
|
-
view.setUint32(4, 36 +
|
|
7003
|
+
view.setUint32(4, 36 + dataSize, true);
|
|
6448
7004
|
writeText(8, "WAVE");
|
|
6449
7005
|
writeText(12, "fmt ");
|
|
6450
7006
|
view.setUint32(16, 16, true);
|
|
6451
7007
|
view.setUint16(20, 1, true);
|
|
6452
7008
|
view.setUint16(22, 1, true);
|
|
6453
|
-
view.setUint32(24,
|
|
6454
|
-
view.setUint32(28,
|
|
6455
|
-
view.setUint16(32,
|
|
6456
|
-
view.setUint16(34, 8, true);
|
|
7009
|
+
view.setUint32(24, sampleRate, true);
|
|
7010
|
+
view.setUint32(28, sampleRate * bytesPerSample, true);
|
|
7011
|
+
view.setUint16(32, bytesPerSample, true);
|
|
7012
|
+
view.setUint16(34, bytesPerSample * 8, true);
|
|
6457
7013
|
writeText(36, "data");
|
|
6458
|
-
view.setUint32(40,
|
|
6459
|
-
bytes.fill(128, 44);
|
|
7014
|
+
view.setUint32(40, dataSize, true);
|
|
6460
7015
|
return new Blob([bytes], { type: "audio/wav" });
|
|
6461
7016
|
};
|
|
6462
7017
|
async resolveSourceUrl(source) {
|
|
@@ -6492,6 +7047,9 @@ var AudioPlayer = class {
|
|
|
6492
7047
|
element.volume = this.volumeValue;
|
|
6493
7048
|
element.muted = this.mutedValue;
|
|
6494
7049
|
element.playbackRate = this.playbackRateValue;
|
|
7050
|
+
if (needsManagedMediaSource()) {
|
|
7051
|
+
element.disableRemotePlayback = true;
|
|
7052
|
+
}
|
|
6495
7053
|
if (this.crossOriginValue !== null) {
|
|
6496
7054
|
element.crossOrigin = this.crossOriginValue;
|
|
6497
7055
|
}
|
|
@@ -6575,6 +7133,14 @@ var CharacterExpressions = class {
|
|
|
6575
7133
|
clearEmotion(mouthMode) {
|
|
6576
7134
|
this.mixer.clearEmotion(mouthMode);
|
|
6577
7135
|
}
|
|
7136
|
+
/** Set a temporary emotion overlay. Timeline emotions and visemes keep their state. */
|
|
7137
|
+
setOverlay(weights) {
|
|
7138
|
+
this.mixer.setOverlay(weights);
|
|
7139
|
+
}
|
|
7140
|
+
/** Ease the temporary overlay back to the current timeline or idle face. */
|
|
7141
|
+
clearOverlay() {
|
|
7142
|
+
this.mixer.clearOverlay();
|
|
7143
|
+
}
|
|
6578
7144
|
/** Reset all expression layer targets to zero. */
|
|
6579
7145
|
reset() {
|
|
6580
7146
|
this.mixer.reset();
|
|
@@ -6749,7 +7315,7 @@ var FacialPerformanceInspector = class {
|
|
|
6749
7315
|
};
|
|
6750
7316
|
|
|
6751
7317
|
// ../../clients/web-client/src/character/idle-face.ts
|
|
6752
|
-
import { Vector3 as
|
|
7318
|
+
import { Vector3 as Vector35 } from "three";
|
|
6753
7319
|
var IdleFace = class {
|
|
6754
7320
|
constructor(vrm, random) {
|
|
6755
7321
|
this.vrm = vrm;
|
|
@@ -6757,7 +7323,7 @@ var IdleFace = class {
|
|
|
6757
7323
|
}
|
|
6758
7324
|
time = 0;
|
|
6759
7325
|
timeline;
|
|
6760
|
-
targetPosition = new
|
|
7326
|
+
targetPosition = new Vector35();
|
|
6761
7327
|
/** Advances by `delta` seconds, turns the eyes, and returns the blink weight. */
|
|
6762
7328
|
update(delta, frame) {
|
|
6763
7329
|
if (Number.isFinite(delta) && delta > 0) {
|
|
@@ -6945,6 +7511,7 @@ var PerformanceWeightMixer = class {
|
|
|
6945
7511
|
};
|
|
6946
7512
|
|
|
6947
7513
|
// ../../clients/web-client/src/character/facial-performance-mixer.ts
|
|
7514
|
+
var VisemeResponseRate = 30;
|
|
6948
7515
|
var DEFAULT_PLAY_DURATION_MS = 1500;
|
|
6949
7516
|
var MOUTH_CHANNELS = [
|
|
6950
7517
|
VRMExpressionPresetName.Aa,
|
|
@@ -6972,7 +7539,8 @@ var FacialPerformanceLayer = {
|
|
|
6972
7539
|
Emotion: "emotion",
|
|
6973
7540
|
Viseme: "viseme",
|
|
6974
7541
|
Blink: "blink",
|
|
6975
|
-
Gaze: "gaze"
|
|
7542
|
+
Gaze: "gaze",
|
|
7543
|
+
Overlay: "overlay"
|
|
6976
7544
|
};
|
|
6977
7545
|
var getFacialPerformanceLayerId = function(channel) {
|
|
6978
7546
|
const normalized = channel.toLowerCase();
|
|
@@ -7123,6 +7691,22 @@ var FacialPerformanceMixer = class {
|
|
|
7123
7691
|
}
|
|
7124
7692
|
this.releaseVisemeMouthMode();
|
|
7125
7693
|
}
|
|
7694
|
+
/** Blend temporary emotions without changing timeline targets or mouth ownership. */
|
|
7695
|
+
setOverlay(weights) {
|
|
7696
|
+
this.clearOverlay();
|
|
7697
|
+
Object.entries(weights).forEach(([channel, weight]) => {
|
|
7698
|
+
if (this.availableChannelSet.has(channel) && getFacialPerformanceLayerId(channel) === FacialPerformanceLayer.Emotion && Number.isFinite(weight)) {
|
|
7699
|
+
this.weightMixer.setTarget(
|
|
7700
|
+
FacialPerformanceLayer.Overlay,
|
|
7701
|
+
channel,
|
|
7702
|
+
weight
|
|
7703
|
+
);
|
|
7704
|
+
}
|
|
7705
|
+
});
|
|
7706
|
+
}
|
|
7707
|
+
clearOverlay() {
|
|
7708
|
+
this.weightMixer.clearLayerTargets(FacialPerformanceLayer.Overlay);
|
|
7709
|
+
}
|
|
7126
7710
|
releaseVisemeMouthMode() {
|
|
7127
7711
|
if (this.activeMouthMode !== "viseme") {
|
|
7128
7712
|
return;
|
|
@@ -7240,7 +7824,9 @@ var FacialPerformanceMixer = class {
|
|
|
7240
7824
|
this.weightMixer.registerLayer({
|
|
7241
7825
|
id: FacialPerformanceLayer.Viseme,
|
|
7242
7826
|
priority: 200,
|
|
7243
|
-
blendMode: "override"
|
|
7827
|
+
blendMode: "override",
|
|
7828
|
+
// A 33 ms time constant: an 80 ms lip closure reaches 90% closed.
|
|
7829
|
+
responseRate: VisemeResponseRate
|
|
7244
7830
|
});
|
|
7245
7831
|
this.weightMixer.registerLayer({
|
|
7246
7832
|
id: FacialPerformanceLayer.Blink,
|
|
@@ -7252,6 +7838,11 @@ var FacialPerformanceMixer = class {
|
|
|
7252
7838
|
priority: 400,
|
|
7253
7839
|
blendMode: "override"
|
|
7254
7840
|
});
|
|
7841
|
+
this.weightMixer.registerLayer({
|
|
7842
|
+
id: FacialPerformanceLayer.Overlay,
|
|
7843
|
+
priority: 500,
|
|
7844
|
+
blendMode: "maximum"
|
|
7845
|
+
});
|
|
7255
7846
|
}
|
|
7256
7847
|
updateIdleFace(delta, values) {
|
|
7257
7848
|
const manager = this.vrm?.expressionManager;
|
|
@@ -7376,11 +7967,11 @@ var FacialPerformanceMixer = class {
|
|
|
7376
7967
|
|
|
7377
7968
|
// ../../clients/web-client/src/character/mesh.ts
|
|
7378
7969
|
import {
|
|
7379
|
-
Euler,
|
|
7970
|
+
Euler as Euler2,
|
|
7380
7971
|
Mesh,
|
|
7381
7972
|
Object3D,
|
|
7382
7973
|
SkinnedMesh,
|
|
7383
|
-
Vector3 as
|
|
7974
|
+
Vector3 as Vector36
|
|
7384
7975
|
} from "three";
|
|
7385
7976
|
var isSkinnedMesh = function(object) {
|
|
7386
7977
|
return object instanceof SkinnedMesh || object.isSkinnedMesh === true;
|
|
@@ -7389,14 +7980,14 @@ var isMesh = function(object) {
|
|
|
7389
7980
|
return isSkinnedMesh(object) || object instanceof Mesh || object.isMesh === true;
|
|
7390
7981
|
};
|
|
7391
7982
|
var applyVector3 = function(target, value) {
|
|
7392
|
-
if (value instanceof
|
|
7983
|
+
if (value instanceof Vector36) {
|
|
7393
7984
|
target.copy(value);
|
|
7394
7985
|
return;
|
|
7395
7986
|
}
|
|
7396
7987
|
target.set(value[0], value[1], value[2]);
|
|
7397
7988
|
};
|
|
7398
7989
|
var applyEuler = function(target, value) {
|
|
7399
|
-
if (value instanceof
|
|
7990
|
+
if (value instanceof Euler2) {
|
|
7400
7991
|
target.copy(value);
|
|
7401
7992
|
return;
|
|
7402
7993
|
}
|
|
@@ -7686,6 +8277,11 @@ var IncrementalStaging = class {
|
|
|
7686
8277
|
const dialogue = timeline.dialogue.map((next, index) => {
|
|
7687
8278
|
const previous = this.timeline.dialogue[index];
|
|
7688
8279
|
if (previous?.type === "speech" && previous.pending && next.type === "speech") {
|
|
8280
|
+
for (const key of Object.keys(previous)) {
|
|
8281
|
+
if (!(key in next)) {
|
|
8282
|
+
Reflect.deleteProperty(previous, key);
|
|
8283
|
+
}
|
|
8284
|
+
}
|
|
7689
8285
|
return Object.assign(previous, next, { pending: next.pending });
|
|
7690
8286
|
}
|
|
7691
8287
|
return next;
|
|
@@ -7748,6 +8344,30 @@ var IncrementalStaging = class {
|
|
|
7748
8344
|
}
|
|
7749
8345
|
};
|
|
7750
8346
|
|
|
8347
|
+
// ../../clients/web-client/src/character/playback-clock.ts
|
|
8348
|
+
var PlaybackClock = class {
|
|
8349
|
+
pausedAt = null;
|
|
8350
|
+
pausedSeconds = 0;
|
|
8351
|
+
wallTime() {
|
|
8352
|
+
return performance.now() / 1e3;
|
|
8353
|
+
}
|
|
8354
|
+
get paused() {
|
|
8355
|
+
return this.pausedAt !== null;
|
|
8356
|
+
}
|
|
8357
|
+
now() {
|
|
8358
|
+
return (this.pausedAt ?? this.wallTime()) - this.pausedSeconds;
|
|
8359
|
+
}
|
|
8360
|
+
pause() {
|
|
8361
|
+
this.pausedAt ??= this.wallTime();
|
|
8362
|
+
}
|
|
8363
|
+
resume() {
|
|
8364
|
+
if (this.pausedAt !== null) {
|
|
8365
|
+
this.pausedSeconds += this.wallTime() - this.pausedAt;
|
|
8366
|
+
this.pausedAt = null;
|
|
8367
|
+
}
|
|
8368
|
+
}
|
|
8369
|
+
};
|
|
8370
|
+
|
|
7751
8371
|
// ../../clients/web-client/src/character/staging-character-cues.ts
|
|
7752
8372
|
var findCharacterSequence = function(characters, sequence, fromIndex) {
|
|
7753
8373
|
if (sequence.length === 0) {
|
|
@@ -7849,12 +8469,6 @@ var buildStagingCharacterCues = function(text, wordTimings, duration, characterT
|
|
|
7849
8469
|
};
|
|
7850
8470
|
|
|
7851
8471
|
// ../../clients/web-client/src/character/staging-player.ts
|
|
7852
|
-
var nowSeconds = function() {
|
|
7853
|
-
if (typeof performance !== "undefined") {
|
|
7854
|
-
return performance.now() / 1e3;
|
|
7855
|
-
}
|
|
7856
|
-
return Date.now() / 1e3;
|
|
7857
|
-
};
|
|
7858
8472
|
var abortError2 = function() {
|
|
7859
8473
|
const error = new Error("Staging playback was cancelled");
|
|
7860
8474
|
error.name = "AbortError";
|
|
@@ -7886,6 +8500,8 @@ var StagingPlayer = class {
|
|
|
7886
8500
|
this.facialMixer = facialMixer;
|
|
7887
8501
|
this.animationController = animationController;
|
|
7888
8502
|
}
|
|
8503
|
+
clock = new PlaybackClock();
|
|
8504
|
+
pauseRevision = 0;
|
|
7889
8505
|
playbackController = null;
|
|
7890
8506
|
generation = 0;
|
|
7891
8507
|
incremental = null;
|
|
@@ -7895,6 +8511,40 @@ var StagingPlayer = class {
|
|
|
7895
8511
|
activeExpression = null;
|
|
7896
8512
|
activeExpressionMouthMode = null;
|
|
7897
8513
|
activeAnimation = null;
|
|
8514
|
+
get paused() {
|
|
8515
|
+
return this.clock.paused;
|
|
8516
|
+
}
|
|
8517
|
+
pause() {
|
|
8518
|
+
this.pauseRevision += 1;
|
|
8519
|
+
this.clock.pause();
|
|
8520
|
+
this.audioPlayer.element?.pause();
|
|
8521
|
+
}
|
|
8522
|
+
async resume() {
|
|
8523
|
+
if (!this.paused) {
|
|
8524
|
+
return;
|
|
8525
|
+
}
|
|
8526
|
+
const generation = this.generation;
|
|
8527
|
+
const pauseRevision = this.pauseRevision;
|
|
8528
|
+
if (this.pendingSpeech?.started && this.pendingSpeech.audio.type !== "silent") {
|
|
8529
|
+
await this.audioPlayer.element?.play();
|
|
8530
|
+
}
|
|
8531
|
+
if (generation !== this.generation) {
|
|
8532
|
+
return;
|
|
8533
|
+
}
|
|
8534
|
+
if (pauseRevision !== this.pauseRevision) {
|
|
8535
|
+
this.audioPlayer.element?.pause();
|
|
8536
|
+
return;
|
|
8537
|
+
}
|
|
8538
|
+
this.clock.resume();
|
|
8539
|
+
}
|
|
8540
|
+
async waitUntilResumed(signal) {
|
|
8541
|
+
while (this.paused) {
|
|
8542
|
+
await waitFrame(signal);
|
|
8543
|
+
}
|
|
8544
|
+
if (signal.aborted) {
|
|
8545
|
+
throw abortError2();
|
|
8546
|
+
}
|
|
8547
|
+
}
|
|
7898
8548
|
async play(timeline, options) {
|
|
7899
8549
|
this.cancel(false);
|
|
7900
8550
|
const controller = new AbortController();
|
|
@@ -7933,6 +8583,7 @@ var StagingPlayer = class {
|
|
|
7933
8583
|
if (controller.signal.aborted) {
|
|
7934
8584
|
throw abortError2();
|
|
7935
8585
|
}
|
|
8586
|
+
await this.waitUntilResumed(controller.signal);
|
|
7936
8587
|
if (block.startTime > cursor) {
|
|
7937
8588
|
beginTimeline();
|
|
7938
8589
|
await this.playTimedRange(
|
|
@@ -8016,12 +8667,12 @@ var StagingPlayer = class {
|
|
|
8016
8667
|
}
|
|
8017
8668
|
}
|
|
8018
8669
|
startCursor(block, mediaClock) {
|
|
8019
|
-
const startedAt =
|
|
8670
|
+
const startedAt = this.clock.now();
|
|
8020
8671
|
this.cursor = () => ({
|
|
8021
8672
|
blockId: block.id,
|
|
8022
8673
|
offsetSeconds: Math.min(
|
|
8023
8674
|
block.endTime - block.startTime,
|
|
8024
|
-
Math.max(0, mediaClock ? mediaClock() :
|
|
8675
|
+
Math.max(0, mediaClock ? mediaClock() : this.clock.now() - startedAt)
|
|
8025
8676
|
)
|
|
8026
8677
|
});
|
|
8027
8678
|
this.onCursor?.(this.cursor());
|
|
@@ -8045,6 +8696,7 @@ var StagingPlayer = class {
|
|
|
8045
8696
|
this.generation += 1;
|
|
8046
8697
|
this.playbackController?.abort();
|
|
8047
8698
|
this.playbackController = null;
|
|
8699
|
+
this.clock.resume();
|
|
8048
8700
|
this.incremental = null;
|
|
8049
8701
|
this.clearCursor();
|
|
8050
8702
|
this.onCursor = void 0;
|
|
@@ -8156,6 +8808,9 @@ var StagingPlayer = class {
|
|
|
8156
8808
|
return;
|
|
8157
8809
|
}
|
|
8158
8810
|
started = true;
|
|
8811
|
+
if (this.paused) {
|
|
8812
|
+
this.audioPlayer.element?.pause();
|
|
8813
|
+
}
|
|
8159
8814
|
if (this.pendingSpeech?.id === speech.id) {
|
|
8160
8815
|
this.pendingSpeech.started = true;
|
|
8161
8816
|
}
|
|
@@ -8176,6 +8831,7 @@ var StagingPlayer = class {
|
|
|
8176
8831
|
let ticking = true;
|
|
8177
8832
|
const tick = async () => {
|
|
8178
8833
|
while (ticking && !signal.aborted) {
|
|
8834
|
+
await this.waitUntilResumed(signal);
|
|
8179
8835
|
if (started) {
|
|
8180
8836
|
observePhonemes();
|
|
8181
8837
|
const localTime = Math.min(
|
|
@@ -8214,14 +8870,15 @@ var StagingPlayer = class {
|
|
|
8214
8870
|
}
|
|
8215
8871
|
}
|
|
8216
8872
|
async playSilentSpeech(speech, timeline, signal, revealCharacters) {
|
|
8217
|
-
const startedAt =
|
|
8873
|
+
const startedAt = this.clock.now();
|
|
8218
8874
|
const duration = Math.max(speech.endTime - speech.startTime, 0);
|
|
8219
8875
|
let nextVisemeIndex = 0;
|
|
8220
8876
|
while (true) {
|
|
8221
8877
|
if (signal.aborted) {
|
|
8222
8878
|
throw abortError2();
|
|
8223
8879
|
}
|
|
8224
|
-
|
|
8880
|
+
await this.waitUntilResumed(signal);
|
|
8881
|
+
const elapsed = Math.min(this.clock.now() - startedAt, duration);
|
|
8225
8882
|
this.applyTracks(speech.startTime + elapsed, true, timeline);
|
|
8226
8883
|
revealCharacters(elapsed);
|
|
8227
8884
|
let viseme = speech.visemes[nextVisemeIndex];
|
|
@@ -8241,13 +8898,14 @@ var StagingPlayer = class {
|
|
|
8241
8898
|
revealCharacters(Number.POSITIVE_INFINITY);
|
|
8242
8899
|
}
|
|
8243
8900
|
async playTimedRange(startTime, endTime, speechActive, timeline, signal) {
|
|
8244
|
-
const startedAt =
|
|
8901
|
+
const startedAt = this.clock.now();
|
|
8245
8902
|
const duration = Math.max(endTime - startTime, 0);
|
|
8246
8903
|
while (true) {
|
|
8247
8904
|
if (signal.aborted) {
|
|
8248
8905
|
throw abortError2();
|
|
8249
8906
|
}
|
|
8250
|
-
|
|
8907
|
+
await this.waitUntilResumed(signal);
|
|
8908
|
+
const elapsed = Math.min(this.clock.now() - startedAt, duration);
|
|
8251
8909
|
this.applyTracks(startTime + elapsed, speechActive, timeline);
|
|
8252
8910
|
if (elapsed >= duration) {
|
|
8253
8911
|
return;
|
|
@@ -8747,13 +9405,10 @@ var CharacterRuntime = class {
|
|
|
8747
9405
|
animationId,
|
|
8748
9406
|
assetProfile: this.assetProfile
|
|
8749
9407
|
}));
|
|
8750
|
-
this.lazyAnimations = new LazyAnimations(
|
|
8751
|
-
|
|
8752
|
-
(id, animation
|
|
8753
|
-
|
|
8754
|
-
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
8755
|
-
}
|
|
8756
|
-
);
|
|
9408
|
+
this.lazyAnimations = new LazyAnimations(loadAnimation, (id, animation) => {
|
|
9409
|
+
this.animations.set(id, animation);
|
|
9410
|
+
this.animationController?.addAnimation?.(id, animation, { loop: false });
|
|
9411
|
+
});
|
|
8757
9412
|
this.performanceRecorder = new PerformanceRecorder({
|
|
8758
9413
|
characterId: this.characterId
|
|
8759
9414
|
});
|
|
@@ -8973,6 +9628,20 @@ var CharacterRuntime = class {
|
|
|
8973
9628
|
});
|
|
8974
9629
|
return unlocked;
|
|
8975
9630
|
}
|
|
9631
|
+
/** Freeze the current NPCP timeline, audio, face, and animation. */
|
|
9632
|
+
pausePlayback() {
|
|
9633
|
+
this.stagingPlayer?.pause();
|
|
9634
|
+
}
|
|
9635
|
+
/** Continue the same timeline. Call from a gesture if the browser requires it. */
|
|
9636
|
+
async resumePlayback() {
|
|
9637
|
+
await this.stagingPlayer?.resume();
|
|
9638
|
+
}
|
|
9639
|
+
/** Cancel current output and pending audio/animation loads. */
|
|
9640
|
+
interruptPlayback() {
|
|
9641
|
+
if (this.animationController) {
|
|
9642
|
+
this.streamPlayback.cleanup();
|
|
9643
|
+
}
|
|
9644
|
+
}
|
|
8976
9645
|
getExpressions() {
|
|
8977
9646
|
return this.facialMixer.getChannelNames();
|
|
8978
9647
|
}
|
|
@@ -9016,6 +9685,20 @@ var CharacterRuntime = class {
|
|
|
9016
9685
|
clearLookAt() {
|
|
9017
9686
|
this.setGazeTarget(null);
|
|
9018
9687
|
}
|
|
9688
|
+
/** Briefly look at an object; release restores the prior target and gaze mode. */
|
|
9689
|
+
lookAtTemporarily(target) {
|
|
9690
|
+
const previous = this.currentGazeTarget;
|
|
9691
|
+
const wasActive = this.lookAtActive;
|
|
9692
|
+
this.lookAt(target);
|
|
9693
|
+
return () => {
|
|
9694
|
+
if (this.currentGazeTarget !== target || !this.lookAtActive) {
|
|
9695
|
+
return;
|
|
9696
|
+
}
|
|
9697
|
+
this.currentGazeTarget = previous;
|
|
9698
|
+
this.lookAtActive = wasActive;
|
|
9699
|
+
this.applyGaze();
|
|
9700
|
+
};
|
|
9701
|
+
}
|
|
9019
9702
|
startPerformanceRecording(options) {
|
|
9020
9703
|
this.performanceRecorder.start(options);
|
|
9021
9704
|
this.facialMixer.capturePerformanceModel();
|
|
@@ -9064,6 +9747,9 @@ var CharacterRuntime = class {
|
|
|
9064
9747
|
}
|
|
9065
9748
|
/** Advance skeletal animations, lip-sync visemes, facial expressions, and VRM physics by `delta` seconds. */
|
|
9066
9749
|
update(delta) {
|
|
9750
|
+
if (this.stagingPlayer?.paused) {
|
|
9751
|
+
return;
|
|
9752
|
+
}
|
|
9067
9753
|
this.animationController?.update(delta);
|
|
9068
9754
|
this.visemePlayer.update(delta);
|
|
9069
9755
|
this.facialMixer.update(delta);
|
|
@@ -9153,6 +9839,18 @@ var ConversationSocket = class {
|
|
|
9153
9839
|
this.socket?.disconnect();
|
|
9154
9840
|
this.socket = null;
|
|
9155
9841
|
}
|
|
9842
|
+
/** An event with an acknowledgement. Needs `ensureConnected` first. */
|
|
9843
|
+
async request(event, payload, timeoutMs = 1e4) {
|
|
9844
|
+
const socket = this.socket;
|
|
9845
|
+
if (!socket?.connected) {
|
|
9846
|
+
throw new Error("The conversation connection is not open.");
|
|
9847
|
+
}
|
|
9848
|
+
return await socket.timeout(timeoutMs).emitWithAck(event, payload);
|
|
9849
|
+
}
|
|
9850
|
+
/** An event without an acknowledgement, in order with the other events. */
|
|
9851
|
+
send(event, payload) {
|
|
9852
|
+
this.socket?.emit(event, payload);
|
|
9853
|
+
}
|
|
9156
9854
|
async ensureConnected(conversationId) {
|
|
9157
9855
|
if (typeof window === "undefined") {
|
|
9158
9856
|
return;
|
|
@@ -9207,6 +9905,41 @@ var ConversationSocket = class {
|
|
|
9207
9905
|
}
|
|
9208
9906
|
this.options.handleStreamEvent(event);
|
|
9209
9907
|
});
|
|
9908
|
+
this.socket.on(SPEECH_EVENTS.partial, (payload) => {
|
|
9909
|
+
const event = payload;
|
|
9910
|
+
if (typeof event?.sessionId === "string" && typeof event.text === "string") {
|
|
9911
|
+
this.options.handleSpeechEvent?.({ type: "partial", event });
|
|
9912
|
+
}
|
|
9913
|
+
});
|
|
9914
|
+
this.socket.on(SPEECH_EVENTS.turn, (payload) => {
|
|
9915
|
+
let candidate;
|
|
9916
|
+
try {
|
|
9917
|
+
candidate = decodePerformance(
|
|
9918
|
+
payload,
|
|
9919
|
+
"turn"
|
|
9920
|
+
);
|
|
9921
|
+
} catch (error) {
|
|
9922
|
+
this.options.handleCallbackError("speech.turn.decode", error);
|
|
9923
|
+
return;
|
|
9924
|
+
}
|
|
9925
|
+
if (candidate?.conversationId !== this.options.getConversationId() || typeof candidate.requestId !== "string" || !candidate.turn || typeof candidate.turn !== "object") {
|
|
9926
|
+
return;
|
|
9927
|
+
}
|
|
9928
|
+
this.options.handleSpeechEvent?.({
|
|
9929
|
+
type: "turn",
|
|
9930
|
+
requestId: candidate.requestId,
|
|
9931
|
+
turn: candidate.turn
|
|
9932
|
+
});
|
|
9933
|
+
});
|
|
9934
|
+
this.socket.on(SPEECH_EVENTS.turnFailed, (payload) => {
|
|
9935
|
+
const event = payload;
|
|
9936
|
+
if (typeof event?.requestId === "string" && event.conversationId === this.options.getConversationId()) {
|
|
9937
|
+
this.options.handleSpeechEvent?.({ type: "turnFailed", event });
|
|
9938
|
+
}
|
|
9939
|
+
});
|
|
9940
|
+
this.socket.on("disconnect", () => {
|
|
9941
|
+
this.options.handleSpeechEvent?.({ type: "disconnect" });
|
|
9942
|
+
});
|
|
9210
9943
|
this.socket.on("conversations:turn", (payload) => {
|
|
9211
9944
|
let candidate;
|
|
9212
9945
|
try {
|
|
@@ -9259,59 +9992,1059 @@ var ConversationSocket = class {
|
|
|
9259
9992
|
}
|
|
9260
9993
|
};
|
|
9261
9994
|
|
|
9262
|
-
// ../../clients/web-client/src/
|
|
9263
|
-
var
|
|
9264
|
-
|
|
9265
|
-
|
|
9266
|
-
|
|
9267
|
-
|
|
9268
|
-
|
|
9269
|
-
|
|
9270
|
-
|
|
9271
|
-
|
|
9272
|
-
|
|
9273
|
-
|
|
9274
|
-
|
|
9275
|
-
|
|
9276
|
-
|
|
9277
|
-
|
|
9278
|
-
|
|
9279
|
-
|
|
9280
|
-
|
|
9281
|
-
|
|
9282
|
-
|
|
9283
|
-
|
|
9284
|
-
|
|
9285
|
-
|
|
9286
|
-
|
|
9287
|
-
|
|
9288
|
-
assetProfile: resolvedProfile
|
|
9289
|
-
})
|
|
9290
|
-
});
|
|
9291
|
-
this.authoring = options.authoring ?? false;
|
|
9292
|
-
this.bakedAsset = options.bakedConversation;
|
|
9293
|
-
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
9294
|
-
this.conversationSocket = new ConversationSocket({
|
|
9295
|
-
baseUrl: this.baseUrl,
|
|
9296
|
-
apiToken: this.apiToken,
|
|
9297
|
-
getConversationId: () => this.conversationId,
|
|
9298
|
-
handleStreamEvent: (event) => {
|
|
9299
|
-
if (event.type !== "text_delta") {
|
|
9300
|
-
this.thinkingState = false;
|
|
9995
|
+
// ../../clients/web-client/src/voice/push-to-talk.ts
|
|
9996
|
+
var INTERACTIVE = 'input, textarea, select, button, a[href], summary, [contenteditable]:not([contenteditable="false"]), [role="button"], [role="textbox"], [role="checkbox"], [role="switch"], [role="menuitem"], [role="tab"], [role="slider"], [role="option"]';
|
|
9997
|
+
var elementOf = function(target) {
|
|
9998
|
+
return target && typeof target.closest === "function" ? target : null;
|
|
9999
|
+
};
|
|
10000
|
+
var bindPushToTalk = function(target, options = {}) {
|
|
10001
|
+
const element = options.element ?? null;
|
|
10002
|
+
const key = options.key === void 0 ? "Space" : options.key;
|
|
10003
|
+
const minHoldMs = options.minHoldMs ?? SPEECH_MIN_HOLD_MS;
|
|
10004
|
+
let active = null;
|
|
10005
|
+
let pointerId = null;
|
|
10006
|
+
let pressedAt = 0;
|
|
10007
|
+
const begin = function(source) {
|
|
10008
|
+
if (active || target.listeningState !== "idle") {
|
|
10009
|
+
return false;
|
|
10010
|
+
}
|
|
10011
|
+
if (options.enabled && !options.enabled()) {
|
|
10012
|
+
return false;
|
|
10013
|
+
}
|
|
10014
|
+
active = source;
|
|
10015
|
+
pressedAt = performance.now();
|
|
10016
|
+
const listening = typeof options.listening === "function" ? options.listening() : options.listening ?? {};
|
|
10017
|
+
target.startListening({ ...listening, endOnSilenceMs: null }).then(
|
|
10018
|
+
(result) => {
|
|
10019
|
+
if (result) {
|
|
10020
|
+
options.onResult?.(result);
|
|
9301
10021
|
}
|
|
9302
|
-
this.streamPlayback.handleEvent(event);
|
|
9303
10022
|
},
|
|
9304
|
-
|
|
9305
|
-
|
|
10023
|
+
(error) => {
|
|
10024
|
+
active = null;
|
|
10025
|
+
options.onError?.(error);
|
|
10026
|
+
}
|
|
10027
|
+
);
|
|
10028
|
+
return true;
|
|
10029
|
+
};
|
|
10030
|
+
const end = function() {
|
|
10031
|
+
if (!active) {
|
|
10032
|
+
return;
|
|
10033
|
+
}
|
|
10034
|
+
active = null;
|
|
10035
|
+
pointerId = null;
|
|
10036
|
+
if (performance.now() - pressedAt < minHoldMs) {
|
|
10037
|
+
target.cancelListening();
|
|
10038
|
+
options.onShortPress?.();
|
|
10039
|
+
return;
|
|
10040
|
+
}
|
|
10041
|
+
const starting = target.listeningState === "starting";
|
|
10042
|
+
target.stopListening().then(
|
|
10043
|
+
(result) => {
|
|
10044
|
+
if (starting && result === null) {
|
|
10045
|
+
options.onShortPress?.();
|
|
10046
|
+
}
|
|
9306
10047
|
},
|
|
9307
|
-
|
|
9308
|
-
|
|
9309
|
-
|
|
9310
|
-
|
|
9311
|
-
|
|
9312
|
-
|
|
9313
|
-
|
|
9314
|
-
|
|
10048
|
+
() => void 0
|
|
10049
|
+
);
|
|
10050
|
+
};
|
|
10051
|
+
const cancel = function() {
|
|
10052
|
+
if (!active) {
|
|
10053
|
+
return;
|
|
10054
|
+
}
|
|
10055
|
+
active = null;
|
|
10056
|
+
pointerId = null;
|
|
10057
|
+
target.cancelListening();
|
|
10058
|
+
};
|
|
10059
|
+
const isOwnElement = function(node) {
|
|
10060
|
+
return Boolean(element && node && element.contains(node));
|
|
10061
|
+
};
|
|
10062
|
+
const onKeyDown = function(event) {
|
|
10063
|
+
if (event.key === "Escape" || event.code === "Escape") {
|
|
10064
|
+
cancel();
|
|
10065
|
+
return;
|
|
10066
|
+
}
|
|
10067
|
+
const node = elementOf(event.target);
|
|
10068
|
+
const ownElement = isOwnElement(node);
|
|
10069
|
+
const pressesElement = ownElement && (event.code === "Space" || event.code === "Enter");
|
|
10070
|
+
if (!pressesElement && (!key || event.code !== key)) {
|
|
10071
|
+
return;
|
|
10072
|
+
}
|
|
10073
|
+
if (event.repeat) {
|
|
10074
|
+
if (active === "key") {
|
|
10075
|
+
event.preventDefault();
|
|
10076
|
+
}
|
|
10077
|
+
return;
|
|
10078
|
+
}
|
|
10079
|
+
if (event.ctrlKey || event.metaKey || event.altKey || event.isComposing) {
|
|
10080
|
+
return;
|
|
10081
|
+
}
|
|
10082
|
+
if (!ownElement && node?.closest(INTERACTIVE)) {
|
|
10083
|
+
return;
|
|
10084
|
+
}
|
|
10085
|
+
if (begin("key")) {
|
|
10086
|
+
event.preventDefault();
|
|
10087
|
+
}
|
|
10088
|
+
};
|
|
10089
|
+
const onKeyUp = function(event) {
|
|
10090
|
+
if (active !== "key") {
|
|
10091
|
+
return;
|
|
10092
|
+
}
|
|
10093
|
+
const ownElement = isOwnElement(elementOf(event.target));
|
|
10094
|
+
if (event.code === key || ownElement && (event.code === "Space" || event.code === "Enter")) {
|
|
10095
|
+
event.preventDefault();
|
|
10096
|
+
end();
|
|
10097
|
+
}
|
|
10098
|
+
};
|
|
10099
|
+
const onVisibility = function() {
|
|
10100
|
+
if (document.visibilityState === "hidden") {
|
|
10101
|
+
cancel();
|
|
10102
|
+
}
|
|
10103
|
+
};
|
|
10104
|
+
const onPointerDown = function(event) {
|
|
10105
|
+
if (event.pointerType === "mouse" && event.button !== 0) {
|
|
10106
|
+
return;
|
|
10107
|
+
}
|
|
10108
|
+
event.preventDefault();
|
|
10109
|
+
if (!begin("pointer")) {
|
|
10110
|
+
return;
|
|
10111
|
+
}
|
|
10112
|
+
pointerId = event.pointerId;
|
|
10113
|
+
try {
|
|
10114
|
+
element?.setPointerCapture(event.pointerId);
|
|
10115
|
+
} catch {
|
|
10116
|
+
}
|
|
10117
|
+
};
|
|
10118
|
+
const onPointerUp = function(event) {
|
|
10119
|
+
if (active === "pointer" && event.pointerId === pointerId) {
|
|
10120
|
+
end();
|
|
10121
|
+
}
|
|
10122
|
+
};
|
|
10123
|
+
const onPointerCancel = function(event) {
|
|
10124
|
+
if (active === "pointer" && event.pointerId === pointerId) {
|
|
10125
|
+
cancel();
|
|
10126
|
+
}
|
|
10127
|
+
};
|
|
10128
|
+
const prevent = function(event) {
|
|
10129
|
+
event.preventDefault();
|
|
10130
|
+
};
|
|
10131
|
+
const restoreStyle = [];
|
|
10132
|
+
if (element) {
|
|
10133
|
+
for (const [property, value] of [
|
|
10134
|
+
["touch-action", "none"],
|
|
10135
|
+
["user-select", "none"],
|
|
10136
|
+
["-webkit-user-select", "none"],
|
|
10137
|
+
["-webkit-touch-callout", "none"]
|
|
10138
|
+
]) {
|
|
10139
|
+
const previous = element.style.getPropertyValue(property);
|
|
10140
|
+
element.style.setProperty(property, value);
|
|
10141
|
+
restoreStyle.push(() => {
|
|
10142
|
+
if (previous) {
|
|
10143
|
+
element.style.setProperty(property, previous);
|
|
10144
|
+
} else {
|
|
10145
|
+
element.style.removeProperty(property);
|
|
10146
|
+
}
|
|
10147
|
+
});
|
|
10148
|
+
}
|
|
10149
|
+
element.addEventListener("pointerdown", onPointerDown);
|
|
10150
|
+
window.addEventListener("pointerup", onPointerUp, true);
|
|
10151
|
+
window.addEventListener("pointercancel", onPointerCancel, true);
|
|
10152
|
+
element.addEventListener("contextmenu", prevent);
|
|
10153
|
+
element.addEventListener("selectstart", prevent);
|
|
10154
|
+
element.addEventListener("touchstart", prevent, { passive: false });
|
|
10155
|
+
}
|
|
10156
|
+
window.addEventListener("keydown", onKeyDown);
|
|
10157
|
+
window.addEventListener("keyup", onKeyUp);
|
|
10158
|
+
window.addEventListener("blur", cancel);
|
|
10159
|
+
document.addEventListener("visibilitychange", onVisibility);
|
|
10160
|
+
return () => {
|
|
10161
|
+
cancel();
|
|
10162
|
+
if (element) {
|
|
10163
|
+
element.removeEventListener("pointerdown", onPointerDown);
|
|
10164
|
+
window.removeEventListener("pointerup", onPointerUp, true);
|
|
10165
|
+
window.removeEventListener("pointercancel", onPointerCancel, true);
|
|
10166
|
+
element.removeEventListener("contextmenu", prevent);
|
|
10167
|
+
element.removeEventListener("selectstart", prevent);
|
|
10168
|
+
element.removeEventListener("touchstart", prevent);
|
|
10169
|
+
restoreStyle.forEach((restore) => restore());
|
|
10170
|
+
}
|
|
10171
|
+
window.removeEventListener("keydown", onKeyDown);
|
|
10172
|
+
window.removeEventListener("keyup", onKeyUp);
|
|
10173
|
+
window.removeEventListener("blur", cancel);
|
|
10174
|
+
document.removeEventListener("visibilitychange", onVisibility);
|
|
10175
|
+
};
|
|
10176
|
+
};
|
|
10177
|
+
|
|
10178
|
+
// ../../clients/web-client/src/voice/microphone-capture.ts
|
|
10179
|
+
var FRAME_SAMPLES = SPEECH_AUDIO_FORMAT.sampleRate * SPEECH_FRAME_MS / 1e3;
|
|
10180
|
+
var MicrophoneError = class extends Error {
|
|
10181
|
+
constructor(code, message) {
|
|
10182
|
+
super(message);
|
|
10183
|
+
this.code = code;
|
|
10184
|
+
this.name = "NpcifyMicrophoneError";
|
|
10185
|
+
}
|
|
10186
|
+
};
|
|
10187
|
+
var Pcm16Framer = class {
|
|
10188
|
+
constructor(inputRate, emit) {
|
|
10189
|
+
this.emit = emit;
|
|
10190
|
+
this.ratio = inputRate / SPEECH_AUDIO_FORMAT.sampleRate;
|
|
10191
|
+
}
|
|
10192
|
+
ratio;
|
|
10193
|
+
frame = new Int16Array(FRAME_SAMPLES);
|
|
10194
|
+
length = 0;
|
|
10195
|
+
sum = 0;
|
|
10196
|
+
count = 0;
|
|
10197
|
+
phase = 0;
|
|
10198
|
+
push(samples) {
|
|
10199
|
+
for (let index = 0; index < samples.length; index += 1) {
|
|
10200
|
+
this.sum += samples[index] ?? 0;
|
|
10201
|
+
this.count += 1;
|
|
10202
|
+
this.phase += 1;
|
|
10203
|
+
while (this.phase >= this.ratio) {
|
|
10204
|
+
this.phase -= this.ratio;
|
|
10205
|
+
this.write(this.sum / Math.max(1, this.count));
|
|
10206
|
+
if (this.ratio >= 1) {
|
|
10207
|
+
this.sum = 0;
|
|
10208
|
+
this.count = 0;
|
|
10209
|
+
}
|
|
10210
|
+
}
|
|
10211
|
+
if (this.ratio < 1) {
|
|
10212
|
+
this.sum = 0;
|
|
10213
|
+
this.count = 0;
|
|
10214
|
+
}
|
|
10215
|
+
}
|
|
10216
|
+
}
|
|
10217
|
+
flush() {
|
|
10218
|
+
if (this.count > 0 && this.ratio >= 1) {
|
|
10219
|
+
this.write(this.sum / this.count);
|
|
10220
|
+
this.sum = 0;
|
|
10221
|
+
this.count = 0;
|
|
10222
|
+
}
|
|
10223
|
+
if (this.length > 0) {
|
|
10224
|
+
this.emit(this.frame.slice(0, this.length));
|
|
10225
|
+
this.length = 0;
|
|
10226
|
+
}
|
|
10227
|
+
}
|
|
10228
|
+
write(value) {
|
|
10229
|
+
const clamped = Math.max(-1, Math.min(1, value));
|
|
10230
|
+
this.frame[this.length] = Math.round(
|
|
10231
|
+
clamped < 0 ? clamped * 32768 : clamped * 32767
|
|
10232
|
+
);
|
|
10233
|
+
this.length += 1;
|
|
10234
|
+
if (this.length === FRAME_SAMPLES) {
|
|
10235
|
+
this.emit(this.frame);
|
|
10236
|
+
this.frame = new Int16Array(FRAME_SAMPLES);
|
|
10237
|
+
this.length = 0;
|
|
10238
|
+
}
|
|
10239
|
+
}
|
|
10240
|
+
};
|
|
10241
|
+
var levelOf = function(pcm) {
|
|
10242
|
+
if (pcm.length === 0) {
|
|
10243
|
+
return 0;
|
|
10244
|
+
}
|
|
10245
|
+
let sum = 0;
|
|
10246
|
+
for (const sample of pcm) {
|
|
10247
|
+
sum += (sample / 32768) ** 2;
|
|
10248
|
+
}
|
|
10249
|
+
return Math.sqrt(sum / pcm.length);
|
|
10250
|
+
};
|
|
10251
|
+
var WORKLET_SOURCE = `
|
|
10252
|
+
const FRAME = ${FRAME_SAMPLES}
|
|
10253
|
+
class NpcifyCaptureProcessor extends AudioWorkletProcessor {
|
|
10254
|
+
constructor() {
|
|
10255
|
+
super()
|
|
10256
|
+
this.ratio = sampleRate / ${SPEECH_AUDIO_FORMAT.sampleRate}
|
|
10257
|
+
this.frame = new Int16Array(FRAME)
|
|
10258
|
+
this.length = 0
|
|
10259
|
+
this.sum = 0
|
|
10260
|
+
this.count = 0
|
|
10261
|
+
this.phase = 0
|
|
10262
|
+
this.port.onmessage = (event) => {
|
|
10263
|
+
if (event.data === 'flush') {
|
|
10264
|
+
if (this.count > 0 && this.ratio >= 1) {
|
|
10265
|
+
this.write(this.sum / this.count)
|
|
10266
|
+
this.sum = 0
|
|
10267
|
+
this.count = 0
|
|
10268
|
+
}
|
|
10269
|
+
if (this.length > 0) {
|
|
10270
|
+
this.post(this.frame.slice(0, this.length))
|
|
10271
|
+
this.length = 0
|
|
10272
|
+
}
|
|
10273
|
+
this.port.postMessage({ type: 'flushed' })
|
|
10274
|
+
}
|
|
10275
|
+
}
|
|
10276
|
+
}
|
|
10277
|
+
write(value) {
|
|
10278
|
+
const clamped = Math.max(-1, Math.min(1, value))
|
|
10279
|
+
this.frame[this.length++] = Math.round(clamped < 0 ? clamped * 32768 : clamped * 32767)
|
|
10280
|
+
if (this.length === FRAME) {
|
|
10281
|
+
this.post(this.frame)
|
|
10282
|
+
this.frame = new Int16Array(FRAME)
|
|
10283
|
+
this.length = 0
|
|
10284
|
+
}
|
|
10285
|
+
}
|
|
10286
|
+
post(pcm) {
|
|
10287
|
+
this.port.postMessage({ type: 'frame', pcm }, [pcm.buffer])
|
|
10288
|
+
}
|
|
10289
|
+
process(inputs) {
|
|
10290
|
+
const channels = inputs[0]
|
|
10291
|
+
if (!channels || channels.length === 0) return true
|
|
10292
|
+
const size = channels[0].length
|
|
10293
|
+
for (let index = 0; index < size; index += 1) {
|
|
10294
|
+
let value = 0
|
|
10295
|
+
for (const channel of channels) value += channel[index]
|
|
10296
|
+
this.sum += value / channels.length
|
|
10297
|
+
this.count += 1
|
|
10298
|
+
this.phase += 1
|
|
10299
|
+
while (this.phase >= this.ratio) {
|
|
10300
|
+
this.phase -= this.ratio
|
|
10301
|
+
this.write(this.sum / Math.max(1, this.count))
|
|
10302
|
+
if (this.ratio >= 1) {
|
|
10303
|
+
this.sum = 0
|
|
10304
|
+
this.count = 0
|
|
10305
|
+
}
|
|
10306
|
+
}
|
|
10307
|
+
if (this.ratio < 1) {
|
|
10308
|
+
this.sum = 0
|
|
10309
|
+
this.count = 0
|
|
10310
|
+
}
|
|
10311
|
+
}
|
|
10312
|
+
return true
|
|
10313
|
+
}
|
|
10314
|
+
}
|
|
10315
|
+
registerProcessor('npcify-capture', NpcifyCaptureProcessor)
|
|
10316
|
+
`;
|
|
10317
|
+
var MicrophoneCapture = class {
|
|
10318
|
+
context = null;
|
|
10319
|
+
workletReady = null;
|
|
10320
|
+
graph = null;
|
|
10321
|
+
generation = 0;
|
|
10322
|
+
async start(onFrame) {
|
|
10323
|
+
this.cancel();
|
|
10324
|
+
const generation = ++this.generation;
|
|
10325
|
+
const AudioContextClass = globalThis.AudioContext ?? globalThis.webkitAudioContext;
|
|
10326
|
+
if (!navigator.mediaDevices?.getUserMedia || !AudioContextClass) {
|
|
10327
|
+
throw new MicrophoneError(
|
|
10328
|
+
"MICROPHONE_UNAVAILABLE",
|
|
10329
|
+
"This browser cannot record audio."
|
|
10330
|
+
);
|
|
10331
|
+
}
|
|
10332
|
+
this.context ??= new AudioContextClass({ latencyHint: "interactive" });
|
|
10333
|
+
const context2 = this.context;
|
|
10334
|
+
const resumed = context2.resume().catch(() => void 0);
|
|
10335
|
+
let stream;
|
|
10336
|
+
try {
|
|
10337
|
+
stream = await navigator.mediaDevices.getUserMedia({
|
|
10338
|
+
audio: {
|
|
10339
|
+
channelCount: 1,
|
|
10340
|
+
echoCancellation: true,
|
|
10341
|
+
noiseSuppression: true,
|
|
10342
|
+
autoGainControl: true
|
|
10343
|
+
}
|
|
10344
|
+
});
|
|
10345
|
+
} catch (error) {
|
|
10346
|
+
const name = error?.name;
|
|
10347
|
+
throw name === "NotAllowedError" || name === "SecurityError" ? new MicrophoneError(
|
|
10348
|
+
"MICROPHONE_PERMISSION_DENIED",
|
|
10349
|
+
"Microphone access was denied."
|
|
10350
|
+
) : new MicrophoneError(
|
|
10351
|
+
"MICROPHONE_UNAVAILABLE",
|
|
10352
|
+
"No microphone is available."
|
|
10353
|
+
);
|
|
10354
|
+
}
|
|
10355
|
+
await resumed;
|
|
10356
|
+
if (generation !== this.generation) {
|
|
10357
|
+
stream.getTracks().forEach((track) => track.stop());
|
|
10358
|
+
return;
|
|
10359
|
+
}
|
|
10360
|
+
const deliver = (pcm) => onFrame({ pcm, level: levelOf(pcm) });
|
|
10361
|
+
const source = context2.createMediaStreamSource(stream);
|
|
10362
|
+
const sink = context2.createGain();
|
|
10363
|
+
sink.gain.value = 0;
|
|
10364
|
+
sink.connect(context2.destination);
|
|
10365
|
+
const graph = await this.workletGraph(context2, stream, source, sink, deliver) ?? this.scriptGraph(context2, stream, source, sink, deliver);
|
|
10366
|
+
if (generation !== this.generation) {
|
|
10367
|
+
this.close(graph, false);
|
|
10368
|
+
return;
|
|
10369
|
+
}
|
|
10370
|
+
this.graph = graph;
|
|
10371
|
+
}
|
|
10372
|
+
async stop() {
|
|
10373
|
+
const graph = this.graph;
|
|
10374
|
+
this.graph = null;
|
|
10375
|
+
this.generation += 1;
|
|
10376
|
+
if (!graph) {
|
|
10377
|
+
return;
|
|
10378
|
+
}
|
|
10379
|
+
try {
|
|
10380
|
+
await graph.flush();
|
|
10381
|
+
} finally {
|
|
10382
|
+
this.close(graph);
|
|
10383
|
+
}
|
|
10384
|
+
}
|
|
10385
|
+
cancel() {
|
|
10386
|
+
const graph = this.graph;
|
|
10387
|
+
this.graph = null;
|
|
10388
|
+
this.generation += 1;
|
|
10389
|
+
if (graph) {
|
|
10390
|
+
this.close(graph);
|
|
10391
|
+
}
|
|
10392
|
+
}
|
|
10393
|
+
close(graph, suspend = true) {
|
|
10394
|
+
graph.stream.getTracks().forEach((track) => track.stop());
|
|
10395
|
+
graph.source.disconnect();
|
|
10396
|
+
graph.node.disconnect();
|
|
10397
|
+
graph.sink.disconnect();
|
|
10398
|
+
if (graph.node instanceof AudioWorkletNode) {
|
|
10399
|
+
graph.node.port.onmessage = null;
|
|
10400
|
+
} else {
|
|
10401
|
+
graph.node.onaudioprocess = null;
|
|
10402
|
+
}
|
|
10403
|
+
if (suspend) {
|
|
10404
|
+
void this.context?.suspend().catch(() => void 0);
|
|
10405
|
+
}
|
|
10406
|
+
}
|
|
10407
|
+
async workletGraph(context2, stream, source, sink, deliver) {
|
|
10408
|
+
if (!context2.audioWorklet || typeof AudioWorkletNode === "undefined") {
|
|
10409
|
+
return null;
|
|
10410
|
+
}
|
|
10411
|
+
this.workletReady ??= (async () => {
|
|
10412
|
+
const url = URL.createObjectURL(
|
|
10413
|
+
new Blob([WORKLET_SOURCE], { type: "application/javascript" })
|
|
10414
|
+
);
|
|
10415
|
+
try {
|
|
10416
|
+
await context2.audioWorklet.addModule(url);
|
|
10417
|
+
return true;
|
|
10418
|
+
} catch {
|
|
10419
|
+
return false;
|
|
10420
|
+
} finally {
|
|
10421
|
+
URL.revokeObjectURL(url);
|
|
10422
|
+
}
|
|
10423
|
+
})();
|
|
10424
|
+
if (!await this.workletReady) {
|
|
10425
|
+
return null;
|
|
10426
|
+
}
|
|
10427
|
+
const node = new AudioWorkletNode(context2, "npcify-capture", {
|
|
10428
|
+
numberOfInputs: 1,
|
|
10429
|
+
numberOfOutputs: 1,
|
|
10430
|
+
channelCount: 1,
|
|
10431
|
+
channelCountMode: "explicit"
|
|
10432
|
+
});
|
|
10433
|
+
let flushed = null;
|
|
10434
|
+
node.port.onmessage = (event) => {
|
|
10435
|
+
if (event.data.type === "frame" && event.data.pcm) {
|
|
10436
|
+
deliver(event.data.pcm);
|
|
10437
|
+
} else if (event.data.type === "flushed") {
|
|
10438
|
+
flushed?.();
|
|
10439
|
+
}
|
|
10440
|
+
};
|
|
10441
|
+
source.connect(node);
|
|
10442
|
+
node.connect(sink);
|
|
10443
|
+
return {
|
|
10444
|
+
stream,
|
|
10445
|
+
source,
|
|
10446
|
+
node,
|
|
10447
|
+
sink,
|
|
10448
|
+
flush: () => new Promise((resolve) => {
|
|
10449
|
+
const timer = setTimeout(resolve, 250);
|
|
10450
|
+
flushed = () => {
|
|
10451
|
+
clearTimeout(timer);
|
|
10452
|
+
resolve();
|
|
10453
|
+
};
|
|
10454
|
+
node.port.postMessage("flush");
|
|
10455
|
+
})
|
|
10456
|
+
};
|
|
10457
|
+
}
|
|
10458
|
+
/** The fallback without AudioWorklet: a ScriptProcessorNode on the main thread. */
|
|
10459
|
+
scriptGraph(context2, stream, source, sink, deliver) {
|
|
10460
|
+
const framer = new Pcm16Framer(context2.sampleRate, deliver);
|
|
10461
|
+
const node = context2.createScriptProcessor(2048, 1, 1);
|
|
10462
|
+
node.onaudioprocess = (event) => {
|
|
10463
|
+
framer.push(event.inputBuffer.getChannelData(0));
|
|
10464
|
+
};
|
|
10465
|
+
source.connect(node);
|
|
10466
|
+
node.connect(sink);
|
|
10467
|
+
return {
|
|
10468
|
+
stream,
|
|
10469
|
+
source,
|
|
10470
|
+
node,
|
|
10471
|
+
sink,
|
|
10472
|
+
flush: () => {
|
|
10473
|
+
framer.flush();
|
|
10474
|
+
return Promise.resolve();
|
|
10475
|
+
}
|
|
10476
|
+
};
|
|
10477
|
+
}
|
|
10478
|
+
};
|
|
10479
|
+
|
|
10480
|
+
// ../../clients/web-client/src/voice/silence-detector.ts
|
|
10481
|
+
var MIN_SPEECH_LEVEL = 0.015;
|
|
10482
|
+
var SPEECH_TO_NOISE = 3;
|
|
10483
|
+
var SPEECH_START_MS = 200;
|
|
10484
|
+
var SilenceDetector = class {
|
|
10485
|
+
constructor(silenceMs) {
|
|
10486
|
+
this.silenceMs = silenceMs;
|
|
10487
|
+
}
|
|
10488
|
+
floor = null;
|
|
10489
|
+
loudMs = 0;
|
|
10490
|
+
quietMs = 0;
|
|
10491
|
+
speaking = false;
|
|
10492
|
+
/** True when the speech has ended. */
|
|
10493
|
+
push(level, frameMs = SPEECH_FRAME_MS) {
|
|
10494
|
+
const threshold = Math.max(
|
|
10495
|
+
MIN_SPEECH_LEVEL,
|
|
10496
|
+
(this.floor ?? MIN_SPEECH_LEVEL / SPEECH_TO_NOISE) * SPEECH_TO_NOISE
|
|
10497
|
+
);
|
|
10498
|
+
if (level >= threshold) {
|
|
10499
|
+
this.loudMs += frameMs;
|
|
10500
|
+
this.quietMs = 0;
|
|
10501
|
+
if (this.loudMs >= SPEECH_START_MS) {
|
|
10502
|
+
this.speaking = true;
|
|
10503
|
+
}
|
|
10504
|
+
return false;
|
|
10505
|
+
}
|
|
10506
|
+
this.floor = this.floor === null ? level : this.floor * 0.9 + level * 0.1;
|
|
10507
|
+
this.loudMs = 0;
|
|
10508
|
+
if (!this.speaking) {
|
|
10509
|
+
return false;
|
|
10510
|
+
}
|
|
10511
|
+
this.quietMs += frameMs;
|
|
10512
|
+
return this.quietMs >= this.silenceMs;
|
|
10513
|
+
}
|
|
10514
|
+
get heardSpeech() {
|
|
10515
|
+
return this.speaking;
|
|
10516
|
+
}
|
|
10517
|
+
};
|
|
10518
|
+
|
|
10519
|
+
// ../../clients/web-client/src/voice/voice-input.ts
|
|
10520
|
+
var SpeechInputError = class extends Error {
|
|
10521
|
+
constructor(code, message) {
|
|
10522
|
+
super(message ?? code);
|
|
10523
|
+
this.code = code;
|
|
10524
|
+
this.name = "NpcifySpeechInputError";
|
|
10525
|
+
}
|
|
10526
|
+
};
|
|
10527
|
+
var deferred = function() {
|
|
10528
|
+
let resolve;
|
|
10529
|
+
let reject;
|
|
10530
|
+
const promise = new Promise((done, fail) => {
|
|
10531
|
+
resolve = done;
|
|
10532
|
+
reject = fail;
|
|
10533
|
+
});
|
|
10534
|
+
promise.catch(() => void 0);
|
|
10535
|
+
return { promise, resolve, reject };
|
|
10536
|
+
};
|
|
10537
|
+
var CODE = /^[A-Z][A-Z0-9_]{2,63}$/u;
|
|
10538
|
+
var speechError = function(error, fallback) {
|
|
10539
|
+
if (error instanceof SpeechInputError) {
|
|
10540
|
+
return error;
|
|
10541
|
+
}
|
|
10542
|
+
if (error instanceof Error && "code" in error && typeof error.code === "string") {
|
|
10543
|
+
return new SpeechInputError(
|
|
10544
|
+
error.code,
|
|
10545
|
+
error.message
|
|
10546
|
+
);
|
|
10547
|
+
}
|
|
10548
|
+
return new SpeechInputError(
|
|
10549
|
+
fallback,
|
|
10550
|
+
error instanceof Error ? error.message : void 0
|
|
10551
|
+
);
|
|
10552
|
+
};
|
|
10553
|
+
var VoiceInput = class {
|
|
10554
|
+
constructor(deps) {
|
|
10555
|
+
this.deps = deps;
|
|
10556
|
+
}
|
|
10557
|
+
state = "idle";
|
|
10558
|
+
session = null;
|
|
10559
|
+
capture = null;
|
|
10560
|
+
/**
|
|
10561
|
+
* The session that the capture records for. A start that was cancelled
|
|
10562
|
+
* never touches the capture when a later session owns it.
|
|
10563
|
+
*/
|
|
10564
|
+
captureOwner = null;
|
|
10565
|
+
turns = /* @__PURE__ */ new Map();
|
|
10566
|
+
get listeningState() {
|
|
10567
|
+
return this.state;
|
|
10568
|
+
}
|
|
10569
|
+
/** A turn that the server started has not delivered its final payload. */
|
|
10570
|
+
get turnPending() {
|
|
10571
|
+
return this.turns.size > 0;
|
|
10572
|
+
}
|
|
10573
|
+
/**
|
|
10574
|
+
* Starts listening. Resolves with the result when the listening ends (by
|
|
10575
|
+
* `stop`, the silence rule or the time limit), or with null when it is
|
|
10576
|
+
* cancelled.
|
|
10577
|
+
*/
|
|
10578
|
+
async start(options = {}) {
|
|
10579
|
+
if (this.session) {
|
|
10580
|
+
throw new SpeechInputError(
|
|
10581
|
+
"SPEECH_ALREADY_LISTENING",
|
|
10582
|
+
"Listening has already started."
|
|
10583
|
+
);
|
|
10584
|
+
}
|
|
10585
|
+
this.deps.assertVoiceAllowed();
|
|
10586
|
+
if (this.turnPending || this.deps.turnInFlight()) {
|
|
10587
|
+
throw new SpeechInputError(
|
|
10588
|
+
"TURN_IN_PROGRESS",
|
|
10589
|
+
"A reply is already being generated. Please wait."
|
|
10590
|
+
);
|
|
10591
|
+
}
|
|
10592
|
+
const conversationId = this.deps.conversationId();
|
|
10593
|
+
if (!conversationId) {
|
|
10594
|
+
throw new SpeechInputError(
|
|
10595
|
+
"CONVERSATION_REQUIRED",
|
|
10596
|
+
"Create a conversation before listening."
|
|
10597
|
+
);
|
|
10598
|
+
}
|
|
10599
|
+
const maxDurationMs = Math.min(
|
|
10600
|
+
SPEECH_MAX_DURATION_MS,
|
|
10601
|
+
Math.max(1e3, options.maxDurationMs ?? SPEECH_MAX_DURATION_MS)
|
|
10602
|
+
);
|
|
10603
|
+
const session = {
|
|
10604
|
+
requestId: globalThis.crypto.randomUUID(),
|
|
10605
|
+
options,
|
|
10606
|
+
sessionId: null,
|
|
10607
|
+
sequence: 0,
|
|
10608
|
+
bytes: 0,
|
|
10609
|
+
pending: [],
|
|
10610
|
+
timer: null,
|
|
10611
|
+
silence: options.endOnSilenceMs && options.endOnSilenceMs > 0 ? new SilenceDetector(options.endOnSilenceMs) : null,
|
|
10612
|
+
stopRequested: false,
|
|
10613
|
+
stopping: null,
|
|
10614
|
+
captureClosing: null,
|
|
10615
|
+
captureClosed: false,
|
|
10616
|
+
disconnected: deferred(),
|
|
10617
|
+
done: deferred()
|
|
10618
|
+
};
|
|
10619
|
+
this.session = session;
|
|
10620
|
+
this.setState("starting");
|
|
10621
|
+
const socket = this.deps.socket();
|
|
10622
|
+
this.capture ??= (this.deps.createCapture ?? (() => new MicrophoneCapture()))();
|
|
10623
|
+
const capture = this.capture;
|
|
10624
|
+
this.captureOwner = session;
|
|
10625
|
+
const [acknowledged, captured] = await Promise.allSettled([
|
|
10626
|
+
socket.ensureConnected(conversationId).then(
|
|
10627
|
+
() => socket.request(SPEECH_EVENTS.start, {
|
|
10628
|
+
conversationId,
|
|
10629
|
+
requestId: session.requestId,
|
|
10630
|
+
parentStagingId: options.parentStagingId,
|
|
10631
|
+
language: options.language ?? null,
|
|
10632
|
+
send: options.send !== false,
|
|
10633
|
+
silent: options.silent === true,
|
|
10634
|
+
renderSessionId: this.deps.renderSessionId(),
|
|
10635
|
+
format: SPEECH_AUDIO_FORMAT
|
|
10636
|
+
})
|
|
10637
|
+
),
|
|
10638
|
+
capture.start((frame) => this.frame(session, frame))
|
|
10639
|
+
]);
|
|
10640
|
+
const ack = acknowledged.status === "fulfilled" ? acknowledged.value : null;
|
|
10641
|
+
const serverSession = ack?.ok ? ack.data.sessionId : null;
|
|
10642
|
+
if (session !== this.session) {
|
|
10643
|
+
this.dropCapture(session);
|
|
10644
|
+
if (serverSession) {
|
|
10645
|
+
socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
|
|
10646
|
+
}
|
|
10647
|
+
return session.done.promise;
|
|
10648
|
+
}
|
|
10649
|
+
let failure = null;
|
|
10650
|
+
if (captured.status === "rejected") {
|
|
10651
|
+
failure = speechError(captured.reason, "MICROPHONE_UNAVAILABLE");
|
|
10652
|
+
} else if (acknowledged.status === "rejected") {
|
|
10653
|
+
failure = speechError(acknowledged.reason, "SPEECH_UNAVAILABLE");
|
|
10654
|
+
} else if (!ack?.ok) {
|
|
10655
|
+
failure = new SpeechInputError(
|
|
10656
|
+
ack && !ack.ok ? ack.error : "SPEECH_UNAVAILABLE"
|
|
10657
|
+
);
|
|
10658
|
+
}
|
|
10659
|
+
if (failure) {
|
|
10660
|
+
this.dropCapture(session);
|
|
10661
|
+
if (serverSession) {
|
|
10662
|
+
socket.send(SPEECH_EVENTS.cancel, { sessionId: serverSession });
|
|
10663
|
+
}
|
|
10664
|
+
this.end(session);
|
|
10665
|
+
session.done.reject(failure);
|
|
10666
|
+
return session.done.promise;
|
|
10667
|
+
}
|
|
10668
|
+
session.sessionId = serverSession;
|
|
10669
|
+
for (const audio of session.pending.splice(0)) {
|
|
10670
|
+
this.sendFrame(session, audio);
|
|
10671
|
+
}
|
|
10672
|
+
if (session.stopRequested) {
|
|
10673
|
+
void this.stop().catch(() => void 0);
|
|
10674
|
+
return session.done.promise;
|
|
10675
|
+
}
|
|
10676
|
+
this.setState("listening");
|
|
10677
|
+
session.timer = setTimeout(() => {
|
|
10678
|
+
void this.stop().catch(() => void 0);
|
|
10679
|
+
}, maxDurationMs);
|
|
10680
|
+
return session.done.promise;
|
|
10681
|
+
}
|
|
10682
|
+
/**
|
|
10683
|
+
* Ends the recording and resolves with the result as soon as the text is
|
|
10684
|
+
* recognised. A stop before the microphone gave any audio (a permission
|
|
10685
|
+
* prompt, a slow device) cancels and resolves null.
|
|
10686
|
+
*/
|
|
10687
|
+
async stop() {
|
|
10688
|
+
const session = this.session;
|
|
10689
|
+
if (!session) {
|
|
10690
|
+
return null;
|
|
10691
|
+
}
|
|
10692
|
+
if (session.stopping) {
|
|
10693
|
+
return session.stopping;
|
|
10694
|
+
}
|
|
10695
|
+
if (!session.sessionId) {
|
|
10696
|
+
if (session.bytes === 0) {
|
|
10697
|
+
this.cancel();
|
|
10698
|
+
return null;
|
|
10699
|
+
}
|
|
10700
|
+
session.stopRequested = true;
|
|
10701
|
+
void this.closeCapture(session);
|
|
10702
|
+
return session.done.promise;
|
|
10703
|
+
}
|
|
10704
|
+
const stopping = deferred();
|
|
10705
|
+
session.stopping = stopping.promise;
|
|
10706
|
+
this.finish(session).then(stopping.resolve, stopping.reject);
|
|
10707
|
+
return session.stopping;
|
|
10708
|
+
}
|
|
10709
|
+
/** Discards the recording. Nothing is recognised or charged. */
|
|
10710
|
+
cancel() {
|
|
10711
|
+
const session = this.session;
|
|
10712
|
+
if (!session || session.stopping) {
|
|
10713
|
+
return;
|
|
10714
|
+
}
|
|
10715
|
+
this.dropCapture(session);
|
|
10716
|
+
if (session.sessionId) {
|
|
10717
|
+
this.deps.socket().send(SPEECH_EVENTS.cancel, { sessionId: session.sessionId });
|
|
10718
|
+
}
|
|
10719
|
+
this.end(session);
|
|
10720
|
+
session.done.resolve(null);
|
|
10721
|
+
}
|
|
10722
|
+
/** The conversation ended: no listening, and no turn waits any more. */
|
|
10723
|
+
reset() {
|
|
10724
|
+
this.cancel();
|
|
10725
|
+
for (const [requestId, turn] of this.turns) {
|
|
10726
|
+
clearTimeout(turn.timer);
|
|
10727
|
+
turn.done.reject(
|
|
10728
|
+
new SpeechInputError("CONVERSATION_ENDED", "The conversation ended.")
|
|
10729
|
+
);
|
|
10730
|
+
this.turns.delete(requestId);
|
|
10731
|
+
}
|
|
10732
|
+
}
|
|
10733
|
+
handleSocketEvent(event) {
|
|
10734
|
+
if (event.type === "partial") {
|
|
10735
|
+
if (this.session?.sessionId === event.event.sessionId) {
|
|
10736
|
+
this.deps.onTranscript({ text: event.event.text, final: false });
|
|
10737
|
+
}
|
|
10738
|
+
return;
|
|
10739
|
+
}
|
|
10740
|
+
if (event.type === "turn") {
|
|
10741
|
+
const turn = this.turns.get(event.requestId);
|
|
10742
|
+
if (!turn) {
|
|
10743
|
+
return;
|
|
10744
|
+
}
|
|
10745
|
+
this.turns.delete(event.requestId);
|
|
10746
|
+
clearTimeout(turn.timer);
|
|
10747
|
+
void this.deps.deliverTurn(event.turn, event.requestId, turn.options.silent === true).then(
|
|
10748
|
+
() => turn.done.resolve(event.turn),
|
|
10749
|
+
(error) => turn.done.reject(error)
|
|
10750
|
+
);
|
|
10751
|
+
return;
|
|
10752
|
+
}
|
|
10753
|
+
if (event.type === "turnFailed") {
|
|
10754
|
+
const turn = this.turns.get(event.event.requestId);
|
|
10755
|
+
if (!turn) {
|
|
10756
|
+
return;
|
|
10757
|
+
}
|
|
10758
|
+
this.turns.delete(event.event.requestId);
|
|
10759
|
+
clearTimeout(turn.timer);
|
|
10760
|
+
const reason = event.event.error;
|
|
10761
|
+
turn.done.reject(
|
|
10762
|
+
new SpeechInputError(CODE.test(reason) ? reason : "TURN_FAILED", reason)
|
|
10763
|
+
);
|
|
10764
|
+
return;
|
|
10765
|
+
}
|
|
10766
|
+
const session = this.session;
|
|
10767
|
+
if (session) {
|
|
10768
|
+
const failure = new SpeechInputError(
|
|
10769
|
+
"SPEECH_DISCONNECTED",
|
|
10770
|
+
session.stopping ? "The connection closed while the speech was recognised." : "The connection closed while listening."
|
|
10771
|
+
);
|
|
10772
|
+
session.disconnected.reject(failure);
|
|
10773
|
+
if (!session.stopping) {
|
|
10774
|
+
this.dropCapture(session);
|
|
10775
|
+
this.end(session);
|
|
10776
|
+
session.done.reject(failure);
|
|
10777
|
+
}
|
|
10778
|
+
}
|
|
10779
|
+
for (const requestId of [...this.turns.keys()]) {
|
|
10780
|
+
this.recover(requestId);
|
|
10781
|
+
}
|
|
10782
|
+
}
|
|
10783
|
+
frame(session, frame) {
|
|
10784
|
+
if (session !== this.session || session.captureClosed) {
|
|
10785
|
+
return;
|
|
10786
|
+
}
|
|
10787
|
+
const audio = new Uint8Array(
|
|
10788
|
+
frame.pcm.buffer,
|
|
10789
|
+
frame.pcm.byteOffset,
|
|
10790
|
+
frame.pcm.byteLength
|
|
10791
|
+
).slice();
|
|
10792
|
+
if (session.sessionId) {
|
|
10793
|
+
this.sendFrame(session, audio);
|
|
10794
|
+
} else if (session.bytes + audio.length <= SPEECH_MAX_AUDIO_BYTES) {
|
|
10795
|
+
session.bytes += audio.length;
|
|
10796
|
+
session.pending.push(audio);
|
|
10797
|
+
}
|
|
10798
|
+
if (!session.stopping && session.silence?.push(frame.level)) {
|
|
10799
|
+
void this.stop().catch(() => void 0);
|
|
10800
|
+
}
|
|
10801
|
+
}
|
|
10802
|
+
sendFrame(session, audio) {
|
|
10803
|
+
if (!session.sessionId) {
|
|
10804
|
+
return;
|
|
10805
|
+
}
|
|
10806
|
+
this.deps.socket().send(SPEECH_EVENTS.audio, {
|
|
10807
|
+
sessionId: session.sessionId,
|
|
10808
|
+
sequence: session.sequence,
|
|
10809
|
+
audio
|
|
10810
|
+
});
|
|
10811
|
+
session.sequence += 1;
|
|
10812
|
+
}
|
|
10813
|
+
async finish(session) {
|
|
10814
|
+
if (session.timer) {
|
|
10815
|
+
clearTimeout(session.timer);
|
|
10816
|
+
}
|
|
10817
|
+
this.setState("recognizing");
|
|
10818
|
+
try {
|
|
10819
|
+
await this.closeCapture(session);
|
|
10820
|
+
const ack = await Promise.race([
|
|
10821
|
+
this.deps.socket().request(
|
|
10822
|
+
SPEECH_EVENTS.stop,
|
|
10823
|
+
{ sessionId: session.sessionId },
|
|
10824
|
+
3e4
|
|
10825
|
+
),
|
|
10826
|
+
session.disconnected.promise
|
|
10827
|
+
]);
|
|
10828
|
+
if (!ack.ok) {
|
|
10829
|
+
throw new SpeechInputError(ack.error);
|
|
10830
|
+
}
|
|
10831
|
+
const result = this.result(session, ack.data);
|
|
10832
|
+
this.end(session);
|
|
10833
|
+
this.deps.onTranscript({ text: result.text, final: true });
|
|
10834
|
+
session.done.resolve(result);
|
|
10835
|
+
return result;
|
|
10836
|
+
} catch (error) {
|
|
10837
|
+
const failure = speechError(error, "SPEECH_RECOGNITION_FAILED");
|
|
10838
|
+
this.end(session);
|
|
10839
|
+
session.done.reject(failure);
|
|
10840
|
+
throw failure;
|
|
10841
|
+
}
|
|
10842
|
+
}
|
|
10843
|
+
result(session, transcript) {
|
|
10844
|
+
const text = transcript.text;
|
|
10845
|
+
const turn = transcript.sent ? this.track(session, text) : null;
|
|
10846
|
+
let sending = null;
|
|
10847
|
+
return {
|
|
10848
|
+
text,
|
|
10849
|
+
language: transcript.language,
|
|
10850
|
+
durationMs: transcript.durationMs,
|
|
10851
|
+
sent: transcript.sent,
|
|
10852
|
+
turn,
|
|
10853
|
+
send: () => {
|
|
10854
|
+
if (turn) {
|
|
10855
|
+
return turn;
|
|
10856
|
+
}
|
|
10857
|
+
if (!text) {
|
|
10858
|
+
return Promise.reject(
|
|
10859
|
+
new SpeechInputError("SPEECH_EMPTY", "No speech was recognised.")
|
|
10860
|
+
);
|
|
10861
|
+
}
|
|
10862
|
+
this.deps.assertVoiceAllowed();
|
|
10863
|
+
sending ??= this.deps.sendText(text, {
|
|
10864
|
+
requestId: session.requestId,
|
|
10865
|
+
parentStagingId: session.options.parentStagingId,
|
|
10866
|
+
silent: session.options.silent === true
|
|
10867
|
+
});
|
|
10868
|
+
return sending;
|
|
10869
|
+
}
|
|
10870
|
+
};
|
|
10871
|
+
}
|
|
10872
|
+
track(session, text) {
|
|
10873
|
+
const done = deferred();
|
|
10874
|
+
const timer = setTimeout(
|
|
10875
|
+
() => this.recover(session.requestId),
|
|
10876
|
+
this.deps.turnTimeoutMs ?? 12e4
|
|
10877
|
+
);
|
|
10878
|
+
this.turns.set(session.requestId, {
|
|
10879
|
+
text,
|
|
10880
|
+
options: session.options,
|
|
10881
|
+
timer,
|
|
10882
|
+
done
|
|
10883
|
+
});
|
|
10884
|
+
this.deps.beginTurn();
|
|
10885
|
+
return done.promise;
|
|
10886
|
+
}
|
|
10887
|
+
/** The idempotent HTTP request returns the turn that the server started. */
|
|
10888
|
+
recover(requestId) {
|
|
10889
|
+
const turn = this.turns.get(requestId);
|
|
10890
|
+
if (!turn) {
|
|
10891
|
+
return;
|
|
10892
|
+
}
|
|
10893
|
+
this.turns.delete(requestId);
|
|
10894
|
+
clearTimeout(turn.timer);
|
|
10895
|
+
this.deps.sendText(turn.text, {
|
|
10896
|
+
requestId,
|
|
10897
|
+
parentStagingId: turn.options.parentStagingId,
|
|
10898
|
+
silent: turn.options.silent === true
|
|
10899
|
+
}).then(turn.done.resolve, turn.done.reject);
|
|
10900
|
+
}
|
|
10901
|
+
/** Stops the capture of `session`; its last partial frame arrives first. */
|
|
10902
|
+
closeCapture(session) {
|
|
10903
|
+
session.captureClosing ??= (async () => {
|
|
10904
|
+
if (this.captureOwner === session) {
|
|
10905
|
+
this.captureOwner = null;
|
|
10906
|
+
await this.capture?.stop();
|
|
10907
|
+
}
|
|
10908
|
+
session.captureClosed = true;
|
|
10909
|
+
})();
|
|
10910
|
+
return session.captureClosing;
|
|
10911
|
+
}
|
|
10912
|
+
/** Ends the capture of `session` without its last frame, unless a later session owns it. */
|
|
10913
|
+
dropCapture(session) {
|
|
10914
|
+
if (this.captureOwner === session) {
|
|
10915
|
+
this.captureOwner = null;
|
|
10916
|
+
this.capture?.cancel();
|
|
10917
|
+
}
|
|
10918
|
+
}
|
|
10919
|
+
end(session) {
|
|
10920
|
+
if (session.timer) {
|
|
10921
|
+
clearTimeout(session.timer);
|
|
10922
|
+
session.timer = null;
|
|
10923
|
+
}
|
|
10924
|
+
if (this.session === session) {
|
|
10925
|
+
this.session = null;
|
|
10926
|
+
this.setState("idle");
|
|
10927
|
+
}
|
|
10928
|
+
}
|
|
10929
|
+
setState(state) {
|
|
10930
|
+
if (this.state !== state) {
|
|
10931
|
+
this.state = state;
|
|
10932
|
+
this.deps.onState(state);
|
|
10933
|
+
}
|
|
10934
|
+
}
|
|
10935
|
+
};
|
|
10936
|
+
|
|
10937
|
+
// ../../clients/web-client/src/npcify.ts
|
|
10938
|
+
var Character = class extends CharacterRuntime {
|
|
10939
|
+
conversationSocket;
|
|
10940
|
+
renderSessionId = null;
|
|
10941
|
+
initialAssetInfo;
|
|
10942
|
+
stagingResultCallback = null;
|
|
10943
|
+
thinkingCallback = null;
|
|
10944
|
+
notifiedStagingIds = /* @__PURE__ */ new Map();
|
|
10945
|
+
liveConversation;
|
|
10946
|
+
baked = null;
|
|
10947
|
+
authoring;
|
|
10948
|
+
normalizeIdleOrientation;
|
|
10949
|
+
bakedAsset;
|
|
10950
|
+
animationRefreshId = 0;
|
|
10951
|
+
voice;
|
|
10952
|
+
configuredInputModes;
|
|
10953
|
+
resolvedInputModes = null;
|
|
10954
|
+
latestOptions = [];
|
|
10955
|
+
listeningStateCallback = null;
|
|
10956
|
+
transcriptCallback = null;
|
|
10957
|
+
conversationId = null;
|
|
10958
|
+
constructor(options) {
|
|
10959
|
+
const resolvedToken = (options.apiToken ?? options.apiKey)?.trim() ?? "";
|
|
10960
|
+
const resolvedProfile = options.assetProfile ?? "original";
|
|
10961
|
+
super({
|
|
10962
|
+
...options,
|
|
10963
|
+
apiToken: resolvedToken,
|
|
10964
|
+
assetProfile: resolvedProfile,
|
|
10965
|
+
loadAnimation: (animationId) => loadVrma({
|
|
10966
|
+
baseUrl: options.baseUrl,
|
|
10967
|
+
characterId: options.characterId,
|
|
10968
|
+
apiToken: resolvedToken,
|
|
10969
|
+
animationId,
|
|
10970
|
+
assetProfile: resolvedProfile
|
|
10971
|
+
})
|
|
10972
|
+
});
|
|
10973
|
+
this.authoring = options.authoring ?? false;
|
|
10974
|
+
this.normalizeIdleOrientation = options.normalizeIdleOrientation ?? false;
|
|
10975
|
+
this.bakedAsset = options.bakedConversation;
|
|
10976
|
+
this.initialAssetInfo = options.initialAssetInfo ?? null;
|
|
10977
|
+
this.configuredInputModes = parseInputModes(options.inputModes);
|
|
10978
|
+
if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
|
|
10979
|
+
resolveInputModes(this.configuredInputModes, {
|
|
10980
|
+
allowCustomReply: false,
|
|
10981
|
+
baked: true
|
|
10982
|
+
});
|
|
10983
|
+
}
|
|
10984
|
+
this.conversationSocket = new ConversationSocket({
|
|
10985
|
+
baseUrl: this.baseUrl,
|
|
10986
|
+
apiToken: this.apiToken,
|
|
10987
|
+
getConversationId: () => this.conversationId,
|
|
10988
|
+
handleStreamEvent: (event) => {
|
|
10989
|
+
if (event.type !== "text_delta") {
|
|
10990
|
+
this.thinkingState = false;
|
|
10991
|
+
}
|
|
10992
|
+
this.streamPlayback.handleEvent(event);
|
|
10993
|
+
},
|
|
10994
|
+
handleTurn: async (turn) => {
|
|
10995
|
+
await this.handleStagingResult(turn, { source: "turn" });
|
|
10996
|
+
},
|
|
10997
|
+
handleCallbackError: this.handleCallbackError,
|
|
10998
|
+
handleSpeechEvent: (event) => this.voice.handleSocketEvent(event)
|
|
10999
|
+
});
|
|
11000
|
+
this.voice = new VoiceInput({
|
|
11001
|
+
socket: () => this.conversationSocket,
|
|
11002
|
+
conversationId: () => this.conversationId,
|
|
11003
|
+
renderSessionId: () => this.renderSessionId,
|
|
11004
|
+
assertVoiceAllowed: () => this.assertInputMode("voice"),
|
|
11005
|
+
turnInFlight: () => this.liveConversation.inFlight,
|
|
11006
|
+
beginTurn: () => {
|
|
11007
|
+
if (this.conversationId) {
|
|
11008
|
+
this.telemetry.begin(this.conversationId);
|
|
11009
|
+
}
|
|
11010
|
+
this.invokeThinkingCallback({ taskId: null, stagingId: null });
|
|
11011
|
+
},
|
|
11012
|
+
deliverTurn: (payload, requestId, silent) => this.handleStagingResult(payload, {
|
|
11013
|
+
source: "api",
|
|
11014
|
+
forcePlayback: payload.taskId !== requestId,
|
|
11015
|
+
silent
|
|
11016
|
+
}),
|
|
11017
|
+
sendText: (text, turnOptions) => {
|
|
11018
|
+
if (!this.conversationId) {
|
|
11019
|
+
return Promise.reject(new Error("Conversation ID is required."));
|
|
11020
|
+
}
|
|
11021
|
+
return this.liveConversation.send(this.conversationId, text, turnOptions);
|
|
11022
|
+
},
|
|
11023
|
+
onState: (state) => {
|
|
11024
|
+
if (this.listeningStateCallback) {
|
|
11025
|
+
invokeSafeCallback(
|
|
11026
|
+
"onListeningState",
|
|
11027
|
+
this.handleCallbackError,
|
|
11028
|
+
() => this.listeningStateCallback?.(state)
|
|
11029
|
+
);
|
|
11030
|
+
}
|
|
11031
|
+
},
|
|
11032
|
+
onTranscript: (event) => {
|
|
11033
|
+
if (this.transcriptCallback) {
|
|
11034
|
+
invokeSafeCallback(
|
|
11035
|
+
"onTranscript",
|
|
11036
|
+
this.handleCallbackError,
|
|
11037
|
+
() => this.transcriptCallback?.(event)
|
|
11038
|
+
);
|
|
11039
|
+
}
|
|
11040
|
+
}
|
|
11041
|
+
});
|
|
11042
|
+
this.liveConversation = new LiveConversation({
|
|
11043
|
+
baseUrl: this.baseUrl,
|
|
11044
|
+
apiToken: this.apiToken,
|
|
11045
|
+
telemetry: this.telemetry,
|
|
11046
|
+
socket: () => this.conversationSocket,
|
|
11047
|
+
renderSession: () => this.renderSessionId,
|
|
9315
11048
|
thinking: () => this.invokeThinkingCallback({ taskId: null, stagingId: null }),
|
|
9316
11049
|
handleResult: (payload, forcePlayback, silent) => this.handleStagingResult(payload, {
|
|
9317
11050
|
source: "api",
|
|
@@ -9331,7 +11064,12 @@ var Character = class extends CharacterRuntime {
|
|
|
9331
11064
|
this.resetAssets();
|
|
9332
11065
|
const progressReporter = createAssetProgressReporter(callback);
|
|
9333
11066
|
const info = await this.getAssetInfo();
|
|
9334
|
-
|
|
11067
|
+
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11068
|
+
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11069
|
+
allowCustomReply: info.allowCustomReply !== false,
|
|
11070
|
+
baked
|
|
11071
|
+
});
|
|
11072
|
+
this.baked = baked ? await loadBakedConversation({
|
|
9335
11073
|
baseUrl: this.baseUrl,
|
|
9336
11074
|
characterId: this.characterId,
|
|
9337
11075
|
apiToken: this.apiToken,
|
|
@@ -9356,7 +11094,8 @@ var Character = class extends CharacterRuntime {
|
|
|
9356
11094
|
}
|
|
9357
11095
|
const animationController = new AnimationController(
|
|
9358
11096
|
this.vrm,
|
|
9359
|
-
info.idleAnimationId ?? null
|
|
11097
|
+
info.idleAnimationId ?? null,
|
|
11098
|
+
{ normalizeIdleOrientation: this.normalizeIdleOrientation }
|
|
9360
11099
|
);
|
|
9361
11100
|
animationController.prepareAnimations = (ids) => this.lazyAnimations.ensureMany(ids);
|
|
9362
11101
|
this.animationController = animationController;
|
|
@@ -9399,6 +11138,10 @@ var Character = class extends CharacterRuntime {
|
|
|
9399
11138
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
9400
11139
|
throw new Error("Baked conversation requires local rendering.");
|
|
9401
11140
|
}
|
|
11141
|
+
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11142
|
+
allowCustomReply: info.allowCustomReply !== false,
|
|
11143
|
+
baked: false
|
|
11144
|
+
});
|
|
9402
11145
|
this.animationController = new RemoteAnimationController();
|
|
9403
11146
|
}
|
|
9404
11147
|
async createConversation(options) {
|
|
@@ -9441,6 +11184,7 @@ var Character = class extends CharacterRuntime {
|
|
|
9441
11184
|
return targetConversationId;
|
|
9442
11185
|
}
|
|
9443
11186
|
async disconnect() {
|
|
11187
|
+
this.voice.reset();
|
|
9444
11188
|
this.thinkingState = false;
|
|
9445
11189
|
if (this.animationController) {
|
|
9446
11190
|
this.streamPlayback.cleanup();
|
|
@@ -9476,6 +11220,10 @@ var Character = class extends CharacterRuntime {
|
|
|
9476
11220
|
maybeMessageOrOptions,
|
|
9477
11221
|
maybeOptions
|
|
9478
11222
|
);
|
|
11223
|
+
this.assertReplyMode(message);
|
|
11224
|
+
if (this.voice.turnPending) {
|
|
11225
|
+
throw new Error("A reply is already being generated. Please wait.");
|
|
11226
|
+
}
|
|
9479
11227
|
if (this.baked) {
|
|
9480
11228
|
const result = this.baked.reply(message, options?.parentStagingId);
|
|
9481
11229
|
if (result.ended) {
|
|
@@ -9515,6 +11263,55 @@ var Character = class extends CharacterRuntime {
|
|
|
9515
11263
|
get isBakedConversation() {
|
|
9516
11264
|
return this.baked !== null;
|
|
9517
11265
|
}
|
|
11266
|
+
/**
|
|
11267
|
+
* The input modes in use: the configured modes, or every mode that the
|
|
11268
|
+
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11269
|
+
*/
|
|
11270
|
+
get inputModes() {
|
|
11271
|
+
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
|
|
11272
|
+
}
|
|
11273
|
+
get listeningState() {
|
|
11274
|
+
return this.voice.listeningState;
|
|
11275
|
+
}
|
|
11276
|
+
/**
|
|
11277
|
+
* Opens the microphone and streams the speech to the server. Resolves with
|
|
11278
|
+
* the result when listening ends (`stopListening`, `endOnSilenceMs` or
|
|
11279
|
+
* `maxDurationMs`), or with null when it is cancelled. The first call asks
|
|
11280
|
+
* for the microphone permission.
|
|
11281
|
+
*/
|
|
11282
|
+
startListening(options) {
|
|
11283
|
+
return this.voice.start(options);
|
|
11284
|
+
}
|
|
11285
|
+
/**
|
|
11286
|
+
* Ends the recording; resolves as soon as the server recognised the text.
|
|
11287
|
+
* Before the microphone gave any audio, it cancels and resolves null.
|
|
11288
|
+
*/
|
|
11289
|
+
stopListening() {
|
|
11290
|
+
return this.voice.stop();
|
|
11291
|
+
}
|
|
11292
|
+
/** Discards the recording. Nothing is recognised or charged. */
|
|
11293
|
+
cancelListening() {
|
|
11294
|
+
this.voice.cancel();
|
|
11295
|
+
}
|
|
11296
|
+
/** Press and hold an element or a key (default Space) to talk. Returns an unbind function. */
|
|
11297
|
+
bindPushToTalk(options) {
|
|
11298
|
+
return bindPushToTalk(this, options);
|
|
11299
|
+
}
|
|
11300
|
+
onListeningState(callback) {
|
|
11301
|
+
return bindCallbackSlot(
|
|
11302
|
+
() => this.listeningStateCallback,
|
|
11303
|
+
(next) => this.listeningStateCallback = next,
|
|
11304
|
+
callback
|
|
11305
|
+
);
|
|
11306
|
+
}
|
|
11307
|
+
/** Partial text while the player speaks (`final: false`), then the final text. */
|
|
11308
|
+
onTranscript(callback) {
|
|
11309
|
+
return bindCallbackSlot(
|
|
11310
|
+
() => this.transcriptCallback,
|
|
11311
|
+
(next) => this.transcriptCallback = next,
|
|
11312
|
+
callback
|
|
11313
|
+
);
|
|
11314
|
+
}
|
|
9518
11315
|
async getConversationWorldline(conversationId = this.conversationId) {
|
|
9519
11316
|
if (!conversationId) {
|
|
9520
11317
|
throw new Error("Conversation ID is required.");
|
|
@@ -9538,6 +11335,13 @@ var Character = class extends CharacterRuntime {
|
|
|
9538
11335
|
timeline: options?.silent ? toSilentTimeline(normalized.timeline) : normalized.timeline
|
|
9539
11336
|
});
|
|
9540
11337
|
}
|
|
11338
|
+
/**
|
|
11339
|
+
* Plays one live staging event from another transport, such as an actor
|
|
11340
|
+
* stage. A timeline event returns its playback result.
|
|
11341
|
+
*/
|
|
11342
|
+
playStreamEvent(event) {
|
|
11343
|
+
return this.streamPlayback.handleEvent(event);
|
|
11344
|
+
}
|
|
9541
11345
|
setRenderSessionId(renderSessionId) {
|
|
9542
11346
|
this.renderSessionId = renderSessionId?.trim() || null;
|
|
9543
11347
|
}
|
|
@@ -9571,6 +11375,7 @@ var Character = class extends CharacterRuntime {
|
|
|
9571
11375
|
async handleStagingResult(payload, options) {
|
|
9572
11376
|
this.thinkingState = false;
|
|
9573
11377
|
const normalized = normalizeStagingResultPayload(payload);
|
|
11378
|
+
this.latestOptions = normalized.options;
|
|
9574
11379
|
if (options.source === "turn") {
|
|
9575
11380
|
this.liveConversation.received(normalized);
|
|
9576
11381
|
}
|
|
@@ -9597,6 +11402,44 @@ var Character = class extends CharacterRuntime {
|
|
|
9597
11402
|
void this.streamPlayback.playPayload(normalized);
|
|
9598
11403
|
}
|
|
9599
11404
|
}
|
|
11405
|
+
/** Throws when the input modes do not include `mode`. */
|
|
11406
|
+
assertInputMode(mode) {
|
|
11407
|
+
if (mode === "voice" && this.baked) {
|
|
11408
|
+
throw new InputModeError(
|
|
11409
|
+
"Voice input is not available in a baked (offline) conversation."
|
|
11410
|
+
);
|
|
11411
|
+
}
|
|
11412
|
+
const modes = this.inputModes;
|
|
11413
|
+
if (!modes.includes(mode)) {
|
|
11414
|
+
throw new InputModeError(
|
|
11415
|
+
`The "${mode}" input mode is off: inputModes is ${JSON.stringify(modes)}.`
|
|
11416
|
+
);
|
|
11417
|
+
}
|
|
11418
|
+
}
|
|
11419
|
+
/**
|
|
11420
|
+
* A typed reply needs the `text` mode; one of the current reply options
|
|
11421
|
+
* needs `options` or `text`. Without a configuration the server's rules
|
|
11422
|
+
* apply alone, as before.
|
|
11423
|
+
*/
|
|
11424
|
+
assertReplyMode(message) {
|
|
11425
|
+
if (!this.configuredInputModes) {
|
|
11426
|
+
return;
|
|
11427
|
+
}
|
|
11428
|
+
const modes = this.inputModes;
|
|
11429
|
+
if (modes.includes("text")) {
|
|
11430
|
+
return;
|
|
11431
|
+
}
|
|
11432
|
+
const reply = message.trim();
|
|
11433
|
+
const isOption = this.latestOptions.some(
|
|
11434
|
+
(option) => option.label.trim() === reply
|
|
11435
|
+
);
|
|
11436
|
+
if (isOption && modes.includes("options")) {
|
|
11437
|
+
return;
|
|
11438
|
+
}
|
|
11439
|
+
throw new InputModeError(
|
|
11440
|
+
isOption ? `Reply options are off: inputModes is ${JSON.stringify(modes)}.` : `Typed replies are off: inputModes is ${JSON.stringify(modes)}. Send one of the reply options or use startListening().`
|
|
11441
|
+
);
|
|
11442
|
+
}
|
|
9600
11443
|
invokeThinkingCallback(payload) {
|
|
9601
11444
|
this.thinkingState = true;
|
|
9602
11445
|
if (this.thinkingCallback) {
|
|
@@ -9609,6 +11452,12 @@ var Character = class extends CharacterRuntime {
|
|
|
9609
11452
|
}
|
|
9610
11453
|
};
|
|
9611
11454
|
var Npcify = class extends Character {
|
|
11455
|
+
constructor(options) {
|
|
11456
|
+
super({
|
|
11457
|
+
...options,
|
|
11458
|
+
normalizeIdleOrientation: options.normalizeIdleOrientation ?? true
|
|
11459
|
+
});
|
|
11460
|
+
}
|
|
9612
11461
|
};
|
|
9613
11462
|
|
|
9614
11463
|
// ../../clients/web-client/src/character/character.ts
|
|
@@ -10189,13 +12038,18 @@ export {
|
|
|
10189
12038
|
CharacterSceneRig,
|
|
10190
12039
|
FacialPerformanceLayer,
|
|
10191
12040
|
FacialPerformanceMixer,
|
|
12041
|
+
INPUT_MODES,
|
|
12042
|
+
InputModeError,
|
|
12043
|
+
MicrophoneError,
|
|
10192
12044
|
Npcify,
|
|
10193
12045
|
Character as NpcifyCharacter,
|
|
10194
12046
|
PERFORMANCE_FILE_EXTENSION,
|
|
10195
12047
|
PERFORMANCE_MEDIA_TYPE,
|
|
10196
12048
|
PerformanceRecordingSchemaVersion,
|
|
10197
12049
|
RemoteAnimationController,
|
|
12050
|
+
SpeechInputError,
|
|
10198
12051
|
analyzePerformanceRecording,
|
|
12052
|
+
bindPushToTalk,
|
|
10199
12053
|
createCharacter,
|
|
10200
12054
|
decodePerformance,
|
|
10201
12055
|
encodePerformance,
|