@npcify/web 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -10
- package/dist/index.d.ts +11 -3
- package/dist/index.js +71 -32
- package/examples/main.js +15 -12
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -193,10 +193,12 @@ must remain private. Calls made with a key are billed to its owner.
|
|
|
193
193
|
|
|
194
194
|
## Voice input and input modes
|
|
195
195
|
|
|
196
|
-
The player can speak a reply
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
196
|
+
The player can speak a reply when the character owner turned voice input on
|
|
197
|
+
(the STT tab of the character in the dashboard, where the owner also chooses
|
|
198
|
+
the speech recognition provider). The SDK records the microphone and streams
|
|
199
|
+
the audio to the Npcify server. At the release, the server recognises the
|
|
200
|
+
speech and starts the turn. The recognised text is the player's message, as a
|
|
201
|
+
typed reply.
|
|
200
202
|
|
|
201
203
|
```js
|
|
202
204
|
// Hold the button, or Space, to talk. The release sends the speech.
|
|
@@ -229,7 +231,7 @@ character.onListeningState((state) => {
|
|
|
229
231
|
`maxDurationMs` (30 s at most) always stops. `stopListening()` resolves with
|
|
230
232
|
`{ text, language, durationMs, sent, turn, send() }`, or with null when the
|
|
231
233
|
microphone gave no audio yet. `cancelListening()` discards the recording;
|
|
232
|
-
|
|
234
|
+
the owner pays only the audio that already went to a streaming provider.
|
|
233
235
|
- `onTranscript(({ text, final }) => …)` gives the final text, and partial
|
|
234
236
|
text when the server's speech model gives it.
|
|
235
237
|
- The first press or `startListening()` asks for the microphone permission.
|
|
@@ -239,14 +241,16 @@ character.onListeningState((state) => {
|
|
|
239
241
|
- `inputModes` (an option of `mountNpcify`, `loadCharacter` and `Npcify`) is
|
|
240
242
|
an array of `'options'`, `'text'` and `'voice'`; one mode forces it. Without
|
|
241
243
|
it, every mode that the character permits is available. A baked
|
|
242
|
-
conversation permits only `['options']`,
|
|
243
|
-
replies permits only `'options'
|
|
244
|
-
`
|
|
244
|
+
conversation permits only `['options']`, a character without custom
|
|
245
|
+
replies permits only `'options'`, and `'voice'` needs the character's voice
|
|
246
|
+
input (`speechInputEnabled` in the character info). Another configuration
|
|
247
|
+
throws an `InputModeError` when the character loads, and listening on a
|
|
248
|
+
character with voice input off throws one too. With `inputModes` set,
|
|
245
249
|
`sendMessage` with typed text needs `'text'`, a reply option needs
|
|
246
250
|
`'options'` or `'text'`, and listening needs `'voice'`.
|
|
247
251
|
`character.inputModes` gives the modes in use.
|
|
248
|
-
- The character owner pays the recognition at the
|
|
249
|
-
the turn's language model.
|
|
252
|
+
- The character owner pays the recognition at the price of the provider that
|
|
253
|
+
the owner chose, as for the turn's language model.
|
|
250
254
|
|
|
251
255
|
## Runtime and lifecycle
|
|
252
256
|
|
package/dist/index.d.ts
CHANGED
|
@@ -43,6 +43,11 @@ interface AvatarPublicInfo {
|
|
|
43
43
|
bakedConversationEnabled?: boolean;
|
|
44
44
|
bakedConversationRevision?: number;
|
|
45
45
|
allowCustomReply?: boolean;
|
|
46
|
+
/**
|
|
47
|
+
* The owner turned voice input on and chose a provider. Voice also needs
|
|
48
|
+
* `allowCustomReply`. Missing on older servers: off.
|
|
49
|
+
*/
|
|
50
|
+
speechInputEnabled?: boolean;
|
|
46
51
|
autoPlay: boolean;
|
|
47
52
|
serverRenderingEnabled: boolean;
|
|
48
53
|
cameraSqt?: Sqt | null;
|
|
@@ -336,7 +341,7 @@ interface ConversationTurnOptions {
|
|
|
336
341
|
requestId?: string;
|
|
337
342
|
}
|
|
338
343
|
|
|
339
|
-
type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply'>>;
|
|
344
|
+
type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply' | 'speechInputEnabled'>>;
|
|
340
345
|
|
|
341
346
|
interface LlmThinkingPayload {
|
|
342
347
|
taskId?: string | null;
|
|
@@ -1150,8 +1155,9 @@ interface NpcifyOptions {
|
|
|
1150
1155
|
* How the player can reply: `'options'`, `'text'` and `'voice'`, in any
|
|
1151
1156
|
* combination; one mode forces it. Without it, every mode that the
|
|
1152
1157
|
* character permits. A baked conversation permits only `['options']`; a
|
|
1153
|
-
* character without custom replies permits only `options
|
|
1154
|
-
* character
|
|
1158
|
+
* character without custom replies permits only `options`; `voice` needs
|
|
1159
|
+
* the character's voice input (`speechInputEnabled`, the STT tab). A mode
|
|
1160
|
+
* that the character does not permit throws an `InputModeError` on load.
|
|
1155
1161
|
*/
|
|
1156
1162
|
inputModes?: InputMode[];
|
|
1157
1163
|
/** Optional error callback. */
|
|
@@ -1537,6 +1543,8 @@ declare class Character extends CharacterRuntime {
|
|
|
1537
1543
|
private readonly voice;
|
|
1538
1544
|
private readonly configuredInputModes;
|
|
1539
1545
|
private resolvedInputModes;
|
|
1546
|
+
/** The character's voice input switch, after the character loads. */
|
|
1547
|
+
private speechInput;
|
|
1540
1548
|
private latestOptions;
|
|
1541
1549
|
private listeningStateCallback;
|
|
1542
1550
|
private transcriptCallback;
|
package/dist/index.js
CHANGED
|
@@ -14,10 +14,6 @@ var GPT_IMAGE_MODELS = [
|
|
|
14
14
|
"gpt-image-2.5-flare",
|
|
15
15
|
"gpt-image-2.5-sunburst"
|
|
16
16
|
];
|
|
17
|
-
var GEMINI_SPEECH_MODEL_VENDORS = {
|
|
18
|
-
"gemini-3.5-transcribe-live": "gemini",
|
|
19
|
-
"gemini-3.5-transcribe": "gemini"
|
|
20
|
-
};
|
|
21
17
|
var AI_ROLES = {
|
|
22
18
|
character_draft: {
|
|
23
19
|
name: "Character draft",
|
|
@@ -59,28 +55,6 @@ var AI_ROLES = {
|
|
|
59
55
|
models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
|
|
60
56
|
options: []
|
|
61
57
|
},
|
|
62
|
-
// Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
|
|
63
|
-
// Audio that arrives while the player speaks. The model selects the vendor.
|
|
64
|
-
speech_input_live: {
|
|
65
|
-
name: "Speech input (live)",
|
|
66
|
-
source: "openai",
|
|
67
|
-
models: [
|
|
68
|
-
"gpt-transcribe",
|
|
69
|
-
"gpt-live-transcribe",
|
|
70
|
-
"gemini-3.5-transcribe-live",
|
|
71
|
-
"gemini-3.5-transcribe"
|
|
72
|
-
],
|
|
73
|
-
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
74
|
-
options: []
|
|
75
|
-
},
|
|
76
|
-
// One complete recording (an upload of an engine SDK).
|
|
77
|
-
speech_input_file: {
|
|
78
|
-
name: "Speech input (file)",
|
|
79
|
-
source: "openai",
|
|
80
|
-
models: ["gpt-transcribe", "gemini-3.5-transcribe"],
|
|
81
|
-
modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
|
|
82
|
-
options: []
|
|
83
|
-
},
|
|
84
58
|
// Priced by OPENAI_PRICES.text of the generator.
|
|
85
59
|
generator_agent: {
|
|
86
60
|
name: "Generator agent",
|
|
@@ -116,7 +90,7 @@ var permittedInputModes = function(permissions) {
|
|
|
116
90
|
if (permissions.baked || !permissions.allowCustomReply) {
|
|
117
91
|
return ["options"];
|
|
118
92
|
}
|
|
119
|
-
return ["options", "text", "voice"];
|
|
93
|
+
return permissions.speechInput ? ["options", "text", "voice"] : ["options", "text"];
|
|
120
94
|
};
|
|
121
95
|
var isInputMode = function(value) {
|
|
122
96
|
return INPUT_MODES.includes(value);
|
|
@@ -159,8 +133,9 @@ var resolveInputModes = function(configured, permissions) {
|
|
|
159
133
|
}
|
|
160
134
|
const refused = configured.filter((mode) => !permitted.includes(mode));
|
|
161
135
|
if (refused.length > 0) {
|
|
136
|
+
const names = refused.map((mode) => `"${mode}"`).join(" and ");
|
|
162
137
|
throw new InputModeError(
|
|
163
|
-
`This character accepts only its reply options (Allow custom replies is off). Remove ${
|
|
138
|
+
permissions.allowCustomReply ? `Voice input is off for this character (its owner turns it on in the STT tab). Remove ${names} from inputModes.` : `This character accepts only its reply options (Allow custom replies is off). Remove ${names} from inputModes.`
|
|
164
139
|
);
|
|
165
140
|
}
|
|
166
141
|
return [...configured];
|
|
@@ -186,6 +161,54 @@ var SPEECH_EVENTS = Object.freeze({
|
|
|
186
161
|
turnFailed: "speech:turn_failed"
|
|
187
162
|
});
|
|
188
163
|
|
|
164
|
+
// ../ts-shared/src/speech-input-providers.ts
|
|
165
|
+
var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
|
|
166
|
+
// The file model also streams: 120 ms slower than gpt-live-transcribe,
|
|
167
|
+
// at a 3.8 times lower price and with fewer errors.
|
|
168
|
+
openai: {
|
|
169
|
+
live: {
|
|
170
|
+
model: "gpt-transcribe",
|
|
171
|
+
releaseToTextMs: 783,
|
|
172
|
+
usdPerMinute: 45e-4,
|
|
173
|
+
approximate: false
|
|
174
|
+
},
|
|
175
|
+
file: {
|
|
176
|
+
model: "gpt-transcribe",
|
|
177
|
+
releaseToTextMs: 867,
|
|
178
|
+
usdPerMinute: 45e-4,
|
|
179
|
+
approximate: false
|
|
180
|
+
}
|
|
181
|
+
},
|
|
182
|
+
gemini: {
|
|
183
|
+
live: {
|
|
184
|
+
model: "gemini-3.5-transcribe-live",
|
|
185
|
+
releaseToTextMs: 266,
|
|
186
|
+
usdPerMinute: 9e-3,
|
|
187
|
+
approximate: true
|
|
188
|
+
},
|
|
189
|
+
file: {
|
|
190
|
+
model: "gemini-3.5-transcribe",
|
|
191
|
+
releaseToTextMs: 1263,
|
|
192
|
+
usdPerMinute: 5e-3,
|
|
193
|
+
approximate: true
|
|
194
|
+
}
|
|
195
|
+
},
|
|
196
|
+
elevenlabs: {
|
|
197
|
+
live: {
|
|
198
|
+
model: "scribe_v2_realtime",
|
|
199
|
+
releaseToTextMs: 208,
|
|
200
|
+
usdPerMinute: 0.39 / 60,
|
|
201
|
+
approximate: false
|
|
202
|
+
},
|
|
203
|
+
file: {
|
|
204
|
+
model: "scribe_v2",
|
|
205
|
+
releaseToTextMs: 925,
|
|
206
|
+
usdPerMinute: 0.22 / 60,
|
|
207
|
+
approximate: false
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
|
|
189
212
|
// ../ts-shared/src/product-usage.ts
|
|
190
213
|
var productUsageEvents = [
|
|
191
214
|
"page_viewed",
|
|
@@ -10951,6 +10974,8 @@ var Character = class extends CharacterRuntime {
|
|
|
10951
10974
|
voice;
|
|
10952
10975
|
configuredInputModes;
|
|
10953
10976
|
resolvedInputModes = null;
|
|
10977
|
+
/** The character's voice input switch, after the character loads. */
|
|
10978
|
+
speechInput = null;
|
|
10954
10979
|
latestOptions = [];
|
|
10955
10980
|
listeningStateCallback = null;
|
|
10956
10981
|
transcriptCallback = null;
|
|
@@ -10978,7 +11003,8 @@ var Character = class extends CharacterRuntime {
|
|
|
10978
11003
|
if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
|
|
10979
11004
|
resolveInputModes(this.configuredInputModes, {
|
|
10980
11005
|
allowCustomReply: false,
|
|
10981
|
-
baked: true
|
|
11006
|
+
baked: true,
|
|
11007
|
+
speechInput: false
|
|
10982
11008
|
});
|
|
10983
11009
|
}
|
|
10984
11010
|
this.conversationSocket = new ConversationSocket({
|
|
@@ -11065,9 +11091,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11065
11091
|
const progressReporter = createAssetProgressReporter(callback);
|
|
11066
11092
|
const info = await this.getAssetInfo();
|
|
11067
11093
|
const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
|
|
11094
|
+
this.speechInput = info.speechInputEnabled === true;
|
|
11068
11095
|
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11069
11096
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11070
|
-
baked
|
|
11097
|
+
baked,
|
|
11098
|
+
speechInput: this.speechInput
|
|
11071
11099
|
});
|
|
11072
11100
|
this.baked = baked ? await loadBakedConversation({
|
|
11073
11101
|
baseUrl: this.baseUrl,
|
|
@@ -11138,9 +11166,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11138
11166
|
if (info.bakedConversationEnabled || this.bakedAsset) {
|
|
11139
11167
|
throw new Error("Baked conversation requires local rendering.");
|
|
11140
11168
|
}
|
|
11169
|
+
this.speechInput = info.speechInputEnabled === true;
|
|
11141
11170
|
this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
|
|
11142
11171
|
allowCustomReply: info.allowCustomReply !== false,
|
|
11143
|
-
baked: false
|
|
11172
|
+
baked: false,
|
|
11173
|
+
speechInput: this.speechInput
|
|
11144
11174
|
});
|
|
11145
11175
|
this.animationController = new RemoteAnimationController();
|
|
11146
11176
|
}
|
|
@@ -11268,7 +11298,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11268
11298
|
* character permits. Before `loadAssets`, the configuration or all modes.
|
|
11269
11299
|
*/
|
|
11270
11300
|
get inputModes() {
|
|
11271
|
-
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
|
|
11301
|
+
return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
|
|
11302
|
+
allowCustomReply: true,
|
|
11303
|
+
baked: false,
|
|
11304
|
+
speechInput: true
|
|
11305
|
+
});
|
|
11272
11306
|
}
|
|
11273
11307
|
get listeningState() {
|
|
11274
11308
|
return this.voice.listeningState;
|
|
@@ -11409,6 +11443,11 @@ var Character = class extends CharacterRuntime {
|
|
|
11409
11443
|
"Voice input is not available in a baked (offline) conversation."
|
|
11410
11444
|
);
|
|
11411
11445
|
}
|
|
11446
|
+
if (mode === "voice" && this.speechInput === false) {
|
|
11447
|
+
throw new InputModeError(
|
|
11448
|
+
"Voice input is off for this character. Its owner turns it on in the STT tab."
|
|
11449
|
+
);
|
|
11450
|
+
}
|
|
11412
11451
|
const modes = this.inputModes;
|
|
11413
11452
|
if (!modes.includes(mode)) {
|
|
11414
11453
|
throw new InputModeError(
|
package/examples/main.js
CHANGED
|
@@ -35,19 +35,22 @@ character.onOptionsReady(({ options }) => {
|
|
|
35
35
|
}
|
|
36
36
|
talk.disabled = false
|
|
37
37
|
})
|
|
38
|
-
// Hold the button, or Space, to talk. The release sends the speech.
|
|
38
|
+
// Hold the button, or Space, to talk. The release sends the speech. Voice
|
|
39
|
+
// needs the character's voice input (its owner turns it on in the STT tab).
|
|
39
40
|
talk.hidden = !character.inputModes.includes('voice')
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
41
|
+
if (!talk.hidden) {
|
|
42
|
+
character.bindPushToTalk({
|
|
43
|
+
element: talk,
|
|
44
|
+
enabled: () => !talk.disabled,
|
|
45
|
+
onResult: ({ sent }) => {
|
|
46
|
+
if (sent) {
|
|
47
|
+
replies.replaceChildren()
|
|
48
|
+
talk.disabled = true
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
onError: report,
|
|
52
|
+
})
|
|
53
|
+
}
|
|
51
54
|
character.onListeningState((state) => {
|
|
52
55
|
talk.textContent =
|
|
53
56
|
{ listening: 'Listening…', recognizing: 'Recognizing…' }[state] ?? 'Hold to talk'
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@npcify/web",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -35,8 +35,8 @@
|
|
|
35
35
|
"tsup": "^8.5.0",
|
|
36
36
|
"typescript": "^5.9.3",
|
|
37
37
|
"@npcify/shared": "^0.1.0",
|
|
38
|
-
"@npcify/
|
|
39
|
-
"@npcify/
|
|
38
|
+
"@npcify/web-client": "^0.0.0",
|
|
39
|
+
"@npcify/vrm-renderer": "^0.1.0"
|
|
40
40
|
},
|
|
41
41
|
"repository": {
|
|
42
42
|
"type": "git",
|