@npcify/web 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -193,10 +193,12 @@ must remain private. Calls made with a key are billed to its owner.
193
193
 
194
194
  ## Voice input and input modes
195
195
 
196
- The player can speak a reply. The SDK records the microphone and streams the
197
- audio to the Npcify server. At the release, the server recognises the speech
198
- and starts the turn. The recognised text is the player's message, as a typed
199
- reply.
196
+ The player can speak a reply when the character owner turned voice input on
197
+ (the STT tab of the character in the dashboard, where the owner also chooses
198
+ the speech recognition provider). The SDK records the microphone and streams
199
+ the audio to the Npcify server. At the release, the server recognises the
200
+ speech and starts the turn. The recognised text is the player's message, as a
201
+ typed reply.
200
202
 
201
203
  ```js
202
204
  // Hold the button, or Space, to talk. The release sends the speech.
@@ -229,7 +231,7 @@ character.onListeningState((state) => {
229
231
  `maxDurationMs` (30 s at most) always stops. `stopListening()` resolves with
230
232
  `{ text, language, durationMs, sent, turn, send() }`, or with null when the
231
233
  microphone gave no audio yet. `cancelListening()` discards the recording;
232
- nothing is charged.
234
+ the owner pays only the audio that already went to a streaming provider.
233
235
  - `onTranscript(({ text, final }) => …)` gives the final text, and partial
234
236
  text when the server's speech model gives it.
235
237
  - The first press or `startListening()` asks for the microphone permission.
@@ -239,14 +241,16 @@ character.onListeningState((state) => {
239
241
  - `inputModes` (an option of `mountNpcify`, `loadCharacter` and `Npcify`) is
240
242
  an array of `'options'`, `'text'` and `'voice'`; one mode forces it. Without
241
243
  it, every mode that the character permits is available. A baked
242
- conversation permits only `['options']`, and a character without custom
243
- replies permits only `'options'`. Another configuration throws an
244
- `InputModeError` when the character loads. With `inputModes` set,
244
+ conversation permits only `['options']`, a character without custom
245
+ replies permits only `'options'`, and `'voice'` needs the character's voice
246
+ input (`speechInputEnabled` in the character info). Another configuration
247
+ throws an `InputModeError` when the character loads, and listening on a
248
+ character with voice input off throws one too. With `inputModes` set,
245
249
  `sendMessage` with typed text needs `'text'`, a reply option needs
246
250
  `'options'` or `'text'`, and listening needs `'voice'`.
247
251
  `character.inputModes` gives the modes in use.
248
- - The character owner pays the recognition at the provider's price, as for
249
- the turn's language model.
252
+ - The character owner pays the recognition at the price of the provider that
253
+ the owner chose, as for the turn's language model.
250
254
 
251
255
  ## Runtime and lifecycle
252
256
 
package/dist/index.d.ts CHANGED
@@ -43,6 +43,11 @@ interface AvatarPublicInfo {
43
43
  bakedConversationEnabled?: boolean;
44
44
  bakedConversationRevision?: number;
45
45
  allowCustomReply?: boolean;
46
+ /**
47
+ * The owner turned voice input on and chose a provider. Voice also needs
48
+ * `allowCustomReply`. Missing on older servers: off.
49
+ */
50
+ speechInputEnabled?: boolean;
46
51
  autoPlay: boolean;
47
52
  serverRenderingEnabled: boolean;
48
53
  cameraSqt?: Sqt | null;
@@ -336,7 +341,7 @@ interface ConversationTurnOptions {
336
341
  requestId?: string;
337
342
  }
338
343
 
339
- type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply'>>;
344
+ type NpcifyAssetInfo = Pick<AvatarPublicInfo, 'animations' | 'idleAnimationId' | 'bakedConversationEnabled' | 'bakedConversationRevision'> & Partial<Pick<AvatarPublicInfo, 'allowCustomReply' | 'speechInputEnabled'>>;
340
345
 
341
346
  interface LlmThinkingPayload {
342
347
  taskId?: string | null;
@@ -1150,8 +1155,9 @@ interface NpcifyOptions {
1150
1155
  * How the player can reply: `'options'`, `'text'` and `'voice'`, in any
1151
1156
  * combination; one mode forces it. Without it, every mode that the
1152
1157
  * character permits. A baked conversation permits only `['options']`; a
1153
- * character without custom replies permits only `options`. A mode that the
1154
- * character does not permit throws an `InputModeError` on load.
1158
+ * character without custom replies permits only `options`; `voice` needs
1159
+ * the character's voice input (`speechInputEnabled`, the STT tab). A mode
1160
+ * that the character does not permit throws an `InputModeError` on load.
1155
1161
  */
1156
1162
  inputModes?: InputMode[];
1157
1163
  /** Optional error callback. */
@@ -1537,6 +1543,8 @@ declare class Character extends CharacterRuntime {
1537
1543
  private readonly voice;
1538
1544
  private readonly configuredInputModes;
1539
1545
  private resolvedInputModes;
1546
+ /** The character's voice input switch, after the character loads. */
1547
+ private speechInput;
1540
1548
  private latestOptions;
1541
1549
  private listeningStateCallback;
1542
1550
  private transcriptCallback;
package/dist/index.js CHANGED
@@ -14,10 +14,6 @@ var GPT_IMAGE_MODELS = [
14
14
  "gpt-image-2.5-flare",
15
15
  "gpt-image-2.5-sunburst"
16
16
  ];
17
- var GEMINI_SPEECH_MODEL_VENDORS = {
18
- "gemini-3.5-transcribe-live": "gemini",
19
- "gemini-3.5-transcribe": "gemini"
20
- };
21
17
  var AI_ROLES = {
22
18
  character_draft: {
23
19
  name: "Character draft",
@@ -59,28 +55,6 @@ var AI_ROLES = {
59
55
  models: ["gemini-3.8-flash-tts", "gemini-3.8-flash-lite-tts"],
60
56
  options: []
61
57
  },
62
- // Priced by SPEECH_RECOGNITION_PRICES of the server (docs/voice-input.md).
63
- // Audio that arrives while the player speaks. The model selects the vendor.
64
- speech_input_live: {
65
- name: "Speech input (live)",
66
- source: "openai",
67
- models: [
68
- "gpt-transcribe",
69
- "gpt-live-transcribe",
70
- "gemini-3.5-transcribe-live",
71
- "gemini-3.5-transcribe"
72
- ],
73
- modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
74
- options: []
75
- },
76
- // One complete recording (an upload of an engine SDK).
77
- speech_input_file: {
78
- name: "Speech input (file)",
79
- source: "openai",
80
- models: ["gpt-transcribe", "gemini-3.5-transcribe"],
81
- modelVendors: GEMINI_SPEECH_MODEL_VENDORS,
82
- options: []
83
- },
84
58
  // Priced by OPENAI_PRICES.text of the generator.
85
59
  generator_agent: {
86
60
  name: "Generator agent",
@@ -116,7 +90,7 @@ var permittedInputModes = function(permissions) {
116
90
  if (permissions.baked || !permissions.allowCustomReply) {
117
91
  return ["options"];
118
92
  }
119
- return ["options", "text", "voice"];
93
+ return permissions.speechInput ? ["options", "text", "voice"] : ["options", "text"];
120
94
  };
121
95
  var isInputMode = function(value) {
122
96
  return INPUT_MODES.includes(value);
@@ -159,8 +133,9 @@ var resolveInputModes = function(configured, permissions) {
159
133
  }
160
134
  const refused = configured.filter((mode) => !permitted.includes(mode));
161
135
  if (refused.length > 0) {
136
+ const names = refused.map((mode) => `"${mode}"`).join(" and ");
162
137
  throw new InputModeError(
163
- `This character accepts only its reply options (Allow custom replies is off). Remove ${refused.map((mode) => `"${mode}"`).join(" and ")} from inputModes.`
138
+ permissions.allowCustomReply ? `Voice input is off for this character (its owner turns it on in the STT tab). Remove ${names} from inputModes.` : `This character accepts only its reply options (Allow custom replies is off). Remove ${names} from inputModes.`
164
139
  );
165
140
  }
166
141
  return [...configured];
@@ -186,6 +161,54 @@ var SPEECH_EVENTS = Object.freeze({
186
161
  turnFailed: "speech:turn_failed"
187
162
  });
188
163
 
164
+ // ../ts-shared/src/speech-input-providers.ts
165
+ var SPEECH_INPUT_PROVIDER_PROFILES = Object.freeze({
166
+ // The file model also streams: 120 ms slower than gpt-live-transcribe,
167
+ // at a 3.8 times lower price and with fewer errors.
168
+ openai: {
169
+ live: {
170
+ model: "gpt-transcribe",
171
+ releaseToTextMs: 783,
172
+ usdPerMinute: 45e-4,
173
+ approximate: false
174
+ },
175
+ file: {
176
+ model: "gpt-transcribe",
177
+ releaseToTextMs: 867,
178
+ usdPerMinute: 45e-4,
179
+ approximate: false
180
+ }
181
+ },
182
+ gemini: {
183
+ live: {
184
+ model: "gemini-3.5-transcribe-live",
185
+ releaseToTextMs: 266,
186
+ usdPerMinute: 9e-3,
187
+ approximate: true
188
+ },
189
+ file: {
190
+ model: "gemini-3.5-transcribe",
191
+ releaseToTextMs: 1263,
192
+ usdPerMinute: 5e-3,
193
+ approximate: true
194
+ }
195
+ },
196
+ elevenlabs: {
197
+ live: {
198
+ model: "scribe_v2_realtime",
199
+ releaseToTextMs: 208,
200
+ usdPerMinute: 0.39 / 60,
201
+ approximate: false
202
+ },
203
+ file: {
204
+ model: "scribe_v2",
205
+ releaseToTextMs: 925,
206
+ usdPerMinute: 0.22 / 60,
207
+ approximate: false
208
+ }
209
+ }
210
+ });
211
+
189
212
  // ../ts-shared/src/product-usage.ts
190
213
  var productUsageEvents = [
191
214
  "page_viewed",
@@ -10951,6 +10974,8 @@ var Character = class extends CharacterRuntime {
10951
10974
  voice;
10952
10975
  configuredInputModes;
10953
10976
  resolvedInputModes = null;
10977
+ /** The character's voice input switch, after the character loads. */
10978
+ speechInput = null;
10954
10979
  latestOptions = [];
10955
10980
  listeningStateCallback = null;
10956
10981
  transcriptCallback = null;
@@ -10978,7 +11003,8 @@ var Character = class extends CharacterRuntime {
10978
11003
  if (this.configuredInputModes && this.bakedAsset && !this.authoring) {
10979
11004
  resolveInputModes(this.configuredInputModes, {
10980
11005
  allowCustomReply: false,
10981
- baked: true
11006
+ baked: true,
11007
+ speechInput: false
10982
11008
  });
10983
11009
  }
10984
11010
  this.conversationSocket = new ConversationSocket({
@@ -11065,9 +11091,11 @@ var Character = class extends CharacterRuntime {
11065
11091
  const progressReporter = createAssetProgressReporter(callback);
11066
11092
  const info = await this.getAssetInfo();
11067
11093
  const baked = !this.authoring && Boolean(this.bakedAsset || info.bakedConversationEnabled);
11094
+ this.speechInput = info.speechInputEnabled === true;
11068
11095
  this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11069
11096
  allowCustomReply: info.allowCustomReply !== false,
11070
- baked
11097
+ baked,
11098
+ speechInput: this.speechInput
11071
11099
  });
11072
11100
  this.baked = baked ? await loadBakedConversation({
11073
11101
  baseUrl: this.baseUrl,
@@ -11138,9 +11166,11 @@ var Character = class extends CharacterRuntime {
11138
11166
  if (info.bakedConversationEnabled || this.bakedAsset) {
11139
11167
  throw new Error("Baked conversation requires local rendering.");
11140
11168
  }
11169
+ this.speechInput = info.speechInputEnabled === true;
11141
11170
  this.resolvedInputModes = resolveInputModes(this.configuredInputModes, {
11142
11171
  allowCustomReply: info.allowCustomReply !== false,
11143
- baked: false
11172
+ baked: false,
11173
+ speechInput: this.speechInput
11144
11174
  });
11145
11175
  this.animationController = new RemoteAnimationController();
11146
11176
  }
@@ -11268,7 +11298,11 @@ var Character = class extends CharacterRuntime {
11268
11298
  * character permits. Before `loadAssets`, the configuration or all modes.
11269
11299
  */
11270
11300
  get inputModes() {
11271
- return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({ allowCustomReply: true, baked: false });
11301
+ return this.resolvedInputModes ?? this.configuredInputModes ?? permittedInputModes({
11302
+ allowCustomReply: true,
11303
+ baked: false,
11304
+ speechInput: true
11305
+ });
11272
11306
  }
11273
11307
  get listeningState() {
11274
11308
  return this.voice.listeningState;
@@ -11409,6 +11443,11 @@ var Character = class extends CharacterRuntime {
11409
11443
  "Voice input is not available in a baked (offline) conversation."
11410
11444
  );
11411
11445
  }
11446
+ if (mode === "voice" && this.speechInput === false) {
11447
+ throw new InputModeError(
11448
+ "Voice input is off for this character. Its owner turns it on in the STT tab."
11449
+ );
11450
+ }
11412
11451
  const modes = this.inputModes;
11413
11452
  if (!modes.includes(mode)) {
11414
11453
  throw new InputModeError(
package/examples/main.js CHANGED
@@ -35,19 +35,22 @@ character.onOptionsReady(({ options }) => {
35
35
  }
36
36
  talk.disabled = false
37
37
  })
38
- // Hold the button, or Space, to talk. The release sends the speech.
38
+ // Hold the button, or Space, to talk. The release sends the speech. Voice
39
+ // needs the character's voice input (its owner turns it on in the STT tab).
39
40
  talk.hidden = !character.inputModes.includes('voice')
40
- character.bindPushToTalk({
41
- element: talk,
42
- enabled: () => !talk.disabled,
43
- onResult: ({ sent }) => {
44
- if (sent) {
45
- replies.replaceChildren()
46
- talk.disabled = true
47
- }
48
- },
49
- onError: report,
50
- })
41
+ if (!talk.hidden) {
42
+ character.bindPushToTalk({
43
+ element: talk,
44
+ enabled: () => !talk.disabled,
45
+ onResult: ({ sent }) => {
46
+ if (sent) {
47
+ replies.replaceChildren()
48
+ talk.disabled = true
49
+ }
50
+ },
51
+ onError: report,
52
+ })
53
+ }
51
54
  character.onListeningState((state) => {
52
55
  talk.textContent =
53
56
  { listening: 'Listening…', recognizing: 'Recognizing…' }[state] ?? 'Hold to talk'
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@npcify/web",
3
- "version": "0.3.0",
3
+ "version": "0.4.0",
4
4
  "description": "Framework-independent Npcify characters, live conversations and baked performances for the web",
5
5
  "type": "module",
6
6
  "license": "UNLICENSED",
@@ -35,8 +35,8 @@
35
35
  "tsup": "^8.5.0",
36
36
  "typescript": "^5.9.3",
37
37
  "@npcify/shared": "^0.1.0",
38
- "@npcify/vrm-renderer": "^0.1.0",
39
- "@npcify/web-client": "^0.0.0"
38
+ "@npcify/web-client": "^0.0.0",
39
+ "@npcify/vrm-renderer": "^0.1.0"
40
40
  },
41
41
  "repository": {
42
42
  "type": "git",