@alia.onl/sdk 8.0.0 → 10.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -1
- package/README.md +11 -3
- package/package.json +4 -4
- package/src/components/AliaChatScreen.tsx +44 -28
- package/src/components/IdentityMark.tsx +3 -0
- package/src/components/Markdown.tsx +89 -8
- package/src/components/voice/VoiceControls.tsx +46 -10
- package/src/hooks/useAliaChat.ts +8 -9
- package/src/hooks/useSpeechToText.ts +48 -14
- package/src/hooks/useTTS.ts +4 -9
- package/src/hooks/useVoiceRoom.ts +51 -30
- package/src/index.ts +6 -5
- package/src/lib/catalogue.ts +116 -103
- package/src/lib/chat-stream.ts +0 -5
- package/src/lib/chat-transport.ts +3 -2
- package/src/lib/keyboard.tsx +19 -1
- package/src/lib/speech-messages.ts +58 -13
- package/src/lib/speech-recognition-types.ts +33 -0
- package/src/lib/speech-recognition.native.ts +55 -0
- package/src/lib/speech-recognition.ts +9 -0
- package/src/lib/speech-synthesis.ts +2 -2
- package/src/lib/voice-turn.ts +3 -4
- package/src/voice.ts +4 -0
- package/src/lib/config.ts +0 -36
package/src/lib/catalogue.ts
CHANGED
|
@@ -1,58 +1,56 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The model catalogue, as the SDK consumes it (`GET /catalogue
|
|
3
|
-
* workstream 5).
|
|
2
|
+
* The model catalogue, as the SDK consumes it (`GET /catalogue`).
|
|
4
3
|
*
|
|
5
|
-
*
|
|
6
|
-
* `
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* offers instead.
|
|
4
|
+
* Alia has no models of its own: the catalogue lists the real models Oxy serves
|
|
5
|
+
* (ids `publisher/model`), and the server decides which one a request with no
|
|
6
|
+
* `model` gets (`defaultModelId`) and which ones a picker shows first
|
|
7
|
+
* (`featuredIds`). This package therefore ships NO model identifier: a caller
|
|
8
|
+
* that names no model sends no `model`, and the server answers with its default.
|
|
11
9
|
*
|
|
12
10
|
* ## Deliberately not React, and deliberately not react-query
|
|
13
11
|
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
* anywhere above it. Introducing one here would turn "the model default came
|
|
19
|
-
* from the server" into a breaking change for every consumer, which is a bad
|
|
20
|
-
* trade for a fallback.
|
|
12
|
+
* The SDK imports `@tanstack/react-query` in no module, so a consumer mounting
|
|
13
|
+
* `<AliaChatSheet>` is not required to have a `QueryClientProvider`. Resolution
|
|
14
|
+
* happens inside `send`, at the moment a request is about to name a model, and a
|
|
15
|
+
* module-level cache keeps that from becoming a fetch per message.
|
|
21
16
|
*
|
|
22
|
-
*
|
|
23
|
-
* request is about to name a model — rather than at render. A module-level cache
|
|
24
|
-
* keeps that from becoming a fetch per message.
|
|
17
|
+
* ## Parsing does not invent
|
|
25
18
|
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* - An entry whose `object` is neither known value is DROPPED. Defaulting it to
|
|
32
|
-
* either kind would break ADR 0003 invariant 1 in one direction or the other.
|
|
33
|
-
* - A response whose entries all fail to parse THROWS rather than reading as an
|
|
34
|
-
* empty catalogue, because "no models" and "we could not read the models"
|
|
35
|
-
* look identical to every caller below and one of them means "offer nothing".
|
|
36
|
-
*
|
|
37
|
-
* This file is a second implementation of that contract rather than a shared
|
|
38
|
-
* one, and that is forced rather than chosen: `@alia.onl/sdk` ships as raw
|
|
39
|
-
* source, so it cannot depend on an unpublished workspace package — a consumer's
|
|
40
|
-
* Metro would fail to resolve it. The three Node-side clients (Codea, Cowork,
|
|
41
|
-
* the CLI) share one instead.
|
|
19
|
+
* An entry without an id or a name is dropped, and a response whose entries all
|
|
20
|
+
* fail to parse THROWS rather than reading as an empty catalogue: "no models"
|
|
21
|
+
* and "we could not read the models" look identical to every caller below, and
|
|
22
|
+
* one of them means "offer nothing".
|
|
42
23
|
*/
|
|
43
24
|
|
|
44
|
-
/** A
|
|
45
|
-
export type
|
|
25
|
+
/** A reasoning effort a model accepts. */
|
|
26
|
+
export type ReasoningEffort = 'low' | 'medium' | 'high';
|
|
27
|
+
|
|
28
|
+
const REASONING_EFFORTS: readonly ReasoningEffort[] = ['low', 'medium', 'high'];
|
|
46
29
|
|
|
47
30
|
export interface CatalogueEntry {
|
|
31
|
+
/** `publisher/model`. */
|
|
48
32
|
readonly id: string;
|
|
49
|
-
readonly
|
|
50
|
-
readonly
|
|
51
|
-
readonly description: string;
|
|
52
|
-
|
|
53
|
-
readonly
|
|
54
|
-
|
|
55
|
-
readonly
|
|
33
|
+
readonly name: string;
|
|
34
|
+
readonly publisher: { readonly id: string; readonly name: string };
|
|
35
|
+
readonly description: string | null;
|
|
36
|
+
readonly contextWindow: number | null;
|
|
37
|
+
readonly maxOutput: number | null;
|
|
38
|
+
readonly inputModalities: readonly string[];
|
|
39
|
+
readonly outputModalities: readonly string[];
|
|
40
|
+
readonly tools: boolean;
|
|
41
|
+
/** The efforts the model accepts, cheapest first. Empty: send none. */
|
|
42
|
+
readonly reasoningEfforts: readonly ReasoningEffort[];
|
|
43
|
+
readonly pricing: { readonly inputPerMTok: string; readonly outputPerMTok: string } | null;
|
|
44
|
+
readonly releasedAt: string | null;
|
|
45
|
+
readonly featured: boolean;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface Catalogue {
|
|
49
|
+
readonly entries: readonly CatalogueEntry[];
|
|
50
|
+
/** What a request without `model` is answered with, or `null` if the server did not say. */
|
|
51
|
+
readonly defaultModelId: string | null;
|
|
52
|
+
/** The ids a picker shows first, in order. */
|
|
53
|
+
readonly featuredIds: readonly string[];
|
|
56
54
|
}
|
|
57
55
|
|
|
58
56
|
type JsonObject = Record<string, unknown>;
|
|
@@ -64,7 +62,15 @@ function asObject(value: unknown): JsonObject | null {
|
|
|
64
62
|
}
|
|
65
63
|
|
|
66
64
|
function asText(value: unknown): string | null {
|
|
67
|
-
return typeof value === 'string' ? value : null;
|
|
65
|
+
return typeof value === 'string' && value !== '' ? value : null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function asCount(value: unknown): number | null {
|
|
69
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function asTexts(value: unknown): string[] {
|
|
73
|
+
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : [];
|
|
68
74
|
}
|
|
69
75
|
|
|
70
76
|
function parseEntry(value: unknown): CatalogueEntry | null {
|
|
@@ -72,27 +78,41 @@ function parseEntry(value: unknown): CatalogueEntry | null {
|
|
|
72
78
|
if (raw === null) return null;
|
|
73
79
|
|
|
74
80
|
const id = asText(raw.id);
|
|
75
|
-
const
|
|
76
|
-
if (id === null ||
|
|
81
|
+
const name = asText(raw.name);
|
|
82
|
+
if (id === null || name === null) return null;
|
|
77
83
|
|
|
78
|
-
|
|
79
|
-
const
|
|
84
|
+
const publisher = asObject(raw.publisher);
|
|
85
|
+
const publisherId = asText(publisher?.id) ?? id.split('/')[0] ?? id;
|
|
86
|
+
const pricing = asObject(raw.pricing);
|
|
87
|
+
const input = asText(pricing?.inputPerMTok);
|
|
88
|
+
const output = asText(pricing?.outputPerMTok);
|
|
80
89
|
|
|
81
90
|
return {
|
|
82
91
|
id,
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
description: asText(raw.description)
|
|
86
|
-
|
|
87
|
-
|
|
92
|
+
name,
|
|
93
|
+
publisher: { id: publisherId, name: asText(publisher?.name) ?? publisherId },
|
|
94
|
+
description: asText(raw.description),
|
|
95
|
+
contextWindow: asCount(raw.contextWindow),
|
|
96
|
+
maxOutput: asCount(raw.maxOutput),
|
|
97
|
+
inputModalities: asTexts(raw.inputModalities),
|
|
98
|
+
outputModalities: asTexts(raw.outputModalities),
|
|
99
|
+
tools: raw.tools === true,
|
|
100
|
+
reasoningEfforts: REASONING_EFFORTS.filter((effort) =>
|
|
101
|
+
asTexts(raw.reasoningEfforts).includes(effort),
|
|
102
|
+
),
|
|
103
|
+
pricing: input !== null && output !== null ? { inputPerMTok: input, outputPerMTok: output } : null,
|
|
104
|
+
releasedAt: asText(raw.releasedAt),
|
|
105
|
+
featured: raw.featured === true,
|
|
88
106
|
};
|
|
89
107
|
}
|
|
90
108
|
|
|
91
|
-
/** Turn a catalogue response into
|
|
92
|
-
export function parseCatalogue(payload: unknown):
|
|
109
|
+
/** Turn a catalogue response into a {@link Catalogue}, or throw. */
|
|
110
|
+
export function parseCatalogue(payload: unknown): Catalogue {
|
|
93
111
|
const body = asObject(payload);
|
|
94
112
|
const data = body === null ? null : body.data;
|
|
95
|
-
if (!Array.isArray(data))
|
|
113
|
+
if (body === null || !Array.isArray(data)) {
|
|
114
|
+
throw new Error('The model catalogue response could not be read.');
|
|
115
|
+
}
|
|
96
116
|
|
|
97
117
|
const entries: CatalogueEntry[] = [];
|
|
98
118
|
for (const value of data) {
|
|
@@ -102,57 +122,53 @@ export function parseCatalogue(payload: unknown): CatalogueEntry[] {
|
|
|
102
122
|
if (data.length > 0 && entries.length === 0) {
|
|
103
123
|
throw new Error('The model catalogue response could not be read.');
|
|
104
124
|
}
|
|
105
|
-
return
|
|
125
|
+
return {
|
|
126
|
+
entries,
|
|
127
|
+
defaultModelId: asText(body.defaultModelId),
|
|
128
|
+
featuredIds: asTexts(body.featuredIds),
|
|
129
|
+
};
|
|
106
130
|
}
|
|
107
131
|
|
|
108
132
|
export interface ModelSelection {
|
|
109
|
-
/** What the caller asked for
|
|
110
|
-
readonly requestedId: string;
|
|
111
|
-
/** What a request should carry
|
|
112
|
-
readonly effectiveId: string;
|
|
133
|
+
/** What the caller asked for; `undefined` is "the server's default". */
|
|
134
|
+
readonly requestedId: string | undefined;
|
|
135
|
+
/** What a request should carry as `model`; `undefined` means omit it. */
|
|
136
|
+
readonly effectiveId: string | undefined;
|
|
137
|
+
/** The entry the request will be answered by, when the catalogue says. */
|
|
138
|
+
readonly entry: CatalogueEntry | null;
|
|
113
139
|
/** `replaced` when the requested identifier is not one the catalogue offers. */
|
|
114
|
-
readonly source: 'requested' | 'replaced';
|
|
140
|
+
readonly source: 'requested' | 'default' | 'replaced';
|
|
115
141
|
}
|
|
116
142
|
|
|
117
143
|
/**
|
|
118
144
|
* Resolve a requested identifier against the catalogue.
|
|
119
145
|
*
|
|
120
|
-
*
|
|
121
|
-
*
|
|
146
|
+
* - **Nothing requested** → omit `model`; the server's default answers.
|
|
122
147
|
* - **No catalogue — not loaded, or the request failed — leaves the choice
|
|
123
|
-
* alone.** Replacing on missing data
|
|
124
|
-
* slow cold start
|
|
125
|
-
* - **An
|
|
126
|
-
*
|
|
127
|
-
* among them; picking a different one is a product decision nobody made.
|
|
128
|
-
* - **`preferredId` is never trusted.** It is a build-time value, so it is
|
|
129
|
-
* checked against the catalogue like any other and falls through when the
|
|
130
|
-
* catalogue does not offer it.
|
|
148
|
+
* alone.** Replacing on missing data would change the model under the user
|
|
149
|
+
* on any slow cold start.
|
|
150
|
+
* - **An identifier the catalogue no longer lists** is not sent (it would be
|
|
151
|
+
* refused); the request omits `model` and the server's default answers.
|
|
131
152
|
*/
|
|
132
153
|
export function resolveSelection(
|
|
133
|
-
requestedId: string,
|
|
134
|
-
|
|
135
|
-
preferredId?: string,
|
|
154
|
+
requestedId: string | null | undefined,
|
|
155
|
+
catalogue: Catalogue | undefined,
|
|
136
156
|
): ModelSelection {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
157
|
+
const requested = requestedId === null || requestedId === '' ? undefined : requestedId;
|
|
158
|
+
const defaultEntry = (): CatalogueEntry | null =>
|
|
159
|
+
catalogue?.entries.find((entry) => entry.id === catalogue.defaultModelId) ?? null;
|
|
140
160
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
return { requestedId, effectiveId: requestedId, source: 'requested' };
|
|
161
|
+
if (requested === undefined) {
|
|
162
|
+
return { requestedId: undefined, effectiveId: undefined, entry: defaultEntry(), source: 'default' };
|
|
144
163
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
offered[0];
|
|
152
|
-
if (replacement === undefined) {
|
|
153
|
-
return { requestedId, effectiveId: requestedId, source: 'requested' };
|
|
164
|
+
if (catalogue === undefined) {
|
|
165
|
+
return { requestedId: requested, effectiveId: requested, entry: null, source: 'requested' };
|
|
166
|
+
}
|
|
167
|
+
const entry = catalogue.entries.find((candidate) => candidate.id === requested);
|
|
168
|
+
if (entry !== undefined) {
|
|
169
|
+
return { requestedId: requested, effectiveId: requested, entry, source: 'requested' };
|
|
154
170
|
}
|
|
155
|
-
return { requestedId, effectiveId:
|
|
171
|
+
return { requestedId: requested, effectiveId: undefined, entry: defaultEntry(), source: 'replaced' };
|
|
156
172
|
}
|
|
157
173
|
|
|
158
174
|
/**
|
|
@@ -162,9 +178,9 @@ export function resolveSelection(
|
|
|
162
178
|
* request. A REJECTED promise is evicted, because caching a failure would make a
|
|
163
179
|
* single cold-start network blip permanent for the life of the process.
|
|
164
180
|
*/
|
|
165
|
-
const inFlight = new Map<string, Promise<
|
|
181
|
+
const inFlight = new Map<string, Promise<Catalogue>>();
|
|
166
182
|
|
|
167
|
-
export function fetchCatalogue(apiUrl: string, accessToken?: string): Promise<
|
|
183
|
+
export function fetchCatalogue(apiUrl: string, accessToken?: string): Promise<Catalogue> {
|
|
168
184
|
const cached = inFlight.get(apiUrl);
|
|
169
185
|
if (cached !== undefined) return cached;
|
|
170
186
|
|
|
@@ -187,23 +203,20 @@ export function clearCatalogueCache(): void {
|
|
|
187
203
|
}
|
|
188
204
|
|
|
189
205
|
/**
|
|
190
|
-
* The
|
|
206
|
+
* The `model` a request should carry, given what the caller asked for, or
|
|
207
|
+
* `undefined` to omit it and let the server's default answer.
|
|
191
208
|
*
|
|
192
209
|
* Never throws: a catalogue that cannot be read leaves the requested identifier
|
|
193
|
-
* alone, which is the same answer as "the catalogue has not loaded yet".
|
|
194
|
-
* consumer that wants to KNOW the catalogue is unreadable calls
|
|
195
|
-
* {@link fetchCatalogue} directly.
|
|
210
|
+
* alone, which is the same answer as "the catalogue has not loaded yet".
|
|
196
211
|
*/
|
|
197
212
|
export async function resolveModelId(
|
|
198
213
|
apiUrl: string,
|
|
199
|
-
requestedId: string,
|
|
214
|
+
requestedId: string | undefined,
|
|
200
215
|
accessToken?: string,
|
|
201
|
-
|
|
202
|
-
)
|
|
203
|
-
if (requestedId.startsWith('mode:')) return requestedId;
|
|
216
|
+
): Promise<string | undefined> {
|
|
217
|
+
if (requestedId === undefined || requestedId === '') return undefined;
|
|
204
218
|
try {
|
|
205
|
-
|
|
206
|
-
return resolveSelection(requestedId, entries, preferredId).effectiveId;
|
|
219
|
+
return resolveSelection(requestedId, await fetchCatalogue(apiUrl, accessToken)).effectiveId;
|
|
207
220
|
} catch {
|
|
208
221
|
return requestedId;
|
|
209
222
|
}
|
package/src/lib/chat-stream.ts
CHANGED
|
@@ -177,11 +177,6 @@ function validateIgnoredNamedEvent(eventName: string, payload: JsonObject): void
|
|
|
177
177
|
requireEventVersion(payload, eventName);
|
|
178
178
|
|
|
179
179
|
switch (eventName) {
|
|
180
|
-
case 'alia.model_switch':
|
|
181
|
-
assertOnlyKeys(payload, ['eventVersion', 'model', 'modelName'], eventName);
|
|
182
|
-
asString(payload.model, `${eventName} model`);
|
|
183
|
-
asString(payload.modelName, `${eventName} modelName`);
|
|
184
|
-
return;
|
|
185
180
|
case 'alia.suggest_new_conversation':
|
|
186
181
|
assertOnlyKeys(payload, ['eventVersion', 'reason'], eventName);
|
|
187
182
|
asString(payload.reason, `${eventName} reason`);
|
|
@@ -16,7 +16,8 @@ export interface AuthenticatedResponseClient {
|
|
|
16
16
|
|
|
17
17
|
export interface AliaChatRequest {
|
|
18
18
|
readonly url: string;
|
|
19
|
-
|
|
19
|
+
/** `publisher/model`; omitted, the server's default model answers. */
|
|
20
|
+
readonly model?: string;
|
|
20
21
|
readonly messages: ReadonlyArray<{ readonly role: string; readonly content: string }>;
|
|
21
22
|
/**
|
|
22
23
|
* `'voice'` asks for an answer meant to be heard: short, conversational, no
|
|
@@ -48,7 +49,7 @@ export async function streamAliaChat(
|
|
|
48
49
|
'Content-Type': 'application/json',
|
|
49
50
|
},
|
|
50
51
|
body: JSON.stringify({
|
|
51
|
-
model: request.model,
|
|
52
|
+
...(request.model === undefined ? {} : { model: request.model }),
|
|
52
53
|
messages: request.messages,
|
|
53
54
|
stream: true,
|
|
54
55
|
...(request.responseMode === undefined ? {} : { responseMode: request.responseMode }),
|
package/src/lib/keyboard.tsx
CHANGED
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
// Web: re-export React Native built-in components as keyboard-controller substitutes
|
|
2
2
|
import React from 'react';
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
ScrollView,
|
|
5
|
+
type ScrollViewProps,
|
|
6
|
+
KeyboardAvoidingView as RNKeyboardAvoidingView,
|
|
7
|
+
type KeyboardAvoidingViewProps as RNKeyboardAvoidingViewProps,
|
|
8
|
+
} from 'react-native';
|
|
4
9
|
|
|
5
10
|
// Accept native-only props so shared components don't cause TS errors
|
|
6
11
|
type KeyboardAwareScrollViewProps = ScrollViewProps & {
|
|
@@ -17,6 +22,19 @@ const KeyboardAwareScrollView = React.forwardRef<ScrollView, KeyboardAwareScroll
|
|
|
17
22
|
);
|
|
18
23
|
KeyboardAwareScrollView.displayName = 'KeyboardAwareScrollView';
|
|
19
24
|
|
|
25
|
+
// `automaticOffset` is keyboard-controller's: measure the view's position in
|
|
26
|
+
// the window instead of trusting its parent-relative layout. The browser moves
|
|
27
|
+
// its own viewport for the keyboard, so on web it has nothing to do.
|
|
28
|
+
type KeyboardAvoidingViewProps = RNKeyboardAvoidingViewProps & {
|
|
29
|
+
automaticOffset?: boolean;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const KeyboardAvoidingView = React.forwardRef<
|
|
33
|
+
React.ComponentRef<typeof RNKeyboardAvoidingView>,
|
|
34
|
+
KeyboardAvoidingViewProps
|
|
35
|
+
>(({ automaticOffset, ...props }, ref) => <RNKeyboardAvoidingView ref={ref} {...props} />);
|
|
36
|
+
KeyboardAvoidingView.displayName = 'KeyboardAvoidingView';
|
|
37
|
+
|
|
20
38
|
export { KeyboardAwareScrollView, KeyboardAvoidingView };
|
|
21
39
|
|
|
22
40
|
// No-op provider on web — keyboard-controller is native-only
|
|
@@ -1,31 +1,76 @@
|
|
|
1
1
|
import type { SpeechRecognitionFailure } from './speech-recognition-types';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* moving recognition onto the device did not change what an error says.
|
|
4
|
+
* Every way dictation or a voice call can fail that a person is told about,
|
|
5
|
+
* as a stable code.
|
|
7
6
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* The hooks expose the code beside the English sentence (`errorCode` /
|
|
8
|
+
* `turnErrorCode` next to `error` / `turnError`) so an app can say it in its
|
|
9
|
+
* own language; the sentence stays for a consumer that has no translations.
|
|
10
10
|
*/
|
|
11
|
-
export
|
|
11
|
+
export type VoiceErrorCode =
|
|
12
|
+
| 'speech-unsupported'
|
|
13
|
+
| 'microphone-denied'
|
|
14
|
+
| 'microphone-busy'
|
|
15
|
+
| 'microphone-missing'
|
|
16
|
+
| 'speech-network'
|
|
17
|
+
| 'speech-language'
|
|
18
|
+
| 'speech-failed'
|
|
19
|
+
| 'voice-unavailable'
|
|
20
|
+
| 'not-authenticated'
|
|
21
|
+
| 'turn-failed'
|
|
22
|
+
| 'no-answer'
|
|
23
|
+
| 'not-played';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The English copy for each code — the same sentences dictation and voice mode
|
|
27
|
+
* used when they recorded through `expo-audio` and LiveKit, so moving
|
|
28
|
+
* recognition onto the device did not change what an error says.
|
|
29
|
+
*/
|
|
30
|
+
export const VOICE_ERROR_MESSAGES: Readonly<Record<VoiceErrorCode, string>> = {
|
|
31
|
+
'speech-unsupported': 'Speech recognition is not available on this device or browser',
|
|
32
|
+
'microphone-denied': 'Microphone permission required',
|
|
33
|
+
'microphone-busy': 'Microphone is in use by another application',
|
|
34
|
+
'microphone-missing': 'No microphone found — check your input devices',
|
|
35
|
+
'speech-network': 'Speech recognition needs a network connection',
|
|
36
|
+
'speech-language': 'Speech recognition does not support this language',
|
|
37
|
+
'speech-failed': 'Speech recognition failed',
|
|
38
|
+
'voice-unavailable': 'Voice is not available on this device',
|
|
39
|
+
'not-authenticated': 'Not authenticated',
|
|
40
|
+
'turn-failed': 'Voice request failed',
|
|
41
|
+
'no-answer': 'No answer came back — try saying it again',
|
|
42
|
+
'not-played': 'The answer could not be played aloud',
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* The code for a recognition failure. `null` for the failures that are not
|
|
47
|
+
* errors: `aborted` is a stop the caller asked for, and `no-speech` is a
|
|
48
|
+
* silence the caller recovers from.
|
|
49
|
+
*/
|
|
50
|
+
export function speechFailureCode(failure: SpeechRecognitionFailure): VoiceErrorCode | null {
|
|
12
51
|
switch (failure.code) {
|
|
13
52
|
case 'aborted':
|
|
14
53
|
case 'no-speech':
|
|
15
54
|
return null;
|
|
16
55
|
case 'unsupported':
|
|
17
|
-
return '
|
|
56
|
+
return 'speech-unsupported';
|
|
18
57
|
case 'not-allowed':
|
|
19
|
-
return '
|
|
58
|
+
return 'microphone-denied';
|
|
20
59
|
case 'audio-capture':
|
|
21
60
|
return failure.detail === 'NotReadableError' || failure.detail === 'TrackStartError'
|
|
22
|
-
? '
|
|
23
|
-
: '
|
|
61
|
+
? 'microphone-busy'
|
|
62
|
+
: 'microphone-missing';
|
|
24
63
|
case 'network':
|
|
25
|
-
return '
|
|
64
|
+
return 'speech-network';
|
|
26
65
|
case 'language-not-supported':
|
|
27
|
-
return '
|
|
66
|
+
return 'speech-language';
|
|
28
67
|
case 'other':
|
|
29
|
-
return '
|
|
68
|
+
return 'speech-failed';
|
|
30
69
|
}
|
|
31
70
|
}
|
|
71
|
+
|
|
72
|
+
/** The English copy a person sees when listening fails; `null` as for `speechFailureCode`. */
|
|
73
|
+
export function speechFailureMessage(failure: SpeechRecognitionFailure): string | null {
|
|
74
|
+
const code = speechFailureCode(failure);
|
|
75
|
+
return code === null ? null : VOICE_ERROR_MESSAGES[code];
|
|
76
|
+
}
|
|
@@ -53,6 +53,11 @@ export interface SpeechRecognitionHandlers {
|
|
|
53
53
|
export interface SpeechRecognitionOptions {
|
|
54
54
|
/** BCP-47 tag, e.g. `es-ES`. */
|
|
55
55
|
readonly lang: string;
|
|
56
|
+
/**
|
|
57
|
+
* Android: the recognition service to use, by package, instead of the
|
|
58
|
+
* system's default. `chooseSpeechRecognizer` picks it.
|
|
59
|
+
*/
|
|
60
|
+
readonly service?: string;
|
|
56
61
|
/**
|
|
57
62
|
* Remove the device's own playback from what the microphone hears, where the
|
|
58
63
|
* platform can (iOS voice processing). Voice mode listens while it speaks,
|
|
@@ -73,3 +78,31 @@ export function clampLevel(level: number): number {
|
|
|
73
78
|
if (!Number.isFinite(level)) return 0;
|
|
74
79
|
return Math.min(1, Math.max(0, level));
|
|
75
80
|
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* What to start a session with, decided before the first one opens.
|
|
84
|
+
*
|
|
85
|
+
* `silent` is set when the device has recognition services and every one of
|
|
86
|
+
* them, asked, named no language at all. A `language-not-supported` from such
|
|
87
|
+
* a device is not about the language: nothing on it recognizes speech (a Pixel
|
|
88
|
+
* without Google's speech services, whose remaining service has no language
|
|
89
|
+
* pack). Where nobody could be asked — the web, iOS — it is `false` and a
|
|
90
|
+
* language error stays one.
|
|
91
|
+
*/
|
|
92
|
+
export type SpeechRecognizerChoice =
|
|
93
|
+
| { readonly lang: string; readonly service?: string; readonly silent: boolean }
|
|
94
|
+
| { readonly failure: SpeechRecognitionFailure };
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The tag in `offered` that recognizes `lang`: the same tag (`en-US`, however
|
|
98
|
+
* the service spells it — `en_US`, `en-us`), else another region of the same
|
|
99
|
+
* language (`es-US` for `es-ES`), else `null`.
|
|
100
|
+
*/
|
|
101
|
+
export function matchSpeechLocale(lang: string, offered: readonly string[]): string | null {
|
|
102
|
+
const norm = (tag: string) => tag.replace(/_/g, '-').toLowerCase();
|
|
103
|
+
const wanted = norm(lang);
|
|
104
|
+
const exact = offered.find((tag) => norm(tag) === wanted);
|
|
105
|
+
if (exact !== undefined) return exact;
|
|
106
|
+
const language = wanted.split('-')[0];
|
|
107
|
+
return offered.find((tag) => norm(tag).split('-')[0] === language) ?? null;
|
|
108
|
+
}
|
|
@@ -19,6 +19,8 @@
|
|
|
19
19
|
import type { ExpoSpeechRecognitionErrorCode, ExpoSpeechRecognitionModule as NativeModule } from 'expo-speech-recognition';
|
|
20
20
|
import {
|
|
21
21
|
clampLevel,
|
|
22
|
+
matchSpeechLocale,
|
|
23
|
+
type SpeechRecognizerChoice,
|
|
22
24
|
type SpeechRecognitionFailure,
|
|
23
25
|
type SpeechRecognitionFailureCode,
|
|
24
26
|
type SpeechRecognitionHandlers,
|
|
@@ -94,6 +96,58 @@ function failureOf(error: ExpoSpeechRecognitionErrorCode): SpeechRecognitionFail
|
|
|
94
96
|
/** `volumechange` is -2..10, and anything below 0 is inaudible. */
|
|
95
97
|
const VOLUME_CEILING = 10;
|
|
96
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Which recognizer to use for `lang`, and the tag it knows the language by.
|
|
101
|
+
*
|
|
102
|
+
* Android leaves recognition to whichever service the device has, and
|
|
103
|
+
* `SpeechRecognizer` answers a language it cannot do only by failing:
|
|
104
|
+
* `language-not-supported`, which dictation showed as "does not support this
|
|
105
|
+
* language" whatever the real reason. On a Pixel the default service is often
|
|
106
|
+
* Android System Intelligence, which recognizes only the languages whose
|
|
107
|
+
* packs are downloaded, while another service on the same phone (Google's
|
|
108
|
+
* speech services) could have done it. So each service is asked what it
|
|
109
|
+
* recognizes — the default first — and the first that has the language wins,
|
|
110
|
+
* under the spelling it uses; a service offering only another region of the
|
|
111
|
+
* language (`es-US` for `es-ES`) is taken over failing.
|
|
112
|
+
*
|
|
113
|
+
* A service that reports nothing (Android 12 and older, a service that does
|
|
114
|
+
* not implement the query) proves nothing, so when none reports, the default
|
|
115
|
+
* is tried as before, with `silent` set if there were services to ask. iOS has
|
|
116
|
+
* none to list and goes straight to Apple's recognizer.
|
|
117
|
+
*/
|
|
118
|
+
export async function chooseSpeechRecognizer(lang: string): Promise<SpeechRecognizerChoice> {
|
|
119
|
+
const module = speechModule();
|
|
120
|
+
if (module === null) return { failure: { code: 'unsupported' } };
|
|
121
|
+
const attempt = <T>(read: () => T, fallback: T): T => {
|
|
122
|
+
try {
|
|
123
|
+
return read();
|
|
124
|
+
} catch {
|
|
125
|
+
return fallback;
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
const preferred = attempt(() => module.getDefaultRecognitionService().packageName, '');
|
|
129
|
+
const services = attempt(() => module.getSpeechRecognitionServices(), [] as string[]);
|
|
130
|
+
const order = [...new Set([preferred, ...services].filter((name) => name !== ''))];
|
|
131
|
+
|
|
132
|
+
let reported = false;
|
|
133
|
+
let asked = 0;
|
|
134
|
+
for (const service of order) {
|
|
135
|
+
let offered: string[];
|
|
136
|
+
try {
|
|
137
|
+
offered = (await module.getSupportedLocales({ androidRecognitionServicePackage: service })).locales;
|
|
138
|
+
} catch {
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
asked += 1;
|
|
142
|
+
if (offered.length === 0) continue;
|
|
143
|
+
reported = true;
|
|
144
|
+
const tag = matchSpeechLocale(lang, offered);
|
|
145
|
+
if (tag !== null) return { lang: tag, service, silent: false };
|
|
146
|
+
}
|
|
147
|
+
if (reported) return { failure: { code: 'language-not-supported' } };
|
|
148
|
+
return { lang, silent: asked > 0 };
|
|
149
|
+
}
|
|
150
|
+
|
|
97
151
|
export function startSpeechRecognition(
|
|
98
152
|
options: SpeechRecognitionOptions,
|
|
99
153
|
handlers: SpeechRecognitionHandlers,
|
|
@@ -146,6 +200,7 @@ export function startSpeechRecognition(
|
|
|
146
200
|
try {
|
|
147
201
|
ExpoSpeechRecognitionModule.start({
|
|
148
202
|
lang: options.lang,
|
|
203
|
+
...(options.service ? { androidRecognitionServicePackage: options.service } : {}),
|
|
149
204
|
interimResults: true,
|
|
150
205
|
continuous: true,
|
|
151
206
|
addsPunctuation: true,
|
|
@@ -31,6 +31,7 @@ import type {
|
|
|
31
31
|
SpeechRecognitionHandlers,
|
|
32
32
|
SpeechRecognitionOptions,
|
|
33
33
|
SpeechRecognitionSession,
|
|
34
|
+
SpeechRecognizerChoice,
|
|
34
35
|
} from './speech-recognition-types';
|
|
35
36
|
|
|
36
37
|
// ── The slice of the Web Speech API this uses ──
|
|
@@ -181,6 +182,14 @@ function meterStream(stream: MediaStream, onLevel: (level: number) => void): ()
|
|
|
181
182
|
* over (voice mode on a silence timer, dictation on the stop button), because
|
|
182
183
|
* the browser's own end-of-speech detection cuts people off mid-thought.
|
|
183
184
|
*/
|
|
185
|
+
/**
|
|
186
|
+
* The browser has one recognizer and no way to ask it which languages it
|
|
187
|
+
* knows: the language goes as given (see the native file for Android's).
|
|
188
|
+
*/
|
|
189
|
+
export async function chooseSpeechRecognizer(lang: string): Promise<SpeechRecognizerChoice> {
|
|
190
|
+
return { lang, silent: false };
|
|
191
|
+
}
|
|
192
|
+
|
|
184
193
|
export function startSpeechRecognition(
|
|
185
194
|
options: SpeechRecognitionOptions,
|
|
186
195
|
handlers: SpeechRecognitionHandlers,
|
|
@@ -15,8 +15,8 @@ export type ProductVoice = 'male' | 'female';
|
|
|
15
15
|
export interface SpeechClipRequest {
|
|
16
16
|
readonly apiUrl: string;
|
|
17
17
|
readonly token: string;
|
|
18
|
-
/**
|
|
19
|
-
readonly model
|
|
18
|
+
/** A speech model; omitted, the server's own speech model answers. */
|
|
19
|
+
readonly model?: string;
|
|
20
20
|
readonly input: string;
|
|
21
21
|
readonly voice: ProductVoice;
|
|
22
22
|
readonly speed?: number;
|
package/src/lib/voice-turn.ts
CHANGED
|
@@ -16,7 +16,6 @@
|
|
|
16
16
|
|
|
17
17
|
import { streamAliaChat, type AuthenticatedResponseClient } from './chat-transport';
|
|
18
18
|
import { resolveModelId } from './catalogue';
|
|
19
|
-
import { PREFERRED_CHAT_MODEL_ID } from './config';
|
|
20
19
|
|
|
21
20
|
export interface VoiceTurnMessage {
|
|
22
21
|
readonly role: 'user' | 'assistant';
|
|
@@ -50,18 +49,18 @@ interface LinkedClientFactory {
|
|
|
50
49
|
export interface AliaVoiceTurnSenderOptions {
|
|
51
50
|
readonly oxyServices: LinkedClientFactory;
|
|
52
51
|
readonly apiUrl: string;
|
|
53
|
-
/**
|
|
52
|
+
/** `publisher/model`, checked against `GET /catalogue` like `useAliaChat`'s; omitted, the server's default. */
|
|
54
53
|
readonly model?: string;
|
|
55
54
|
readonly agentId?: string;
|
|
56
55
|
}
|
|
57
56
|
|
|
58
57
|
/** The default sender: `POST /v1/chat/completions` as `useAliaChat` sends it, marked as voice. */
|
|
59
58
|
export function createAliaVoiceTurnSender(options: AliaVoiceTurnSenderOptions): VoiceTurnSender {
|
|
60
|
-
const { oxyServices, apiUrl, model
|
|
59
|
+
const { oxyServices, apiUrl, model, agentId } = options;
|
|
61
60
|
return async ({ text, history, signal, onText }) => {
|
|
62
61
|
const linked = oxyServices.createLinkedClient({ baseURL: apiUrl });
|
|
63
62
|
try {
|
|
64
|
-
const effectiveModel = await resolveModelId(apiUrl, model
|
|
63
|
+
const effectiveModel = await resolveModelId(apiUrl, model);
|
|
65
64
|
signal.throwIfAborted();
|
|
66
65
|
let answer = '';
|
|
67
66
|
await streamAliaChat(
|