@burdenoff/fe-libs 2026.827.1 → 2026.903.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/index.d.ts +4 -0
- package/dist/shared/assistant/index.d.ts.map +1 -1
- package/dist/shared/assistant/voice/turnVisibility.d.ts +162 -0
- package/dist/shared/assistant/voice/turnVisibility.d.ts.map +1 -0
- package/dist/shared/assistant/voice/turnVisibility.js +36 -0
- package/dist/shared/assistant/voice/useVoiceInput.d.ts +81 -0
- package/dist/shared/assistant/voice/useVoiceInput.d.ts.map +1 -0
- package/dist/shared/assistant/voice/useVoiceInput.js +100 -0
- package/dist/shared/components/assistantWidget/useVoiceInput.d.ts +24 -6
- package/dist/shared/components/assistantWidget/useVoiceInput.d.ts.map +1 -1
- package/dist/shared/components/assistantWidget/useVoiceInput.js +17 -37
- package/dist/shared-assistant.js +9 -7
- package/package.json +1 -1
|
@@ -12,4 +12,8 @@ export { isQuotaExhaustedError } from './gatewayClient';
|
|
|
12
12
|
export { buildProgress, getAssistantMedia, sanitize, isRecoverable, mergeMessagesById, mapSessionToHistory, } from './progress';
|
|
13
13
|
export { buildPromptRequestBody, findExistingSandbox, createAssistantSandbox, waitForSandboxRunning, waitForAssistantServiceReady, createAssistantSession, getAssistantSession, sendAssistantPromptAsync, getAssistantMessages, getAssistantStreamAccess, extendAssistantSandboxTTL, sendHeartbeat, getAssistantArtifacts, downloadAssistantArtifact, checkAssistantAiCredits, reportAssistantAiTokenUsage, } from './api';
|
|
14
14
|
export type { AssistantMode, AssistantPromptPart, PageContext, SandboxAssistantConfig, SandboxAssistantTransport, AssistantSendArgs, PrewarmLevel, AssistantRawMessage, AssistantRawMessagePart, AssistantWidgetMessage, AssistantArtifact, AssistantSandboxAuthContext, AssistantSessionInfo, DownloadedAssistantArtifact, StreamAccess, } from './types';
|
|
15
|
+
export { isVoiceTurn, voiceSessionIdOf, voiceMarker, chatTurns, voiceTurnsOnly, groupBySession, groupConsecutiveBySession, } from './voice/turnVisibility';
|
|
16
|
+
export type { VoiceSessionGroup, VoiceTranscriptTurn } from './voice/turnVisibility';
|
|
17
|
+
export { useVoiceInput } from './voice/useVoiceInput';
|
|
18
|
+
export type { UseVoiceInputOptions, UseVoiceInputResult, VoiceInputError } from './voice/useVoiceInput';
|
|
15
19
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shared/assistant/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,+BAA+B,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAKpB,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,EACrB,4BAA4B,EAC5B,sBAAsB,EACtB,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,EACxB,yBAAyB,EACzB,aAAa,EACb,qBAAqB,EACrB,yBAAyB,EACzB,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,OAAO,CAAC;AAEf,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,iBAAiB,EACjB,YAAY,EACZ,mBAAmB,EACnB,uBAAuB,EACvB,sBAAsB,EACtB,iBAAiB,EACjB,2BAA2B,EAC3B,oBAAoB,EACpB,2BAA2B,EAC3B,YAAY,GACb,MAAM,SAAS,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/shared/assistant/index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,4BAA4B,EAAE,MAAM,gCAAgC,CAAC;AAC9E,OAAO,EAAE,+BAA+B,EAAE,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,aAAa,EACb,iBAAiB,EACjB,mBAAmB,GACpB,MAAM,YAAY,CAAC;AAKpB,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,EACrB,4BAA4B,EAC5B,sBAAsB,EACtB,mBAAmB,EACnB,wBAAwB,EACxB,oBAAoB,EACpB,wBAAwB,EACxB,yBAAyB,EACzB,aAAa,EACb,qBAAqB,EACrB,yBAAyB,EACzB,uBAAuB,EACvB,2BAA2B,GAC5B,MAAM,OAAO,CAAC;AAEf,YAAY,EACV,aAAa,EACb,mBAAmB,EACnB,WAAW,EACX,sBAAsB,EACtB,yBAAyB,EACzB,iBAAiB,EACjB,YAAY,EACZ,mBAAmB,EACnB,uBAAuB,EACvB,sBAAsB,EACtB,iBAAiB,EACjB,2BAA2B,EAC3B,oBAAoB,EACpB,2BAA2B,EAC3B,YAAY,GACb,MAAM,SAAS,CAAC;AACjB,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,WAAW,EACX,SAAS,EACT,cAAc,EACd,cAAc,EACd,yBAAyB,GAC1B,MAAM,wBAAwB,CAAC;AAChC,YAAY,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,YAAY,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Which surface does an assistant turn belong to — typed chat, or live voice?
|
|
3
|
+
*
|
|
4
|
+
* Shared by every product, for the same reason the transport is: voice was built
|
|
5
|
+
* as an overlay over the chat thread in the first product that needed it, so
|
|
6
|
+
* both surfaces write into ONE conversation. Without a marker there is no way to
|
|
7
|
+
* tell a spoken turn from a typed one once it is stored, and every voice
|
|
8
|
+
* exchange shows up in the typed thread.
|
|
9
|
+
*
|
|
10
|
+
* One marker fixes both directions:
|
|
11
|
+
* - the chat panel hides turns that happened in voice
|
|
12
|
+
* - the voice panel shows ONLY turns that happened in voice
|
|
13
|
+
*
|
|
14
|
+
* ★ Deliberately a DISPLAY split, not separate persistence. It survives a reload
|
|
15
|
+
* because the chat transcript is never rehydrated from the backend — the store is
|
|
16
|
+
* the only source of truth for what either panel shows. The backend session stays
|
|
17
|
+
* shared, so the model keeps full context across both surfaces, which is what
|
|
18
|
+
* makes "carry on where we left off" work. Genuinely separate backend sessions
|
|
19
|
+
* are a larger change and would cost that.
|
|
20
|
+
*
|
|
21
|
+
* ★ Product-agnostic by construction: it knows about turns and surfaces, never
|
|
22
|
+
* about crops, workflows or nodes. Nothing here needs to change for a product
|
|
23
|
+
* whose modes are `fluidgrids-node` rather than `api-calls`.
|
|
24
|
+
*/
|
|
25
|
+
interface TurnLike {
|
|
26
|
+
metadata?: ({
|
|
27
|
+
voice?: boolean;
|
|
28
|
+
voiceSessionId?: string;
|
|
29
|
+
} & Record<string, unknown>) | undefined;
|
|
30
|
+
}
|
|
31
|
+
/** True when this turn happened inside a live voice session. */
|
|
32
|
+
export declare function isVoiceTurn(message: TurnLike): boolean;
|
|
33
|
+
/**
|
|
34
|
+
* WHICH voice session produced this turn, if we know.
|
|
35
|
+
*
|
|
36
|
+
* ★ `undefined` is a real answer, not a failure: every spoken turn recorded
|
|
37
|
+
* before session ids existed carries `voice: true` and nothing else, and those
|
|
38
|
+
* conversations are in users' localStorage today. It means "some earlier
|
|
39
|
+
* session" and must never be read as "not spoken" — `isVoiceTurn` remains the
|
|
40
|
+
* only thing that decides which surface a turn belongs to.
|
|
41
|
+
*/
|
|
42
|
+
export declare function voiceSessionIdOf(message: TurnLike): string | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* The marker as a fragment to spread into a message's `metadata`.
|
|
45
|
+
*
|
|
46
|
+
* ★ `patchMessage` REPLACES `metadata` instead of merging it, so every write
|
|
47
|
+
* that rebuilds the object has to re-assert this or the turn silently demotes
|
|
48
|
+
* itself to a chat turn. That is exactly what shipped: the marker was stamped
|
|
49
|
+
* at creation and destroyed the instant the reply completed, so a spoken answer
|
|
50
|
+
* appeared in the typed thread AND disappeared from the voice transcript at the
|
|
51
|
+
* same moment — the transcript could only ever show the user's questions.
|
|
52
|
+
*
|
|
53
|
+
* Spreading one helper at all four sites is what keeps them from drifting;
|
|
54
|
+
* `voiceMarkerCallSitesAreComplete` in the test file fails the build if a new
|
|
55
|
+
* metadata write forgets it.
|
|
56
|
+
*
|
|
57
|
+
* ★ Takes the SESSION ID rather than a boolean, and stamps both fields from it.
|
|
58
|
+
* There used to be a separate `dictated` flag that call sites passed straight
|
|
59
|
+
* through, which made it possible to mark a turn as spoken while recording no
|
|
60
|
+
* session — a state with no meaning that only the transcript would ever notice.
|
|
61
|
+
* One argument means the two can never disagree.
|
|
62
|
+
*
|
|
63
|
+
* An absent id yields an EMPTY object, so a typed turn gets no marker at all:
|
|
64
|
+
* `chatTurns` is allow-by-default, and a stray `voice: false` would be a second
|
|
65
|
+
* way of saying the same thing.
|
|
66
|
+
*/
|
|
67
|
+
export declare function voiceMarker(sessionId: string | undefined): {
|
|
68
|
+
voice?: true;
|
|
69
|
+
voiceSessionId?: string;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* The turns the CHAT panel should show: everything that was not spoken.
|
|
73
|
+
*
|
|
74
|
+
* Untagged turns are treated as chat, so every conversation recorded before the
|
|
75
|
+
* marker existed keeps rendering exactly as it did.
|
|
76
|
+
*/
|
|
77
|
+
export declare function chatTurns<T extends TurnLike>(messages: readonly T[]): T[];
|
|
78
|
+
/**
|
|
79
|
+
* The turns the VOICE panel should show: only what was spoken.
|
|
80
|
+
*
|
|
81
|
+
* This is also what scopes the voice transcript to the session. Previously it
|
|
82
|
+
* took the last six messages of the whole conversation, so opening voice on an
|
|
83
|
+
* existing thread immediately painted typed replies — the wall of raw markdown
|
|
84
|
+
* in the operator's screenshot.
|
|
85
|
+
*/
|
|
86
|
+
export declare function voiceTurnsOnly<T extends TurnLike>(messages: readonly T[]): T[];
|
|
87
|
+
/** One microphone-on-to-End, and the turns it produced, oldest first. */
|
|
88
|
+
export interface VoiceSessionGroup<T> {
|
|
89
|
+
/**
|
|
90
|
+
* `undefined` for turns recorded before session ids existed. Rendered as one
|
|
91
|
+
* leading group rather than dropped — it is real history the user spoke.
|
|
92
|
+
*/
|
|
93
|
+
sessionId: string | undefined;
|
|
94
|
+
turns: T[];
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Split spoken turns into the sessions that produced them.
|
|
98
|
+
*
|
|
99
|
+
* The transcript shows a conversation's whole spoken history, which is the more
|
|
100
|
+
* useful of the two options — you can go back to what you asked last time — but
|
|
101
|
+
* run together it reads as one long exchange, and a question from a session
|
|
102
|
+
* three days ago looks like it was just asked. Grouping is what makes showing
|
|
103
|
+
* everything honest.
|
|
104
|
+
*
|
|
105
|
+
* ★ Groups by RUNS, not by collecting every turn with the same id. Turns are
|
|
106
|
+
* stored in the order they happened, so a run IS a session; gathering by id
|
|
107
|
+
* instead would silently reorder history if two sessions ever interleaved, and
|
|
108
|
+
* would hide it rather than show it if they did.
|
|
109
|
+
*
|
|
110
|
+
* ★ Consecutive un-stamped turns collapse into a single leading group. Treating
|
|
111
|
+
* each as its own session would print a divider between every pre-existing turn
|
|
112
|
+
* — one long exchange shattered into fragments, which is worse than the
|
|
113
|
+
* undivided version this is meant to fix.
|
|
114
|
+
*/
|
|
115
|
+
export declare function groupBySession<T extends TurnLike>(turns: readonly T[]): VoiceSessionGroup<T>[];
|
|
116
|
+
/**
|
|
117
|
+
* The same grouping for turns that have already been flattened for display.
|
|
118
|
+
*
|
|
119
|
+
* The transcript renders `{id, role, text}`, not `Message`, so it cannot reach
|
|
120
|
+
* `metadata`. Passing the extractor keeps ONE implementation of the run logic
|
|
121
|
+
* instead of a second copy that drifts — the subtle part is the `undefined`
|
|
122
|
+
* merge, and having it in two places is how one of them stops doing it.
|
|
123
|
+
*/
|
|
124
|
+
export declare function groupConsecutiveBySession<T>(turns: readonly T[], sessionIdOf: (turn: T) => string | undefined): VoiceSessionGroup<T>[];
|
|
125
|
+
/**
|
|
126
|
+
* A spoken turn flattened for display.
|
|
127
|
+
*
|
|
128
|
+
* ★ Shared by every component between the chat area and the transcript, so the
|
|
129
|
+
* session id is carried in the TYPE and not merely at runtime. The intermediate
|
|
130
|
+
* props used to declare `{id, role, text}` while the objects flowing through
|
|
131
|
+
* happened to have more: it compiled, and the dividers worked, purely because
|
|
132
|
+
* JavaScript keeps the extra property. Any future refactor that rebuilt one of
|
|
133
|
+
* those objects would have dropped the id and silently run every session back
|
|
134
|
+
* together — the exact defect this field was added to fix, reintroduced with no
|
|
135
|
+
* type error.
|
|
136
|
+
*/
|
|
137
|
+
export interface VoiceTranscriptTurn {
|
|
138
|
+
id: string;
|
|
139
|
+
role: 'user' | 'assistant';
|
|
140
|
+
text: string;
|
|
141
|
+
/**
|
|
142
|
+
* When the turn happened, ISO.
|
|
143
|
+
*
|
|
144
|
+
* Carried for the SAVED transcript rather than for the on-screen one: a file
|
|
145
|
+
* that outlives the session is read later, when "when did I ask that?" is the
|
|
146
|
+
* actual question, and a record without times answers it badly.
|
|
147
|
+
*/
|
|
148
|
+
timestamp: string;
|
|
149
|
+
/**
|
|
150
|
+
* `undefined` on turns recorded before session ids existed.
|
|
151
|
+
*
|
|
152
|
+
* ★ REQUIRED-but-nullable rather than optional, and the difference is the
|
|
153
|
+
* whole point. With `sessionId?: string`, an object rebuilt without the key is
|
|
154
|
+
* still structurally assignable — so a refactor that mapped turns through and
|
|
155
|
+
* dropped it would compile, and every session would silently run back
|
|
156
|
+
* together. Written this way, omitting the key is a type error and passing
|
|
157
|
+
* `undefined` is a decision someone made on purpose.
|
|
158
|
+
*/
|
|
159
|
+
sessionId: string | undefined;
|
|
160
|
+
}
|
|
161
|
+
export {};
|
|
162
|
+
//# sourceMappingURL=turnVisibility.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"turnVisibility.d.ts","sourceRoot":"","sources":["../../../../src/shared/assistant/voice/turnVisibility.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,UAAU,QAAQ;IAChB,QAAQ,CAAC,EAAE,CAAC;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;CACjG;AAED,gEAAgE;AAChE,wBAAgB,WAAW,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAEtD;AAED;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAGtE;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG;IAAE,KAAK,CAAC,EAAE,IAAI,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAEpG;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,CAAC,SAAS,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,CAEzE;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,GAAG,CAAC,EAAE,CAE9E;AAED,yEAAyE;AACzE,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC;;;OAGG;IACH,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,EAAE,CAAC,EAAE,CAAC;CACZ;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAE9F;AAED;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CAAC,CAAC,EACzC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,SAAS,GAC3C,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAWxB;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;;;OASG;IACH,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/B"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
//#region src/shared/assistant/voice/turnVisibility.ts
|
|
2
|
+
function e(e) {
|
|
3
|
+
return e.metadata?.voice === !0;
|
|
4
|
+
}
|
|
5
|
+
function t(e) {
|
|
6
|
+
let t = e.metadata?.voiceSessionId;
|
|
7
|
+
return typeof t == "string" && t.length > 0 ? t : void 0;
|
|
8
|
+
}
|
|
9
|
+
function n(e) {
|
|
10
|
+
return e ? {
|
|
11
|
+
voice: !0,
|
|
12
|
+
voiceSessionId: e
|
|
13
|
+
} : {};
|
|
14
|
+
}
|
|
15
|
+
function r(t) {
|
|
16
|
+
return t.filter((t) => !e(t));
|
|
17
|
+
}
|
|
18
|
+
function i(t) {
|
|
19
|
+
return t.filter(e);
|
|
20
|
+
}
|
|
21
|
+
function a(e) {
|
|
22
|
+
return o(e, t);
|
|
23
|
+
}
|
|
24
|
+
function o(e, t) {
|
|
25
|
+
let n = [];
|
|
26
|
+
for (let r of e) {
|
|
27
|
+
let e = t(r), i = n[n.length - 1];
|
|
28
|
+
i && i.sessionId === e ? i.turns.push(r) : n.push({
|
|
29
|
+
sessionId: e,
|
|
30
|
+
turns: [r]
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
return n;
|
|
34
|
+
}
|
|
35
|
+
//#endregion
|
|
36
|
+
export { r as chatTurns, a as groupBySession, o as groupConsecutiveBySession, e as isVoiceTurn, n as voiceMarker, t as voiceSessionIdOf, i as voiceTurnsOnly };
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Speech-to-text for the assistant composer (Workstream 2 of the AI Assistant
|
|
3
|
+
* V2 multimodal effort).
|
|
4
|
+
*
|
|
5
|
+
* Input-only, browser-native (Web Speech API). It degrades gracefully:
|
|
6
|
+
* `isSupported` is false on engines without SpeechRecognition (notably Firefox),
|
|
7
|
+
* and the composer hides the mic entirely in that case rather than showing a
|
|
8
|
+
* dead control.
|
|
9
|
+
*
|
|
10
|
+
* ⚠️ This does NOT mean the audio stays on the device. Chrome's
|
|
11
|
+
* `webkitSpeechRecognition` is a NETWORK service — the audio is streamed to the
|
|
12
|
+
* browser vendor for recognition, and the supported language list is theirs, not
|
|
13
|
+
* ours. We neither receive nor store it, which is all an earlier version of this
|
|
14
|
+
* comment ("no audio ever leaves the device through us") actually claimed; read
|
|
15
|
+
* literally it implied something stronger and untrue. Whether recording farmers'
|
|
16
|
+
* speech this way needs disclosure or consent is a product/legal call, not a
|
|
17
|
+
* code one.
|
|
18
|
+
*
|
|
19
|
+
* Output (reading replies aloud) is a separate concern and lives in
|
|
20
|
+
* `AssistantSpeechProvider`; an earlier version of this note said the assistant
|
|
21
|
+
* does not speak, which stopped being true when BOFF-6290 shipped.
|
|
22
|
+
*
|
|
23
|
+
* Differences from the fe-libs `assistantWidget/useVoiceInput` this is modelled
|
|
24
|
+
* on, all of which the composer needs:
|
|
25
|
+
* - **interim results are exposed**, so the user sees words appear while
|
|
26
|
+
* speaking instead of staring at a silent button until a phrase finalises;
|
|
27
|
+
* - **errors are surfaced** (`error`) so a denied mic permission can be
|
|
28
|
+
* explained rather than looking like a no-op;
|
|
29
|
+
* - **the recogniser language follows the app locale** (a Tamil or Hindi
|
|
30
|
+
* farmer dictating into an `en-US` recogniser gets garbage);
|
|
31
|
+
* - **`stop()` resolves the final transcript** through `onFinalize`, so the
|
|
32
|
+
* composer can commit exactly once when the user stops.
|
|
33
|
+
*/
|
|
34
|
+
export type VoiceInputError = 'not-allowed' | 'no-speech' | 'audio-capture' | 'network' | 'language-not-supported' | 'unknown';
|
|
35
|
+
export interface UseVoiceInputOptions {
|
|
36
|
+
/**
|
|
37
|
+
* Called with each newly finalised phrase. The composer appends it to the
|
|
38
|
+
* draft; keeping the committed text in the caller (rather than accumulating
|
|
39
|
+
* here) means the user can freely edit while dictation continues.
|
|
40
|
+
*/
|
|
41
|
+
onFinalize: (text: string) => void;
|
|
42
|
+
/**
|
|
43
|
+
* BCP-47 tag to listen for. Omit to follow the app locale, which is what this
|
|
44
|
+
* hook did unconditionally before (BOFF-7108).
|
|
45
|
+
*
|
|
46
|
+
* The two are genuinely independent: a farmer whose interface is English may
|
|
47
|
+
* well speak Tamil, and deriving the recogniser language from the UI meant the
|
|
48
|
+
* only way to dictate Tamil was to translate the entire app.
|
|
49
|
+
*/
|
|
50
|
+
language?: string;
|
|
51
|
+
}
|
|
52
|
+
export interface UseVoiceInputResult {
|
|
53
|
+
isSupported: boolean;
|
|
54
|
+
isListening: boolean;
|
|
55
|
+
/** Requested, not yet capturing. Distinguishes starting up from listening. */
|
|
56
|
+
isStarting: boolean;
|
|
57
|
+
/**
|
|
58
|
+
* Starting OR listening — "the microphone is busy".
|
|
59
|
+
*
|
|
60
|
+
* ★ Most callers mean THIS and were testing `isListening`, which was the same
|
|
61
|
+
* thing only while capture was assumed the instant start() returned. Exposing
|
|
62
|
+
* it once stops four call sites disagreeing about what counts as active.
|
|
63
|
+
*/
|
|
64
|
+
isActive: boolean;
|
|
65
|
+
/** Words recognised but not yet finalised — render as a live hint. */
|
|
66
|
+
interim: string;
|
|
67
|
+
error: VoiceInputError | null;
|
|
68
|
+
/**
|
|
69
|
+
* BCP-47 tag the recogniser was asked for. Exposed so error copy can name the
|
|
70
|
+
* language that failed ("Tamil dictation is not supported…") rather than
|
|
71
|
+
* leaving the user to guess which language the browser rejected.
|
|
72
|
+
*/
|
|
73
|
+
language: string;
|
|
74
|
+
start: () => void;
|
|
75
|
+
stop: () => void;
|
|
76
|
+
/** Start if idle, stop if listening — what the mic button binds to. */
|
|
77
|
+
toggle: () => void;
|
|
78
|
+
clearError: () => void;
|
|
79
|
+
}
|
|
80
|
+
export declare function useVoiceInput({ onFinalize, language: requested }: UseVoiceInputOptions): UseVoiceInputResult;
|
|
81
|
+
//# sourceMappingURL=useVoiceInput.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"useVoiceInput.d.ts","sourceRoot":"","sources":["../../../../src/shared/assistant/voice/useVoiceInput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AA+DH,MAAM,MAAM,eAAe,GACzB,aAAa,GAAG,WAAW,GAAG,eAAe,GAAG,SAAS,GAAG,wBAAwB,GAAG,SAAS,CAAC;AA+BnG,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,UAAU,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,8EAA8E;IAC9E,UAAU,EAAE,OAAO,CAAC;IACpB;;;;;;OAMG;IACH,QAAQ,EAAE,OAAO,CAAC;IAClB,sEAAsE;IACtE,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,eAAe,GAAG,IAAI,CAAC;IAC9B;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,uEAAuE;IACvE,MAAM,EAAE,MAAM,IAAI,CAAC;IACnB,UAAU,EAAE,MAAM,IAAI,CAAC;CACxB;AAED,wBAAgB,aAAa,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,oBAAoB,GAAG,mBAAmB,CA+K5G"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { useI18n as e } from "../../providers/shell/I18nProvider.js";
|
|
2
|
+
import { useCallback as t, useEffect as n, useRef as r, useState as i } from "react";
|
|
3
|
+
//#region src/shared/assistant/voice/useVoiceInput.ts
|
|
4
|
+
var a = {
|
|
5
|
+
en: "en-US",
|
|
6
|
+
es: "es-ES",
|
|
7
|
+
fr: "fr-FR",
|
|
8
|
+
de: "de-DE",
|
|
9
|
+
ja: "ja-JP",
|
|
10
|
+
zh: "zh-CN",
|
|
11
|
+
ta: "ta-IN",
|
|
12
|
+
hi: "hi-IN",
|
|
13
|
+
ar: "ar-SA"
|
|
14
|
+
};
|
|
15
|
+
function o() {
|
|
16
|
+
if (typeof window > "u") return null;
|
|
17
|
+
let e = window;
|
|
18
|
+
return e.SpeechRecognition ?? e.webkitSpeechRecognition ?? null;
|
|
19
|
+
}
|
|
20
|
+
function s(e) {
|
|
21
|
+
switch (e) {
|
|
22
|
+
case "not-allowed":
|
|
23
|
+
case "service-not-allowed": return "not-allowed";
|
|
24
|
+
case "no-speech": return "no-speech";
|
|
25
|
+
case "audio-capture": return "audio-capture";
|
|
26
|
+
case "network": return "network";
|
|
27
|
+
case "language-not-supported": return "language-not-supported";
|
|
28
|
+
default: return "unknown";
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
function c({ onFinalize: c, language: l }) {
|
|
32
|
+
let { locale: u } = e(), [d, f] = i(!1), [p, m] = i(!1), [h, g] = i(""), [_, v] = i(null), y = r(null), b = r(0), x = r(c);
|
|
33
|
+
n(() => {
|
|
34
|
+
x.current = c;
|
|
35
|
+
}, [c]);
|
|
36
|
+
let S = o() !== null, C = l ?? a[u] ?? "en-US";
|
|
37
|
+
n(() => () => {
|
|
38
|
+
y.current?.abort(), y.current = null;
|
|
39
|
+
}, []);
|
|
40
|
+
let w = t(() => {
|
|
41
|
+
let e = o();
|
|
42
|
+
if (!e || y.current) return;
|
|
43
|
+
let t = new e();
|
|
44
|
+
t.continuous = !0, t.interimResults = !0, t.lang = C;
|
|
45
|
+
let n = b.current;
|
|
46
|
+
t.onresult = (e) => {
|
|
47
|
+
if (n !== b.current) return;
|
|
48
|
+
let t = "", r = "";
|
|
49
|
+
for (let n = e.resultIndex; n < e.results.length; n += 1) {
|
|
50
|
+
let i = e.results[n], a = i[0]?.transcript ?? "";
|
|
51
|
+
i.isFinal ? t += a : r += a;
|
|
52
|
+
}
|
|
53
|
+
t.trim() && x.current(t.trim()), g(r.trim());
|
|
54
|
+
}, t.onerror = (e) => {
|
|
55
|
+
if (n !== b.current) return;
|
|
56
|
+
let t = s(e.error);
|
|
57
|
+
t !== "no-speech" && v(t);
|
|
58
|
+
}, t.onend = () => {
|
|
59
|
+
n === b.current && (y.current = null, m(!1), f(!1), g(""));
|
|
60
|
+
}, t.onstart = () => {
|
|
61
|
+
n === b.current && (m(!1), f(!0));
|
|
62
|
+
}, m(!0);
|
|
63
|
+
try {
|
|
64
|
+
t.start();
|
|
65
|
+
} catch {
|
|
66
|
+
y.current = null, m(!1), f(!1);
|
|
67
|
+
return;
|
|
68
|
+
}
|
|
69
|
+
y.current = t, v(null), g("");
|
|
70
|
+
}, [C]), T = t(() => {
|
|
71
|
+
let e = y.current;
|
|
72
|
+
y.current = null, b.current += 1, e?.stop(), m(!1), f(!1), g("");
|
|
73
|
+
}, []), E = r(C);
|
|
74
|
+
n(() => {
|
|
75
|
+
E.current !== C && (E.current = C, T(), v(null));
|
|
76
|
+
}, [C, T]);
|
|
77
|
+
let D = t(() => {
|
|
78
|
+
d || p ? T() : w();
|
|
79
|
+
}, [
|
|
80
|
+
d,
|
|
81
|
+
p,
|
|
82
|
+
w,
|
|
83
|
+
T
|
|
84
|
+
]), O = t(() => v(null), []);
|
|
85
|
+
return {
|
|
86
|
+
isSupported: S,
|
|
87
|
+
isListening: d,
|
|
88
|
+
isStarting: p,
|
|
89
|
+
isActive: d || p,
|
|
90
|
+
interim: h,
|
|
91
|
+
error: _,
|
|
92
|
+
language: C,
|
|
93
|
+
start: w,
|
|
94
|
+
stop: T,
|
|
95
|
+
toggle: D,
|
|
96
|
+
clearError: O
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
//#endregion
|
|
100
|
+
export { c as useVoiceInput };
|
|
@@ -1,15 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* The widget's dictation hook — now a thin wrapper over the shared engine.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* ★ WHY A WRAPPER AND NOT A REPLACEMENT. This file's API (`transcript`, `reset`)
|
|
5
|
+
* is what every shell mounting `AssistantWidget` already calls. The shared engine
|
|
6
|
+
* has a different, better contract — it reports finalised phrases through a
|
|
7
|
+
* callback instead of accumulating a string — so swapping it in directly would
|
|
8
|
+
* have broken every one of those consumers at once, for a change that is meant to
|
|
9
|
+
* FIX them.
|
|
10
|
+
*
|
|
11
|
+
* ★ WHAT CONSUMERS SILENTLY GAIN. This file was 122 lines and the engine it now
|
|
12
|
+
* delegates to is 350, because that engine carries fixes this copy never got:
|
|
13
|
+
*
|
|
14
|
+
* - `onstart` as the real capture signal. `isListening` set after `start()`
|
|
15
|
+
* returns only means the request was accepted, so the button reported
|
|
16
|
+
* "listening" while the microphone was still closed.
|
|
17
|
+
* - `isStarting`, cleared on every terminal path — the permission window used to
|
|
18
|
+
* leave the control dead with no way to recover but a reload.
|
|
19
|
+
* - `no-speech` and the other transient recogniser errors treated as recoverable
|
|
20
|
+
* rather than fatal.
|
|
21
|
+
*
|
|
22
|
+
* Those were fixed in one product weeks ago. Every other product kept the broken
|
|
23
|
+
* button until this wrapper landed.
|
|
24
|
+
*
|
|
25
|
+
* Deprecated in favour of importing the engine directly: the callback contract
|
|
26
|
+
* loses nothing and avoids accumulating a string the caller may not want.
|
|
8
27
|
*/
|
|
9
28
|
export interface UseVoiceInputReturn {
|
|
10
29
|
isSupported: boolean;
|
|
11
30
|
isListening: boolean;
|
|
12
|
-
/** Final transcript accumulated since the last `reset()`. */
|
|
13
31
|
transcript: string;
|
|
14
32
|
start: () => void;
|
|
15
33
|
stop: () => void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"useVoiceInput.d.ts","sourceRoot":"","sources":["../../../../src/shared/components/assistantWidget/useVoiceInput.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"useVoiceInput.d.ts","sourceRoot":"","sources":["../../../../src/shared/components/assistantWidget/useVoiceInput.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,OAAO,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,IAAI,EAAE,MAAM,IAAI,CAAC;IACjB,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB;AAED,wBAAgB,aAAa,CAAC,IAAI,SAAU,GAAG,mBAAmB,CA4BjE"}
|
|
@@ -1,41 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { useVoiceInput as e } from "../../assistant/voice/useVoiceInput.js";
|
|
2
|
+
import { useCallback as t, useRef as n, useState as r } from "react";
|
|
2
3
|
//#region src/shared/components/assistantWidget/useVoiceInput.ts
|
|
3
|
-
function i() {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
return
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if (!d || u.current) return;
|
|
18
|
-
let e = new d();
|
|
19
|
-
e.continuous = !0, e.interimResults = !0, e.lang = a, e.onresult = (e) => {
|
|
20
|
-
let t = "";
|
|
21
|
-
for (let n = e.resultIndex; n < e.results.length; n++) {
|
|
22
|
-
let r = e.results[n];
|
|
23
|
-
r.isFinal && (t += r[0].transcript);
|
|
24
|
-
}
|
|
25
|
-
t && l((e) => e ? `${e} ${t}` : t);
|
|
26
|
-
}, e.onerror = () => {
|
|
27
|
-
s(!1), u.current = null;
|
|
28
|
-
}, e.onend = () => {
|
|
29
|
-
s(!1), u.current = null;
|
|
30
|
-
}, u.current = e, e.start(), s(!0);
|
|
31
|
-
}, [d, a]),
|
|
32
|
-
stop: e(() => {
|
|
33
|
-
u.current &&= (u.current.stop(), null), s(!1);
|
|
34
|
-
}, []),
|
|
35
|
-
reset: e(() => {
|
|
36
|
-
l("");
|
|
37
|
-
}, [])
|
|
4
|
+
function i(i = "en-US") {
|
|
5
|
+
let [a, o] = r(""), s = n((e) => {
|
|
6
|
+
o((t) => t ? `${t} ${e}` : e);
|
|
7
|
+
}), c = e({
|
|
8
|
+
onFinalize: (e) => s.current(e),
|
|
9
|
+
language: i
|
|
10
|
+
}), l = t(() => o(""), []);
|
|
11
|
+
return {
|
|
12
|
+
isSupported: c.isSupported,
|
|
13
|
+
isListening: c.isActive,
|
|
14
|
+
transcript: a,
|
|
15
|
+
start: c.start,
|
|
16
|
+
stop: c.stop,
|
|
17
|
+
reset: l
|
|
38
18
|
};
|
|
39
19
|
}
|
|
40
20
|
//#endregion
|
|
41
|
-
export {
|
|
21
|
+
export { i as useVoiceInput };
|
package/dist/shared-assistant.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
|
|
1
|
+
import { useVoiceInput as e } from "./shared/assistant/voice/useVoiceInput.js";
|
|
2
|
+
import { isQuotaExhaustedError as t } from "./shared/assistant/gatewayClient.js";
|
|
3
|
+
import { buildPromptRequestBody as n, checkAssistantAiCredits as r, createAssistantSandbox as i, createAssistantSession as a, downloadAssistantArtifact as o, extendAssistantSandboxTTL as s, findExistingSandbox as c, getAssistantArtifacts as l, getAssistantMessages as u, getAssistantSession as d, getAssistantStreamAccess as f, reportAssistantAiTokenUsage as p, sendAssistantPromptAsync as m, sendHeartbeat as h, waitForAssistantServiceReady as g, waitForSandboxRunning as _ } from "./shared/assistant/api.js";
|
|
4
|
+
import { AssistantRuntimeManager as v } from "./shared/assistant/runtime.js";
|
|
5
|
+
import { buildProgress as y, getAssistantMedia as b, isRecoverable as x, mapSessionToHistory as S, mergeMessagesById as C, sanitize as w } from "./shared/assistant/progress.js";
|
|
6
|
+
import { createSandboxAssistantTransport as T } from "./shared/assistant/createSandboxAssistantTransport.js";
|
|
7
|
+
import { useSandboxAssistantTransport as E } from "./shared/assistant/useSandboxAssistantTransport.js";
|
|
8
|
+
import { chatTurns as D, groupBySession as O, groupConsecutiveBySession as k, isVoiceTurn as A, voiceMarker as j, voiceSessionIdOf as M, voiceTurnsOnly as N } from "./shared/assistant/voice/turnVisibility.js";
|
|
9
|
+
export { v as AssistantRuntimeManager, y as buildProgress, n as buildPromptRequestBody, D as chatTurns, r as checkAssistantAiCredits, i as createAssistantSandbox, a as createAssistantSession, T as createSandboxAssistantTransport, o as downloadAssistantArtifact, s as extendAssistantSandboxTTL, c as findExistingSandbox, l as getAssistantArtifacts, b as getAssistantMedia, u as getAssistantMessages, d as getAssistantSession, f as getAssistantStreamAccess, O as groupBySession, k as groupConsecutiveBySession, t as isQuotaExhaustedError, x as isRecoverable, A as isVoiceTurn, S as mapSessionToHistory, C as mergeMessagesById, p as reportAssistantAiTokenUsage, w as sanitize, m as sendAssistantPromptAsync, h as sendHeartbeat, E as useSandboxAssistantTransport, e as useVoiceInput, j as voiceMarker, M as voiceSessionIdOf, N as voiceTurnsOnly, g as waitForAssistantServiceReady, _ as waitForSandboxRunning };
|