@hartren/assistant-js 0.0.0-stage → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +140 -0
- package/LICENSE +21 -0
- package/README.md +273 -2
- package/THIRD_PARTY_NOTICES.md +917 -0
- package/dist/CopilotAgent/AuthenticationService.d.ts +47 -0
- package/dist/CopilotAgent/CommandsManager.d.ts +20 -0
- package/dist/CopilotAgent/CopilotAgent.d.ts +148 -0
- package/dist/CopilotAgent/CopilotProxy.d.ts +17 -0
- package/dist/CopilotAgent/UserSessionService.d.ts +9 -0
- package/dist/CopilotAgent/index.d.ts +2 -0
- package/dist/CustomSpeechRecognition/CustomSpeechRecognition.d.ts +93 -0
- package/dist/MicrophoneAccess/MicrophoneAccess.d.ts +65 -0
- package/dist/SpeakingService/SpeakingService.d.ts +14 -0
- package/dist/SpeakingService/index.d.ts +2 -0
- package/dist/SpeechRecognitionService/ResultsGroupingService.d.ts +44 -0
- package/dist/SpeechRecognitionService/SpeechRecognitionService.d.ts +108 -0
- package/dist/SpeechRecognitionService/index.d.ts +2 -0
- package/dist/SpeechRecognitionService/speechConfig.d.ts +96 -0
- package/dist/debugLog.d.ts +9 -0
- package/dist/enums.d.ts +42 -0
- package/dist/httpError.d.ts +9 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.js +2 -0
- package/dist/index.js.LICENSE.txt +5 -0
- package/dist/types.d.ts +57 -0
- package/package.json +54 -5
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/** Host-supplied token provider: called for every request; resolves the current user token. */
|
|
2
|
+
export type AccessTokenProvider = () => Promise<string | null | undefined> | string | null | undefined;
|
|
3
|
+
/** Called after the API answered 401 for `rejectedToken`; resolves a different token to retry with, or null. */
|
|
4
|
+
export type UnauthorizedHandler = (rejectedToken: string) => Promise<string | null | undefined> | string | null | undefined;
|
|
5
|
+
/**
|
|
6
|
+
* Token handling for the API calls and the speech socket.
|
|
7
|
+
*
|
|
8
|
+
* Preferred: the host passes `getAccessToken` (for example `() => auth.getAccessToken()` from
|
|
9
|
+
* `@hartren/auth`) and optionally `onUnauthorized`. The library then never exchanges or refreshes
|
|
10
|
+
* anything itself: it asks the provider on every request and keeps no copy of the token.
|
|
11
|
+
*
|
|
12
|
+
* Deprecated (to be removed in a future major version): `pat`, the browser PAT exchange (`/token/exchange`)
|
|
13
|
+
* and the legacy refresh (`/token/refresh`). They run only when a PAT is passed and the provider
|
|
14
|
+
* is missing or returns nothing.
|
|
15
|
+
*/
|
|
16
|
+
export declare class AuthenticationService {
|
|
17
|
+
private pat;
|
|
18
|
+
private accessToken;
|
|
19
|
+
private refreshToken;
|
|
20
|
+
private tokenExpiration;
|
|
21
|
+
private refreshTimeout;
|
|
22
|
+
private pendingRefreshPromise;
|
|
23
|
+
private pendingExchangePromise;
|
|
24
|
+
private _baseUrl;
|
|
25
|
+
private _userId;
|
|
26
|
+
private _getAccessToken;
|
|
27
|
+
private _onUnauthorized;
|
|
28
|
+
constructor(copilotApi: any, pat: any, getAccessToken?: AccessTokenProvider | null, onUnauthorized?: UnauthorizedHandler | null);
|
|
29
|
+
/** Deprecated: the userId for the PAT exchange. Does nothing without a PAT. */
|
|
30
|
+
setUserId(userId: string | null): void;
|
|
31
|
+
getAccessToken(): Promise<string>;
|
|
32
|
+
/**
|
|
33
|
+
* After a 401 for `rejectedToken`: resolves a different token to retry with once, or null.
|
|
34
|
+
* Provider mode asks `onUnauthorized` (or, without it, the provider again); legacy mode drops the
|
|
35
|
+
* cached token and refreshes or re-exchanges.
|
|
36
|
+
*/
|
|
37
|
+
handleUnauthorized(rejectedToken: string): Promise<string | null>;
|
|
38
|
+
/** One PAT exchange shared by concurrent callers. */
|
|
39
|
+
private exchangeOnce;
|
|
40
|
+
private exchangePatForTokens;
|
|
41
|
+
private refreshTokens;
|
|
42
|
+
private performTokenRefresh;
|
|
43
|
+
private handleTokenResponse;
|
|
44
|
+
private scheduleProactiveRefresh;
|
|
45
|
+
private isTokenExpired;
|
|
46
|
+
private clearRefreshTimer;
|
|
47
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
export declare class CommandsManager {
|
|
2
|
+
private commands;
|
|
3
|
+
constructor({}?: {});
|
|
4
|
+
/**
|
|
5
|
+
* Register a new command with the command manager
|
|
6
|
+
*
|
|
7
|
+
* @method
|
|
8
|
+
* @param {string} commandName - Unique name identifying the command
|
|
9
|
+
* @param {CommandDefinition} definition - {@link CommandDefinition}
|
|
10
|
+
*/
|
|
11
|
+
registerCommand(commandName: any, definition: any): void;
|
|
12
|
+
/**
|
|
13
|
+
*
|
|
14
|
+
* @method
|
|
15
|
+
* @param {String} commandName
|
|
16
|
+
* @param {Object} [options={}] - Extra options to pass the command. Like a mousedown event
|
|
17
|
+
*/
|
|
18
|
+
runCommand(commandName: string, options?: {}): any;
|
|
19
|
+
}
|
|
20
|
+
export default CommandsManager;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import type { AccessTokenProvider, UnauthorizedHandler } from './AuthenticationService';
|
|
2
|
+
import { CopilotActivityState, CopilotMode } from '../types';
|
|
3
|
+
import { SpeechErrorCode, SpeechProvider, SpeechRetryOptions } from '../SpeechRecognitionService/speechConfig';
|
|
4
|
+
import { MicrophoneEnvironment, SpeechRecognitionFailure } from '../MicrophoneAccess/MicrophoneAccess';
|
|
5
|
+
export interface CopilotWidgetState {
|
|
6
|
+
isListening?: boolean;
|
|
7
|
+
isSpeeking?: boolean;
|
|
8
|
+
isMuted?: boolean;
|
|
9
|
+
isEnabled?: boolean;
|
|
10
|
+
state?: string;
|
|
11
|
+
recognationState?: string;
|
|
12
|
+
mode?: CopilotMode;
|
|
13
|
+
activity?: CopilotActivityState;
|
|
14
|
+
}
|
|
15
|
+
export interface CopilotWidgetResult {
|
|
16
|
+
result?: string;
|
|
17
|
+
}
|
|
18
|
+
export interface CopilotError {
|
|
19
|
+
message: string;
|
|
20
|
+
error?: any;
|
|
21
|
+
stopAgent?: boolean;
|
|
22
|
+
asPopup?: boolean;
|
|
23
|
+
/**
|
|
24
|
+
* Speech errors: one of SpeechErrorCodes. Strategy errors from the opt-in browser provider
|
|
25
|
+
* carry a SpeechRecognitionFailure (e.g. 'mic-denied') instead.
|
|
26
|
+
*/
|
|
27
|
+
code?: SpeechErrorCode | SpeechRecognitionFailure;
|
|
28
|
+
/** Speech retries: attempt number (1-based) and the retry budget. */
|
|
29
|
+
attempt?: number;
|
|
30
|
+
maxRetries?: number;
|
|
31
|
+
retryInMs?: number;
|
|
32
|
+
/** Speech: true when no more retries will happen. */
|
|
33
|
+
final?: boolean;
|
|
34
|
+
/** Speech: what failed ('connect-failed', 'socket-closed', 'audio-setup-failed', ...). */
|
|
35
|
+
reason?: string;
|
|
36
|
+
/** Speech: WebSocket close code, when a socket closed. */
|
|
37
|
+
closeCode?: number;
|
|
38
|
+
/** Microphone failures: the browser's error name (e.g. 'NotAllowedError'), never its raw message. */
|
|
39
|
+
detail?: string;
|
|
40
|
+
/** Microphone failures: what the browser reported about microphone access. */
|
|
41
|
+
environment?: MicrophoneEnvironment;
|
|
42
|
+
}
|
|
43
|
+
export interface RecognitionResult {
|
|
44
|
+
transcript?: string;
|
|
45
|
+
timestamp?: number;
|
|
46
|
+
}
|
|
47
|
+
interface ConstructorParams {
|
|
48
|
+
commands: any;
|
|
49
|
+
copilotApi: any;
|
|
50
|
+
clientId?: string | null;
|
|
51
|
+
language?: string;
|
|
52
|
+
/** @deprecated Browser PAT exchange; pass `getAccessToken` instead. To be removed in a future major version. */
|
|
53
|
+
pat?: string;
|
|
54
|
+
threshold?: number;
|
|
55
|
+
defaultContext?: string;
|
|
56
|
+
/**
|
|
57
|
+
* The user's token for every API call and the speech socket, asked for on each request (no copy
|
|
58
|
+
* is kept), e.g. `() => auth.getAccessToken()` from `@hartren/auth`. Resolve null when signed out.
|
|
59
|
+
*/
|
|
60
|
+
getAccessToken?: AccessTokenProvider | null;
|
|
61
|
+
/**
|
|
62
|
+
* Optional. Called once when the API answers 401 for a token, e.g.
|
|
63
|
+
* `(t) => auth.handleUnauthorized(t)`. Resolve a different token to retry the request once, or null.
|
|
64
|
+
*/
|
|
65
|
+
onUnauthorized?: UnauthorizedHandler | null;
|
|
66
|
+
/**
|
|
67
|
+
* Origin of the hosted speech WebSockets (`/ws/speech/transcribe`, `/ws/speech/speak`).
|
|
68
|
+
* Defaults to the origin of `copilotApi` with a ws(s) scheme.
|
|
69
|
+
*/
|
|
70
|
+
transcribeWssUrl?: string | null;
|
|
71
|
+
mode?: string | null;
|
|
72
|
+
/**
|
|
73
|
+
* 'deepgram' (default): hosted transcription/TTS only. 'browser': explicit opt-in to the browser
|
|
74
|
+
* Web Speech APIs, which can send audio/text to the browser vendor (no BAA). Never automatic.
|
|
75
|
+
*/
|
|
76
|
+
speechProvider?: SpeechProvider | string | null;
|
|
77
|
+
/** Reconnect policy for the hosted transcription socket. */
|
|
78
|
+
speechRetry?: SpeechRetryOptions | null;
|
|
79
|
+
/**
|
|
80
|
+
* Default false. When true (hosted provider only), dictation switches to the browser's
|
|
81
|
+
* SpeechRecognition when the transcription socket cannot connect or drops mid-session while the
|
|
82
|
+
* browser is online, and returns to the hosted service on 'online' or when the next dictation
|
|
83
|
+
* starts. Offline behaves as without the option. WARNING: browser speech can send audio to the browser vendor (e.g. Google,
|
|
84
|
+
* Microsoft), with no BAA. Enable only where that is acceptable (never with PHI).
|
|
85
|
+
*/
|
|
86
|
+
browserTranscriptionFallback?: boolean | null;
|
|
87
|
+
/**
|
|
88
|
+
* Logs dictated text, prompts, AI replies and raw API responses to the console. Off by default:
|
|
89
|
+
* that text can be PHI. For local debugging only. Same as `setCopilotDebugLogging(true)`.
|
|
90
|
+
*/
|
|
91
|
+
debug?: boolean;
|
|
92
|
+
}
|
|
93
|
+
export default class CopilotAgent {
|
|
94
|
+
#private;
|
|
95
|
+
constructor({ commands, copilotApi, clientId, language, pat, threshold, getAccessToken, onUnauthorized, transcribeWssUrl, mode, speechProvider: requestedSpeechProvider, speechRetry, browserTranscriptionFallback, debug, }: ConstructorParams);
|
|
96
|
+
private _setActivity;
|
|
97
|
+
getOrStartSession(): Promise<any>;
|
|
98
|
+
setSession(sessionId: string): void;
|
|
99
|
+
registerCommands(commands?: object): void;
|
|
100
|
+
private engageCopilot;
|
|
101
|
+
private executeTools;
|
|
102
|
+
addMessage(newMessage: string): void;
|
|
103
|
+
private _processAndSendMessage;
|
|
104
|
+
private _handleResponse;
|
|
105
|
+
sendMessage(message: any, clearLast: any): Promise<void>;
|
|
106
|
+
startListening(): void;
|
|
107
|
+
stopListening(forceStop?: boolean): void;
|
|
108
|
+
getState(): {
|
|
109
|
+
isListening: boolean;
|
|
110
|
+
isSpeeking: boolean;
|
|
111
|
+
isMuted: boolean;
|
|
112
|
+
isEnabled: boolean;
|
|
113
|
+
state: string;
|
|
114
|
+
recognationState: string;
|
|
115
|
+
mode: CopilotMode;
|
|
116
|
+
activity: CopilotActivityState;
|
|
117
|
+
};
|
|
118
|
+
/** Returns the current operating mode of the agent. */
|
|
119
|
+
getMode(): CopilotMode;
|
|
120
|
+
/** True when the agent is running in silent mode (never speaks). */
|
|
121
|
+
isSilent(): boolean;
|
|
122
|
+
/** True when the agent is in minimal mode (backend content mode; no local speech change). */
|
|
123
|
+
isMinimal(): boolean;
|
|
124
|
+
/** @deprecated Sets the userId used for the PAT exchange; does nothing without a PAT. To be removed in a future major version. */
|
|
125
|
+
setUserId(userId: string | null): void;
|
|
126
|
+
setMode(mode?: string | null): void;
|
|
127
|
+
private _broadCastEvt;
|
|
128
|
+
private _onCopilotResult;
|
|
129
|
+
/** Which speech provider this agent uses ('deepgram' unless the host opted into 'browser'). */
|
|
130
|
+
getSpeechProvider(): SpeechProvider;
|
|
131
|
+
/** True while dictation runs on the browser fallback (only with browserTranscriptionFallback: true). */
|
|
132
|
+
isUsingBrowserTranscriptionFallback(): boolean;
|
|
133
|
+
speak(message: string, onCompleteEvt?: any): Promise<void>;
|
|
134
|
+
mute(): void;
|
|
135
|
+
unMute(): void;
|
|
136
|
+
isEnabled(): boolean;
|
|
137
|
+
enable(enabled?: boolean): void;
|
|
138
|
+
onStateChange(listener: (state: CopilotWidgetState) => void): void;
|
|
139
|
+
onCopilotResult(listener: (state: CopilotWidgetResult) => void): void;
|
|
140
|
+
onCopilotError(listener: (state: CopilotError) => void): void;
|
|
141
|
+
onRecognitionResult(listener: (result: RecognitionResult) => void): void;
|
|
142
|
+
onRecognitionSilent(listener: (result: {
|
|
143
|
+
isSilent: boolean;
|
|
144
|
+
}) => void): void;
|
|
145
|
+
stopSession(): void;
|
|
146
|
+
getAccessToken(): Promise<any>;
|
|
147
|
+
}
|
|
148
|
+
export {};
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type AccessTokenProvider, type UnauthorizedHandler } from "./AuthenticationService";
|
|
2
|
+
import type { CopilotMode } from "../types";
|
|
3
|
+
export default class CopilotProxy {
|
|
4
|
+
private _session;
|
|
5
|
+
private _baseUrl;
|
|
6
|
+
private _AuthenticationService;
|
|
7
|
+
private _clientId?;
|
|
8
|
+
constructor(copilotApi: any, pat: any, getAccessToken?: AccessTokenProvider | null, clientId?: string | null, onUnauthorized?: UnauthorizedHandler | null);
|
|
9
|
+
setUserId(userId: string | null): void;
|
|
10
|
+
/** Creates a session. Mode is not sent here — it is applied per ViewerCopilot turn. */
|
|
11
|
+
startSession(): Promise<any>;
|
|
12
|
+
stopSession(): Promise<void>;
|
|
13
|
+
/** Sends a ViewerCopilot turn. `mode` is per-turn so the backend applies the right policy. */
|
|
14
|
+
engageCopilot(session: string, prompt: string, type: string, tools: any[], fromVoice: boolean, clearLast?: boolean, mode?: CopilotMode): Promise<any>;
|
|
15
|
+
private _callApi;
|
|
16
|
+
getAccessToken(): Promise<any>;
|
|
17
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
import { MicVAD } from '@ricky0123/vad-web';
|
|
3
|
+
import { SpeechRetryOptions } from '../SpeechRecognitionService/speechConfig';
|
|
4
|
+
export default class CustomSpeechRecognition {
|
|
5
|
+
lang: string;
|
|
6
|
+
continuous: boolean;
|
|
7
|
+
interimResults: boolean;
|
|
8
|
+
maxAlternatives: number;
|
|
9
|
+
onresult: ((event?: any) => void) | null;
|
|
10
|
+
onerror: ((event?: any) => void) | null;
|
|
11
|
+
onstart: (() => void) | null;
|
|
12
|
+
onspeechstart: (() => void) | null;
|
|
13
|
+
onspeechend: (() => void) | null;
|
|
14
|
+
onend: (() => void) | null;
|
|
15
|
+
socket: WebSocket | null;
|
|
16
|
+
socketUrl: string | null;
|
|
17
|
+
vad: MicVAD | null;
|
|
18
|
+
isRecording: boolean;
|
|
19
|
+
sessionService: {
|
|
20
|
+
getOrStartSession: () => Promise<string>;
|
|
21
|
+
} | null;
|
|
22
|
+
events: EventEmitter;
|
|
23
|
+
copilotIsSpeeking: boolean;
|
|
24
|
+
stream: MediaStream | null;
|
|
25
|
+
transcribeWssUrl: string | null;
|
|
26
|
+
private _abortAudioSetup;
|
|
27
|
+
private _hadTranscriptionPayload;
|
|
28
|
+
private _userRequestedStop;
|
|
29
|
+
private _activeTtsContext;
|
|
30
|
+
private _forceSpeakSettle;
|
|
31
|
+
private _getAccessToken;
|
|
32
|
+
private _retry;
|
|
33
|
+
private _retryAttempt;
|
|
34
|
+
private _retryTimer;
|
|
35
|
+
private _connecting;
|
|
36
|
+
private _openedAt;
|
|
37
|
+
/** Bumped by every connect and by stop(); callbacks from an older connect are ignored. */
|
|
38
|
+
private _connectGen;
|
|
39
|
+
/** Microphone start failures in a row; a repeat failure with an unknown permission state is a block. */
|
|
40
|
+
private _failedAttempts;
|
|
41
|
+
/**
|
|
42
|
+
* Optional hook asked on every hosted connection failure before a retry is scheduled. Returning
|
|
43
|
+
* true means the caller took over (browserTranscriptionFallback); no retry or error is raised here.
|
|
44
|
+
*/
|
|
45
|
+
private _onConnectionFailure;
|
|
46
|
+
/** A connection that stayed open at least this long was healthy: its close gets a fresh retry budget. */
|
|
47
|
+
static HEALTHY_CONNECTION_MS: number;
|
|
48
|
+
constructor(transcribeWssUrl?: string, options?: {
|
|
49
|
+
getAccessToken?: (() => Promise<string> | string | null | undefined) | null;
|
|
50
|
+
retry?: SpeechRetryOptions | null;
|
|
51
|
+
onConnectionFailure?: ((failure: {
|
|
52
|
+
reason: string;
|
|
53
|
+
closeCode?: number;
|
|
54
|
+
}) => boolean) | null;
|
|
55
|
+
});
|
|
56
|
+
init({ sessionService, events }: {
|
|
57
|
+
sessionService: any;
|
|
58
|
+
events: EventEmitter;
|
|
59
|
+
}): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* URL `${transcribeWssUrl}${path}?sessionId=...` (no token) and the subprotocols. Browsers cannot set
|
|
62
|
+
* headers on a WebSocket, so the access token goes in the Sec-WebSocket-Protocol header as
|
|
63
|
+
* `bearer.<token>` next to `hartren.speech.v1`, which the Hartren API echoes. The token is never put in
|
|
64
|
+
* the URL, so it cannot reach access logs. Throws when the session or token cannot be obtained
|
|
65
|
+
* (handled as a connection failure).
|
|
66
|
+
*/
|
|
67
|
+
private _speechSocket;
|
|
68
|
+
start(): Promise<void>;
|
|
69
|
+
/** Closes and forgets the current socket without running its handlers (they check the generation). */
|
|
70
|
+
private _detachSocket;
|
|
71
|
+
private _connect;
|
|
72
|
+
/**
|
|
73
|
+
* Retries the hosted transcription with exponential backoff, emitting an error event on each
|
|
74
|
+
* attempt, then a final error. Browser speech is used instead only when the host opted into
|
|
75
|
+
* `browserTranscriptionFallback` (the onConnectionFailure hook takes over); never by default.
|
|
76
|
+
*/
|
|
77
|
+
private _handleConnectionFailure;
|
|
78
|
+
/** Configuration problem (refused URL): a final error, no retry, no socket. */
|
|
79
|
+
private _reportNotConfigured;
|
|
80
|
+
/** A final error that no retry can fix: report once, stop, no retry. */
|
|
81
|
+
private _reportFinal;
|
|
82
|
+
private _teardownAudio;
|
|
83
|
+
stop(): void;
|
|
84
|
+
abort(): void;
|
|
85
|
+
/**
|
|
86
|
+
* Immediately cancel any in-progress WS TTS playback.
|
|
87
|
+
* Closes the AudioContext (stopping all scheduled sources) and force-settles the
|
|
88
|
+
* pending speak() Promise so that SpeakinEnded is emitted and control returns to
|
|
89
|
+
* the caller. Safe to call when no speech is active.
|
|
90
|
+
*/
|
|
91
|
+
stopSpeaking(): void;
|
|
92
|
+
speak(text: any): Promise<void>;
|
|
93
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
export type SpeechRecognitionFailure = 'mic-denied' | 'mic-dismissed' | 'mic-os-blocked' | 'mic-policy-blocked' | 'mic-no-device' | 'mic-constraints' | 'mic-busy' | 'mic-unsupported' | 'mic-insecure-context' | 'mic-failed' | 'session-failed' | 'socket-failed' | 'vad-asset-failed' | 'engine-unavailable';
|
|
2
|
+
/** Whether the browser will say what it thinks of the microphone. Not every browser will. */
|
|
3
|
+
export type MicrophonePermission = 'granted' | 'denied' | 'prompt' | 'unknown';
|
|
4
|
+
export interface MicrophoneEnvironment {
|
|
5
|
+
secureContext: boolean;
|
|
6
|
+
hasMediaDevices: boolean;
|
|
7
|
+
embedded: boolean;
|
|
8
|
+
policyAllows: boolean | null;
|
|
9
|
+
permission: MicrophonePermission;
|
|
10
|
+
}
|
|
11
|
+
export interface FailureFacts {
|
|
12
|
+
message: string;
|
|
13
|
+
stopAgent: boolean;
|
|
14
|
+
popup: boolean;
|
|
15
|
+
retryable: boolean;
|
|
16
|
+
}
|
|
17
|
+
export declare const SPEECH_RECOGNITION_FAILURES: Record<SpeechRecognitionFailure, FailureFacts>;
|
|
18
|
+
export declare class SpeechRecognitionStartError extends Error {
|
|
19
|
+
readonly failure: SpeechRecognitionFailure;
|
|
20
|
+
readonly originalError: any;
|
|
21
|
+
constructor(failure: SpeechRecognitionFailure, originalError?: any);
|
|
22
|
+
}
|
|
23
|
+
export declare function isEmbedded(): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Whether this page is allowed to ask for the microphone at all, according to the embedding page's
|
|
26
|
+
* permissions policy.
|
|
27
|
+
*
|
|
28
|
+
* This is the probe that separates "the user blocked it" from "the page that embeds us blocks it".
|
|
29
|
+
* Both arrive from `getUserMedia` as an identical `NotAllowedError`, so without this the two are
|
|
30
|
+
* indistinguishable and the user is sent to a browser setting that is already correct - advice they
|
|
31
|
+
* cannot act on, for a decision only the embedding page can reverse.
|
|
32
|
+
*
|
|
33
|
+
* `null` means the browser cannot answer, which is a real and common case; callers must treat it as
|
|
34
|
+
* "unknown" and never as permission granted.
|
|
35
|
+
*/
|
|
36
|
+
export declare function readPolicyAllowsMicrophone(): boolean | null;
|
|
37
|
+
/**
|
|
38
|
+
* What the browser thinks of the microphone permission.
|
|
39
|
+
*
|
|
40
|
+
* Never load-bearing. Firefox throws `TypeError` for the `microphone` name, and older browsers have
|
|
41
|
+
* no Permissions API at all, so `'unknown'` is an expected answer rather than an error, and every
|
|
42
|
+
* caller has to stay correct without it.
|
|
43
|
+
*/
|
|
44
|
+
export declare function readMicrophonePermission(): Promise<MicrophonePermission>;
|
|
45
|
+
/**
|
|
46
|
+
* Everything this browser will say about its microphone, in the form the classifier reads.
|
|
47
|
+
*
|
|
48
|
+
* The permission is the one part that has to be awaited, and for a policy-blocked frame it is not
|
|
49
|
+
* even asked: Chrome reports `denied` there, which would name the user's own settings as the cause
|
|
50
|
+
* of a decision the embedding page made.
|
|
51
|
+
*/
|
|
52
|
+
export declare function readMicrophoneEnvironment(): Promise<MicrophoneEnvironment>;
|
|
53
|
+
export interface ClassifyOptions {
|
|
54
|
+
/**
|
|
55
|
+
* True when an attempt in this session has already failed.
|
|
56
|
+
*
|
|
57
|
+
* This exists for the browsers that cannot report their permission state - Firefox above all. On
|
|
58
|
+
* a first failure an unknown state is best read as a dismissed prompt, because offering a retry
|
|
59
|
+
* that works costs nothing. On a repeat failure the same evidence now points the other way: a
|
|
60
|
+
* block that survives an explicit retry is a block, and continuing to offer "Try Again" would
|
|
61
|
+
* trap the user in a loop that can never succeed.
|
|
62
|
+
*/
|
|
63
|
+
repeatAttempt?: boolean;
|
|
64
|
+
}
|
|
65
|
+
export declare function classifyMicrophoneFailure(error: unknown, environment: MicrophoneEnvironment, options?: ClassifyOptions): SpeechRecognitionFailure;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
export declare enum stateEvents {
|
|
3
|
+
SpeakinStarted = "speakingStarted",
|
|
4
|
+
SpeakinEnded = "speakingEnded"
|
|
5
|
+
}
|
|
6
|
+
export default class SpeakingService {
|
|
7
|
+
isListening: boolean;
|
|
8
|
+
isMuted: boolean;
|
|
9
|
+
private _isActive;
|
|
10
|
+
private _events;
|
|
11
|
+
constructor(events: EventEmitter);
|
|
12
|
+
speak(textOrAudio: string, onCompleteEvt?: () => void): void;
|
|
13
|
+
isActive(): boolean;
|
|
14
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export type Result = {
|
|
2
|
+
transcript: string;
|
|
3
|
+
timestamp: number;
|
|
4
|
+
};
|
|
5
|
+
declare class ResultsGroupingService {
|
|
6
|
+
private resultsBuffer;
|
|
7
|
+
private onResultsReadyCallback;
|
|
8
|
+
private threshold;
|
|
9
|
+
private executionTimeout;
|
|
10
|
+
constructor(threshold: number);
|
|
11
|
+
/**
|
|
12
|
+
* Adds a new result to the buffer and initiates execution.
|
|
13
|
+
* @param result - The result to add.
|
|
14
|
+
*/
|
|
15
|
+
addResult(result: Result): void;
|
|
16
|
+
/**
|
|
17
|
+
* Sets the callback to be called when results are ready.
|
|
18
|
+
* @param callback - The callback function to execute with grouped results.
|
|
19
|
+
*/
|
|
20
|
+
onResultsReady(callback: (results: string[]) => void): void;
|
|
21
|
+
/**
|
|
22
|
+
* Sends the recognized results to the callback.
|
|
23
|
+
*/
|
|
24
|
+
private sendRecognizedResult;
|
|
25
|
+
/**
|
|
26
|
+
* Initiates the execution timeout.
|
|
27
|
+
*/
|
|
28
|
+
private initiateExecution;
|
|
29
|
+
/**
|
|
30
|
+
* Delays execution by clearing the existing timeout and re-initiating it.
|
|
31
|
+
*/
|
|
32
|
+
private delayExecution;
|
|
33
|
+
/**
|
|
34
|
+
* Cleans up resources and sends any remaining results.
|
|
35
|
+
*/
|
|
36
|
+
cleanup(): void;
|
|
37
|
+
/**
|
|
38
|
+
* Formats a timestamp into a human-readable string.
|
|
39
|
+
* @param ms - The timestamp in milliseconds.
|
|
40
|
+
* @returns The formatted time string.
|
|
41
|
+
*/
|
|
42
|
+
formatTime(ms: number): string;
|
|
43
|
+
}
|
|
44
|
+
export default ResultsGroupingService;
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
import UserSessionService from '../CopilotAgent/UserSessionService';
|
|
3
|
+
import { SpeechProvider, SpeechRetryOptions } from './speechConfig';
|
|
4
|
+
export declare enum stateEvents {
|
|
5
|
+
StateChange = "recognationStateChange"
|
|
6
|
+
}
|
|
7
|
+
interface SpeechRecognitionServiceConfig {
|
|
8
|
+
language?: string;
|
|
9
|
+
threshold?: number;
|
|
10
|
+
events: EventEmitter;
|
|
11
|
+
sessionService: UserSessionService;
|
|
12
|
+
transcribeWssUrl?: string;
|
|
13
|
+
/** 'deepgram' (default, hosted) or 'browser' (explicit opt-in to Web Speech). */
|
|
14
|
+
speechProvider?: SpeechProvider | string | null;
|
|
15
|
+
/** Bearer token for the speech WebSocket handshake (sent as the `bearer.<token>` subprotocol, never in the URL). */
|
|
16
|
+
getAccessToken?: (() => Promise<string> | string | null | undefined) | null;
|
|
17
|
+
speechRetry?: SpeechRetryOptions | null;
|
|
18
|
+
/**
|
|
19
|
+
* Opt-in (default false): when the hosted transcription socket cannot connect or drops
|
|
20
|
+
* mid-session (while the browser reports itself online), dictate with the browser's
|
|
21
|
+
* SpeechRecognition until the connection is back. Browser speech can send audio to the browser vendor: only where that is acceptable.
|
|
22
|
+
*/
|
|
23
|
+
browserTranscriptionFallback?: boolean | null;
|
|
24
|
+
}
|
|
25
|
+
export default class SpeechRecognitionService {
|
|
26
|
+
private recognition;
|
|
27
|
+
private _wsTtsRecognition;
|
|
28
|
+
private resultsGroupingService;
|
|
29
|
+
private isRecognitionInProgress;
|
|
30
|
+
private stoppedByUser;
|
|
31
|
+
private _isActive;
|
|
32
|
+
private copilotIsSpeeking;
|
|
33
|
+
private _events;
|
|
34
|
+
private _language;
|
|
35
|
+
private _provider;
|
|
36
|
+
private _unavailable;
|
|
37
|
+
/** Why hosted TTS is unavailable, when it is because the speech URL was refused. */
|
|
38
|
+
private _ttsUnavailable;
|
|
39
|
+
/** Hosted recognizer (kept while the browser fallback is in use, so dictation can hand back). */
|
|
40
|
+
private _hostedRecognition;
|
|
41
|
+
/** browserTranscriptionFallback: true only when the host explicitly opted in. */
|
|
42
|
+
private _fallbackEnabled;
|
|
43
|
+
/** Browser SpeechRecognition for the fallback, constructed lazily on the first failure only. */
|
|
44
|
+
private _browserFallback;
|
|
45
|
+
private _usingBrowserFallback;
|
|
46
|
+
private _fallbackUnsupportedLogged;
|
|
47
|
+
/** window 'online' listener, attached only while a dictation with the fallback is in progress. */
|
|
48
|
+
private _networkListenersAttached;
|
|
49
|
+
private readonly _onOnline;
|
|
50
|
+
/**
|
|
51
|
+
* The fallback is active but the browser is offline: Web Speech (cloud-based in Chrome/Edge) can
|
|
52
|
+
* only fail with 'network', so it is paused (no restarts, no popups) until 'online'.
|
|
53
|
+
*/
|
|
54
|
+
private _fallbackPausedOffline;
|
|
55
|
+
private _listenerLastResultTime;
|
|
56
|
+
private _listenerResultTimeout;
|
|
57
|
+
constructor({ events, language, threshold, sessionService, transcribeWssUrl, speechProvider: provider, getAccessToken, speechRetry, browserTranscriptionFallback, }: SpeechRecognitionServiceConfig);
|
|
58
|
+
/** True while dictation runs on the browser fallback (browserTranscriptionFallback only). */
|
|
59
|
+
isUsingBrowserFallback(): boolean;
|
|
60
|
+
private isOffline;
|
|
61
|
+
private setNetworkListeners;
|
|
62
|
+
/** The fallback recognizer, built on first use. Null when the browser has no SpeechRecognition. */
|
|
63
|
+
private getBrowserFallback;
|
|
64
|
+
/**
|
|
65
|
+
* browserTranscriptionFallback: moves the dictation in progress from the hosted socket to the
|
|
66
|
+
* browser's SpeechRecognition. Returns false (the hosted recognizer then retries as usual) when the
|
|
67
|
+
* option is off, no dictation is in progress, the browser is offline (cloud Web Speech cannot work
|
|
68
|
+
* either, so offline behaves exactly as without the option), or there is no SpeechRecognition.
|
|
69
|
+
*/
|
|
70
|
+
private switchToBrowserFallback;
|
|
71
|
+
/** Leaves the browser fallback and makes the hosted recognizer current again (does not start it). */
|
|
72
|
+
private handBackToHosted;
|
|
73
|
+
/**
|
|
74
|
+
* While the fallback is active and the browser is offline, pauses it instead of restarting:
|
|
75
|
+
* returns true when the caller must not restart or report. Logged once per pause (fixed text).
|
|
76
|
+
*/
|
|
77
|
+
private pauseFallbackIfOffline;
|
|
78
|
+
/** Connection is back: hand the dictation in progress back to the hosted service. */
|
|
79
|
+
private handleOnline;
|
|
80
|
+
getProvider(): SpeechProvider;
|
|
81
|
+
/** True when replies can be spoken through the hosted /ws/speech/speak endpoint. */
|
|
82
|
+
hasHostedTts(): boolean;
|
|
83
|
+
/** Message and reason for tts-not-configured when hosted TTS was refused (e.g. remote ws://). */
|
|
84
|
+
getTtsUnavailable(): {
|
|
85
|
+
message: string;
|
|
86
|
+
reason: string;
|
|
87
|
+
} | null;
|
|
88
|
+
private bindEngineListeners;
|
|
89
|
+
private clearSilentListenerTimeout;
|
|
90
|
+
private setActive;
|
|
91
|
+
startRecognition(onTextRecognized: (results: string[]) => void): void;
|
|
92
|
+
stopRecognition(forceStop?: boolean): void;
|
|
93
|
+
private safeRestartRecognition;
|
|
94
|
+
private handleRecognitionError;
|
|
95
|
+
isActive(): boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Speaks text via the hosted /ws/speech/speak endpoint. Returns false when there is no hosted
|
|
98
|
+
* TTS (the caller decides; CopilotAgent never falls back to browser TTS unless the host chose
|
|
99
|
+
* speechProvider 'browser'). Rejects when hosted TTS fails.
|
|
100
|
+
*/
|
|
101
|
+
speak(text: string, onCompleteEvt?: () => void): Promise<boolean>;
|
|
102
|
+
/**
|
|
103
|
+
* Stops any active WS TTS playback immediately.
|
|
104
|
+
* No-op when using native browser speech recognition.
|
|
105
|
+
*/
|
|
106
|
+
stopSpeaking(): void;
|
|
107
|
+
}
|
|
108
|
+
export {};
|