@capgo/capacitor-speech-recognition 8.0.9 → 8.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -20
- package/android/src/main/java/app/capgo/speechrecognition/Constants.java +2 -0
- package/android/src/main/java/app/capgo/speechrecognition/SpeechRecognitionPlugin.java +983 -137
- package/dist/docs.json +392 -21
- package/dist/esm/definitions.d.ts +178 -3
- package/dist/esm/definitions.js.map +1 -1
- package/dist/esm/web.d.ts +5 -1
- package/dist/esm/web.js +12 -0
- package/dist/esm/web.js.map +1 -1
- package/dist/plugin.cjs.js +12 -0
- package/dist/plugin.cjs.js.map +1 -1
- package/dist/plugin.js +12 -0
- package/dist/plugin.js.map +1 -1
- package/ios/Sources/SpeechRecognitionPlugin/SpeechAnalyzerRecognitionSession.swift +437 -0
- package/ios/Sources/SpeechRecognitionPlugin/SpeechRecognitionPlugin.swift +622 -70
- package/package.json +2 -1
|
@@ -5,7 +5,12 @@ import android.app.Activity;
|
|
|
5
5
|
import android.content.Intent;
|
|
6
6
|
import android.os.Build;
|
|
7
7
|
import android.os.Bundle;
|
|
8
|
+
import android.os.Handler;
|
|
9
|
+
import android.os.Looper;
|
|
10
|
+
import android.speech.ModelDownloadListener;
|
|
8
11
|
import android.speech.RecognitionListener;
|
|
12
|
+
import android.speech.RecognitionSupport;
|
|
13
|
+
import android.speech.RecognitionSupportCallback;
|
|
9
14
|
import android.speech.RecognizerIntent;
|
|
10
15
|
import android.speech.SpeechRecognizer;
|
|
11
16
|
import androidx.activity.result.ActivityResult;
|
|
@@ -23,6 +28,7 @@ import com.getcapacitor.annotation.PermissionCallback;
|
|
|
23
28
|
import java.util.ArrayList;
|
|
24
29
|
import java.util.List;
|
|
25
30
|
import java.util.Locale;
|
|
31
|
+
import java.util.concurrent.Executor;
|
|
26
32
|
import java.util.concurrent.locks.ReentrantLock;
|
|
27
33
|
import org.json.JSONArray;
|
|
28
34
|
|
|
@@ -34,24 +40,60 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
34
40
|
|
|
35
41
|
public static final String SPEECH_RECOGNITION = "speechRecognition";
|
|
36
42
|
private static final String TAG = "SpeechRecognition";
|
|
37
|
-
private static final String PLUGIN_VERSION = "
|
|
43
|
+
private static final String PLUGIN_VERSION = "8.0.10";
|
|
44
|
+
private static final int FORCE_STOP_TIMEOUT_MS = 1500;
|
|
45
|
+
private static final int STOP_FALLBACK_TIMEOUT_MS = 500;
|
|
46
|
+
private static final int CONTINUOUS_RESTART_DELAY_MS = 100;
|
|
47
|
+
|
|
48
|
+
private enum ListeningState {
|
|
49
|
+
IDLE,
|
|
50
|
+
STARTING,
|
|
51
|
+
STARTED,
|
|
52
|
+
STOPPING
|
|
53
|
+
}
|
|
38
54
|
|
|
39
55
|
private Receiver languageReceiver;
|
|
40
56
|
private SpeechRecognizer speechRecognizer;
|
|
57
|
+
private boolean speechRecognizerUsesOnDevice = false;
|
|
41
58
|
private final ReentrantLock lock = new ReentrantLock();
|
|
59
|
+
private final Handler handler = new Handler(Looper.getMainLooper());
|
|
42
60
|
private boolean listening = false;
|
|
43
61
|
private JSONArray previousPartialResults = new JSONArray();
|
|
44
62
|
|
|
63
|
+
private Runnable forceStopRunnable;
|
|
64
|
+
private boolean forceStopped = false;
|
|
65
|
+
private boolean pttButtonHeld = false;
|
|
66
|
+
private boolean continuousPTTMode = false;
|
|
67
|
+
private boolean popupSessionActive = false;
|
|
68
|
+
private boolean popupSessionCancelled = false;
|
|
69
|
+
private StringBuilder accumulatedResults = new StringBuilder();
|
|
70
|
+
|
|
71
|
+
private String lastLanguage = Locale.getDefault().toLanguageTag();
|
|
72
|
+
private int lastMaxResults = MAX_RESULTS;
|
|
73
|
+
private String lastPrompt;
|
|
74
|
+
private boolean lastPartialResults = false;
|
|
75
|
+
private int lastAllowForSilence = 0;
|
|
76
|
+
private boolean lastUseOnDeviceRecognition = false;
|
|
77
|
+
|
|
78
|
+
private PluginCall activeStartCall;
|
|
79
|
+
private ListeningState state = ListeningState.IDLE;
|
|
80
|
+
private long sessionId = 0;
|
|
81
|
+
private long recognizerGeneration = 0;
|
|
82
|
+
private String pendingStopReason;
|
|
83
|
+
|
|
45
84
|
@Override
|
|
46
85
|
public void load() {
|
|
47
86
|
super.load();
|
|
48
87
|
bridge
|
|
49
88
|
.getWebView()
|
|
50
89
|
.post(() -> {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
90
|
+
try {
|
|
91
|
+
lock.lock();
|
|
92
|
+
recreateIdleRecognizerLocked();
|
|
93
|
+
Logger.info(getLogTag(), "Instantiated SpeechRecognizer in load()");
|
|
94
|
+
} finally {
|
|
95
|
+
lock.unlock();
|
|
96
|
+
}
|
|
55
97
|
});
|
|
56
98
|
}
|
|
57
99
|
|
|
@@ -61,52 +103,323 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
61
103
|
call.resolve(new JSObject().put("available", val));
|
|
62
104
|
}
|
|
63
105
|
|
|
106
|
+
@PluginMethod
|
|
107
|
+
public void isOnDeviceRecognitionAvailable(PluginCall call) {
|
|
108
|
+
String language = call.getString("language", Locale.getDefault().toLanguageTag());
|
|
109
|
+
if (!canUseOnDeviceRecognition()) {
|
|
110
|
+
call.resolve(new JSObject().put("available", false));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
final SpeechRecognizer supportChecker;
|
|
115
|
+
try {
|
|
116
|
+
supportChecker = SpeechRecognizer.createOnDeviceSpeechRecognizer(bridge.getActivity());
|
|
117
|
+
} catch (UnsupportedOperationException ex) {
|
|
118
|
+
call.resolve(new JSObject().put("available", false));
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
Intent intent = buildRecognizerIntent(language, MAX_RESULTS, null, false, 0, true);
|
|
123
|
+
supportChecker.checkRecognitionSupport(
|
|
124
|
+
intent,
|
|
125
|
+
mainExecutor(),
|
|
126
|
+
new RecognitionSupportCallback() {
|
|
127
|
+
@Override
|
|
128
|
+
public void onSupportResult(RecognitionSupport support) {
|
|
129
|
+
boolean available =
|
|
130
|
+
isLanguageSupported(language, support.getInstalledOnDeviceLanguages()) ||
|
|
131
|
+
isLanguageSupported(language, support.getSupportedOnDeviceLanguages()) ||
|
|
132
|
+
isLanguageSupported(language, support.getPendingOnDeviceLanguages());
|
|
133
|
+
supportChecker.destroy();
|
|
134
|
+
call.resolve(new JSObject().put("available", available));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
@Override
|
|
138
|
+
public void onError(int error) {
|
|
139
|
+
Logger.warn(TAG, "On-device recognition support check failed: " + getErrorText(error));
|
|
140
|
+
supportChecker.destroy();
|
|
141
|
+
call.resolve(new JSObject().put("available", false));
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
64
147
|
@PluginMethod
|
|
65
148
|
public void start(PluginCall call) {
|
|
66
149
|
if (!SpeechRecognizer.isRecognitionAvailable(bridge.getContext())) {
|
|
67
|
-
Logger.warn(TAG, "start() called but speech recognizer unavailable");
|
|
68
150
|
call.unavailable(NOT_AVAILABLE);
|
|
69
151
|
return;
|
|
70
152
|
}
|
|
71
153
|
|
|
72
154
|
if (getPermissionState(SPEECH_RECOGNITION) != PermissionState.GRANTED) {
|
|
73
|
-
Logger.warn(TAG, "start() missing RECORD_AUDIO permission");
|
|
74
155
|
call.reject(MISSING_PERMISSION);
|
|
75
156
|
return;
|
|
76
157
|
}
|
|
77
158
|
|
|
78
|
-
String language = call.getString("language", Locale.getDefault().
|
|
159
|
+
String language = call.getString("language", Locale.getDefault().toLanguageTag());
|
|
79
160
|
int maxResults = call.getInt("maxResults", MAX_RESULTS);
|
|
80
161
|
String prompt = call.getString("prompt", null);
|
|
81
162
|
boolean partialResults = call.getBoolean("partialResults", false);
|
|
82
163
|
boolean popup = call.getBoolean("popup", false);
|
|
164
|
+
boolean useOnDeviceRecognition = call.getBoolean("useOnDeviceRecognition", false);
|
|
83
165
|
int allowForSilence = call.getInt("allowForSilence", 0);
|
|
166
|
+
boolean continuousPTT = call.getBoolean("continuousPTT", false);
|
|
167
|
+
|
|
168
|
+
if (useOnDeviceRecognition && popup) {
|
|
169
|
+
call.reject("On-device recognition is not supported with popup mode on Android.");
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
if (continuousPTT && popup) {
|
|
174
|
+
call.reject("continuousPTT is only supported with inline recognition on Android.");
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (continuousPTT && !partialResults) {
|
|
179
|
+
call.reject("continuousPTT requires partialResults: true.");
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
if (useOnDeviceRecognition && !canUseOnDeviceRecognition()) {
|
|
184
|
+
call.unavailable("On-device speech recognition is not available on this device.");
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
final long currentSessionId;
|
|
189
|
+
try {
|
|
190
|
+
lock.lock();
|
|
191
|
+
if (state != ListeningState.IDLE) {
|
|
192
|
+
call.reject("Speech recognition is already running.");
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
cancelPendingForceStopLocked();
|
|
196
|
+
forceStopped = false;
|
|
197
|
+
pendingStopReason = null;
|
|
198
|
+
resetPartialResultsCache();
|
|
199
|
+
accumulatedResults = new StringBuilder();
|
|
200
|
+
continuousPTTMode = continuousPTT;
|
|
201
|
+
popupSessionActive = false;
|
|
202
|
+
popupSessionCancelled = false;
|
|
203
|
+
lastLanguage = language;
|
|
204
|
+
lastMaxResults = maxResults;
|
|
205
|
+
lastPrompt = prompt;
|
|
206
|
+
lastPartialResults = partialResults;
|
|
207
|
+
lastAllowForSilence = allowForSilence;
|
|
208
|
+
lastUseOnDeviceRecognition = useOnDeviceRecognition;
|
|
209
|
+
activeStartCall = null;
|
|
210
|
+
sessionId++;
|
|
211
|
+
currentSessionId = sessionId;
|
|
212
|
+
state = ListeningState.STARTING;
|
|
213
|
+
} finally {
|
|
214
|
+
lock.unlock();
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
emitListeningState("startingListening", currentSessionId, "userStart", null, null);
|
|
218
|
+
|
|
84
219
|
Logger.info(
|
|
85
220
|
TAG,
|
|
86
221
|
String.format(
|
|
87
|
-
|
|
222
|
+
Locale.US,
|
|
223
|
+
"Starting recognition | sessionId=%d lang=%s maxResults=%d partial=%s popup=%s onDevice=%s allowForSilence=%d continuousPTT=%s",
|
|
224
|
+
currentSessionId,
|
|
88
225
|
language,
|
|
89
226
|
maxResults,
|
|
90
227
|
partialResults,
|
|
91
228
|
popup,
|
|
92
|
-
|
|
229
|
+
useOnDeviceRecognition,
|
|
230
|
+
allowForSilence,
|
|
231
|
+
continuousPTT
|
|
93
232
|
)
|
|
94
233
|
);
|
|
95
|
-
|
|
234
|
+
|
|
235
|
+
beginListening(
|
|
236
|
+
language,
|
|
237
|
+
maxResults,
|
|
238
|
+
prompt,
|
|
239
|
+
partialResults,
|
|
240
|
+
popup,
|
|
241
|
+
call,
|
|
242
|
+
allowForSilence,
|
|
243
|
+
useOnDeviceRecognition,
|
|
244
|
+
currentSessionId,
|
|
245
|
+
false
|
|
246
|
+
);
|
|
96
247
|
}
|
|
97
248
|
|
|
98
249
|
@PluginMethod
|
|
99
|
-
public void stop(
|
|
100
|
-
|
|
250
|
+
public void stop(PluginCall call) {
|
|
251
|
+
final long currentSessionId;
|
|
252
|
+
final boolean popupActive;
|
|
253
|
+
PluginCall popupStartCall = null;
|
|
101
254
|
try {
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
255
|
+
lock.lock();
|
|
256
|
+
if (state == ListeningState.IDLE && !listening) {
|
|
257
|
+
call.resolve();
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
currentSessionId = sessionId;
|
|
262
|
+
pendingStopReason = "userStop";
|
|
263
|
+
if (state != ListeningState.STOPPING) {
|
|
264
|
+
state = ListeningState.STOPPING;
|
|
265
|
+
emitListeningState("stoppingListening", currentSessionId, "userStop", null, null);
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
popupActive = popupSessionActive;
|
|
269
|
+
if (popupActive) {
|
|
270
|
+
cancelPendingForceStopLocked();
|
|
271
|
+
popupSessionCancelled = true;
|
|
272
|
+
popupStartCall = activeStartCall;
|
|
273
|
+
activeStartCall = null;
|
|
274
|
+
}
|
|
275
|
+
} finally {
|
|
276
|
+
lock.unlock();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
if (popupStartCall != null) {
|
|
280
|
+
popupStartCall.reject("Recognition stopped before final results were produced.");
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
if (popupActive) {
|
|
284
|
+
call.resolve();
|
|
105
285
|
return;
|
|
106
286
|
}
|
|
287
|
+
|
|
288
|
+
bridge
|
|
289
|
+
.getWebView()
|
|
290
|
+
.post(() -> {
|
|
291
|
+
try {
|
|
292
|
+
lock.lock();
|
|
293
|
+
cancelPendingForceStopLocked();
|
|
294
|
+
if (speechRecognizer != null) {
|
|
295
|
+
try {
|
|
296
|
+
speechRecognizer.stopListening();
|
|
297
|
+
} catch (Exception ignored) {}
|
|
298
|
+
}
|
|
299
|
+
listening(false);
|
|
300
|
+
scheduleFinishFallbackLocked(currentSessionId, "userStop", null, STOP_FALLBACK_TIMEOUT_MS);
|
|
301
|
+
} finally {
|
|
302
|
+
lock.unlock();
|
|
303
|
+
}
|
|
304
|
+
});
|
|
107
305
|
call.resolve();
|
|
108
306
|
}
|
|
109
307
|
|
|
308
|
+
@PluginMethod
|
|
309
|
+
public void forceStop(PluginCall call) {
|
|
310
|
+
final int timeout = call.getInt("timeout", FORCE_STOP_TIMEOUT_MS);
|
|
311
|
+
final long currentSessionId;
|
|
312
|
+
final boolean popupActive;
|
|
313
|
+
PluginCall popupStartCall = null;
|
|
314
|
+
try {
|
|
315
|
+
lock.lock();
|
|
316
|
+
if (state == ListeningState.IDLE && !listening) {
|
|
317
|
+
call.resolve();
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
currentSessionId = sessionId;
|
|
322
|
+
pendingStopReason = "forceStop";
|
|
323
|
+
if (state != ListeningState.STOPPING) {
|
|
324
|
+
state = ListeningState.STOPPING;
|
|
325
|
+
emitListeningState("stoppingListening", currentSessionId, "forceStop", null, null);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
popupActive = popupSessionActive;
|
|
329
|
+
if (popupActive) {
|
|
330
|
+
cancelPendingForceStopLocked();
|
|
331
|
+
popupSessionCancelled = true;
|
|
332
|
+
popupStartCall = activeStartCall;
|
|
333
|
+
activeStartCall = null;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
cancelPendingForceStopLocked();
|
|
337
|
+
if (!popupActive && speechRecognizer != null) {
|
|
338
|
+
try {
|
|
339
|
+
speechRecognizer.stopListening();
|
|
340
|
+
} catch (Exception ignored) {}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
forceStopRunnable = () -> {
|
|
344
|
+
PluginCall startCallToReject = null;
|
|
345
|
+
JSObject forcedPayload = null;
|
|
346
|
+
try {
|
|
347
|
+
lock.lock();
|
|
348
|
+
if (currentSessionId != sessionId || (state == ListeningState.IDLE && !listening)) {
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
forceStopped = true;
|
|
353
|
+
startCallToReject = activeStartCall;
|
|
354
|
+
activeStartCall = null;
|
|
355
|
+
forcedPayload = buildForcedPartialResultLocked();
|
|
356
|
+
} finally {
|
|
357
|
+
lock.unlock();
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
if (startCallToReject != null) {
|
|
361
|
+
startCallToReject.reject("Recognition force stopped before final results were produced.");
|
|
362
|
+
}
|
|
363
|
+
if (forcedPayload != null) {
|
|
364
|
+
notifyListeners(PARTIAL_RESULTS_EVENT, forcedPayload);
|
|
365
|
+
}
|
|
366
|
+
finishSession(currentSessionId, "forceStop", null);
|
|
367
|
+
};
|
|
368
|
+
if (!popupActive) {
|
|
369
|
+
handler.postDelayed(forceStopRunnable, timeout);
|
|
370
|
+
}
|
|
371
|
+
} finally {
|
|
372
|
+
lock.unlock();
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
if (popupStartCall != null) {
|
|
376
|
+
popupStartCall.reject("Recognition force stopped before final results were produced.");
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
if (popupActive) {
|
|
380
|
+
call.resolve();
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
call.resolve();
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
@PluginMethod
|
|
388
|
+
public void getLastPartialResult(PluginCall call) {
|
|
389
|
+
try {
|
|
390
|
+
lock.lock();
|
|
391
|
+
JSObject result = new JSObject();
|
|
392
|
+
String text = buildCurrentTranscriptTextLocked();
|
|
393
|
+
result.put("available", !text.isEmpty());
|
|
394
|
+
result.put("text", text);
|
|
395
|
+
if (previousPartialResults.length() > 0) {
|
|
396
|
+
result.put("matches", previousPartialResults);
|
|
397
|
+
}
|
|
398
|
+
call.resolve(result);
|
|
399
|
+
} catch (Exception ex) {
|
|
400
|
+
call.reject(ex.getLocalizedMessage());
|
|
401
|
+
} finally {
|
|
402
|
+
lock.unlock();
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
@PluginMethod
|
|
407
|
+
public void setPTTState(PluginCall call) {
|
|
408
|
+
boolean held = call.getBoolean("held", false);
|
|
409
|
+
try {
|
|
410
|
+
lock.lock();
|
|
411
|
+
pttButtonHeld = held;
|
|
412
|
+
if (held) {
|
|
413
|
+
accumulatedResults = new StringBuilder();
|
|
414
|
+
forceStopped = false;
|
|
415
|
+
pendingStopReason = null;
|
|
416
|
+
}
|
|
417
|
+
call.resolve();
|
|
418
|
+
} finally {
|
|
419
|
+
lock.unlock();
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
110
423
|
@PluginMethod
|
|
111
424
|
public void getSupportedLanguages(PluginCall call) {
|
|
112
425
|
if (languageReceiver == null) {
|
|
@@ -147,9 +460,7 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
147
460
|
|
|
148
461
|
@PluginMethod
|
|
149
462
|
public void getPluginVersion(PluginCall call) {
|
|
150
|
-
|
|
151
|
-
ret.put("version", PLUGIN_VERSION);
|
|
152
|
-
call.resolve(ret);
|
|
463
|
+
call.resolve(new JSObject().put("version", PLUGIN_VERSION));
|
|
153
464
|
}
|
|
154
465
|
|
|
155
466
|
@PermissionCallback
|
|
@@ -160,37 +471,108 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
160
471
|
|
|
161
472
|
@ActivityCallback
|
|
162
473
|
private void listeningResult(PluginCall call, ActivityResult result) {
|
|
163
|
-
|
|
164
|
-
|
|
474
|
+
long currentSessionId;
|
|
475
|
+
String finalReason;
|
|
476
|
+
boolean popupCancelled;
|
|
477
|
+
try {
|
|
478
|
+
lock.lock();
|
|
479
|
+
currentSessionId = sessionId;
|
|
480
|
+
finalReason = pendingStopReason != null ? pendingStopReason : "results";
|
|
481
|
+
popupCancelled = popupSessionCancelled;
|
|
482
|
+
popupSessionActive = false;
|
|
483
|
+
popupSessionCancelled = false;
|
|
484
|
+
activeStartCall = null;
|
|
485
|
+
} finally {
|
|
486
|
+
lock.unlock();
|
|
165
487
|
}
|
|
166
488
|
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
489
|
+
if (call != null && !popupCancelled) {
|
|
490
|
+
int resultCode = result.getResultCode();
|
|
491
|
+
if (resultCode == Activity.RESULT_OK) {
|
|
492
|
+
try {
|
|
493
|
+
ArrayList<String> matchesList = result.getData().getStringArrayListExtra(RecognizerIntent.EXTRA_RESULTS);
|
|
494
|
+
call.resolve(new JSObject().put("matches", new JSArray(matchesList)));
|
|
495
|
+
} catch (Exception ex) {
|
|
496
|
+
call.reject(ex.getMessage());
|
|
497
|
+
}
|
|
498
|
+
} else {
|
|
499
|
+
call.reject(Integer.toString(resultCode));
|
|
176
500
|
}
|
|
177
|
-
} else {
|
|
178
|
-
call.reject(Integer.toString(resultCode));
|
|
179
501
|
}
|
|
180
502
|
|
|
181
|
-
|
|
182
|
-
listening(false);
|
|
183
|
-
lock.unlock();
|
|
503
|
+
finishSession(currentSessionId, finalReason, null);
|
|
184
504
|
}
|
|
185
505
|
|
|
186
506
|
private void beginListening(
|
|
187
507
|
String language,
|
|
188
508
|
int maxResults,
|
|
189
509
|
String prompt,
|
|
190
|
-
|
|
510
|
+
boolean partialResults,
|
|
191
511
|
boolean showPopup,
|
|
192
512
|
PluginCall call,
|
|
193
|
-
int allowForSilence
|
|
513
|
+
int allowForSilence,
|
|
514
|
+
boolean useOnDeviceRecognition,
|
|
515
|
+
long currentSessionId,
|
|
516
|
+
boolean restarting
|
|
517
|
+
) {
|
|
518
|
+
Intent intent = buildRecognizerIntent(language, maxResults, prompt, partialResults, allowForSilence, useOnDeviceRecognition);
|
|
519
|
+
|
|
520
|
+
if (showPopup) {
|
|
521
|
+
try {
|
|
522
|
+
lock.lock();
|
|
523
|
+
listening(true);
|
|
524
|
+
state = ListeningState.STARTED;
|
|
525
|
+
popupSessionActive = true;
|
|
526
|
+
popupSessionCancelled = false;
|
|
527
|
+
activeStartCall = call;
|
|
528
|
+
} finally {
|
|
529
|
+
lock.unlock();
|
|
530
|
+
}
|
|
531
|
+
emitListeningState("started", currentSessionId, restarting ? "results" : "userStart", null, "started");
|
|
532
|
+
startActivityForResult(call, intent, "listeningResult");
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
bridge
|
|
537
|
+
.getWebView()
|
|
538
|
+
.post(() -> {
|
|
539
|
+
try {
|
|
540
|
+
lock.lock();
|
|
541
|
+
if (currentSessionId != sessionId) {
|
|
542
|
+
return;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
resetPartialResultsCache();
|
|
546
|
+
rebuildRecognizerLocked(call, partialResults, useOnDeviceRecognition, currentSessionId);
|
|
547
|
+
if (!partialResults && call != null) {
|
|
548
|
+
activeStartCall = call;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
if (useOnDeviceRecognition) {
|
|
552
|
+
beginOnDeviceListening(intent, language, partialResults, call, currentSessionId, restarting);
|
|
553
|
+
} else {
|
|
554
|
+
startInlineListening(intent, partialResults, call, currentSessionId, restarting);
|
|
555
|
+
}
|
|
556
|
+
} catch (Exception ex) {
|
|
557
|
+
Logger.error(TAG, "Error starting listening", ex);
|
|
558
|
+
if (call != null) {
|
|
559
|
+
call.reject(ex.getMessage());
|
|
560
|
+
}
|
|
561
|
+
emitErrorEvent("START_FAILED", ex.getMessage(), currentSessionId);
|
|
562
|
+
finishSession(currentSessionId, "error", "START_FAILED");
|
|
563
|
+
} finally {
|
|
564
|
+
lock.unlock();
|
|
565
|
+
}
|
|
566
|
+
});
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
private Intent buildRecognizerIntent(
|
|
570
|
+
String language,
|
|
571
|
+
int maxResults,
|
|
572
|
+
String prompt,
|
|
573
|
+
boolean partialResults,
|
|
574
|
+
int allowForSilence,
|
|
575
|
+
boolean useOnDeviceRecognition
|
|
194
576
|
) {
|
|
195
577
|
Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
|
|
196
578
|
intent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
|
|
@@ -200,6 +582,10 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
200
582
|
intent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, partialResults);
|
|
201
583
|
intent.putExtra("android.speech.extra.DICTATION_MODE", partialResults);
|
|
202
584
|
|
|
585
|
+
if (useOnDeviceRecognition) {
|
|
586
|
+
intent.putExtra(RecognizerIntent.EXTRA_PREFER_OFFLINE, true);
|
|
587
|
+
}
|
|
588
|
+
|
|
203
589
|
if (allowForSilence > 0) {
|
|
204
590
|
intent.putExtra(RecognizerIntent.EXTRA_SEGMENTED_SESSION, true);
|
|
205
591
|
intent.putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, allowForSilence);
|
|
@@ -210,61 +596,361 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
210
596
|
intent.putExtra(RecognizerIntent.EXTRA_PROMPT, prompt);
|
|
211
597
|
}
|
|
212
598
|
|
|
213
|
-
|
|
214
|
-
|
|
599
|
+
return intent;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
private void rebuildRecognizerLocked(PluginCall call, boolean partialResults, boolean useOnDeviceRecognition, long currentSessionId) {
|
|
603
|
+
if (speechRecognizer != null && speechRecognizerUsesOnDevice != useOnDeviceRecognition) {
|
|
604
|
+
destroyCurrentRecognizerLocked();
|
|
605
|
+
}
|
|
606
|
+
|
|
607
|
+
if (speechRecognizer == null) {
|
|
608
|
+
speechRecognizer = useOnDeviceRecognition
|
|
609
|
+
? SpeechRecognizer.createOnDeviceSpeechRecognizer(bridge.getActivity())
|
|
610
|
+
: SpeechRecognizer.createSpeechRecognizer(bridge.getActivity());
|
|
611
|
+
speechRecognizerUsesOnDevice = useOnDeviceRecognition;
|
|
215
612
|
} else {
|
|
216
|
-
|
|
217
|
-
.
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
613
|
+
try {
|
|
614
|
+
speechRecognizer.cancel();
|
|
615
|
+
} catch (Exception ignored) {}
|
|
616
|
+
speechRecognizerUsesOnDevice = useOnDeviceRecognition;
|
|
617
|
+
}
|
|
618
|
+
|
|
619
|
+
recognizerGeneration++;
|
|
620
|
+
SpeechRecognitionListener listener = new SpeechRecognitionListener(currentSessionId, recognizerGeneration);
|
|
621
|
+
listener.setCall(call);
|
|
622
|
+
listener.setPartialResults(partialResults);
|
|
623
|
+
speechRecognizer.setRecognitionListener(listener);
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
private void beginOnDeviceListening(
|
|
627
|
+
Intent intent,
|
|
628
|
+
String language,
|
|
629
|
+
boolean partialResults,
|
|
630
|
+
PluginCall call,
|
|
631
|
+
long currentSessionId,
|
|
632
|
+
boolean restarting
|
|
633
|
+
) {
|
|
634
|
+
speechRecognizer.checkRecognitionSupport(
|
|
635
|
+
intent,
|
|
636
|
+
mainExecutor(),
|
|
637
|
+
new RecognitionSupportCallback() {
|
|
638
|
+
@Override
|
|
639
|
+
public void onSupportResult(RecognitionSupport support) {
|
|
640
|
+
boolean installed = isLanguageSupported(language, support.getInstalledOnDeviceLanguages());
|
|
641
|
+
boolean supported =
|
|
642
|
+
installed ||
|
|
643
|
+
isLanguageSupported(language, support.getSupportedOnDeviceLanguages()) ||
|
|
644
|
+
isLanguageSupported(language, support.getPendingOnDeviceLanguages());
|
|
227
645
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
listener.setPartialResults(partialResults);
|
|
232
|
-
speechRecognizer.setRecognitionListener(listener);
|
|
233
|
-
speechRecognizer.startListening(intent);
|
|
234
|
-
listening(true);
|
|
235
|
-
if (partialResults) {
|
|
236
|
-
call.resolve();
|
|
646
|
+
if (!supported) {
|
|
647
|
+
if (call != null) {
|
|
648
|
+
call.reject("On-device recognition is not available for language: " + language);
|
|
237
649
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
650
|
+
emitErrorEvent(
|
|
651
|
+
"UNSUPPORTED_LOCALE",
|
|
652
|
+
"On-device recognition is not available for language: " + language,
|
|
653
|
+
currentSessionId
|
|
654
|
+
);
|
|
655
|
+
finishSession(currentSessionId, "error", "UNSUPPORTED_LOCALE");
|
|
656
|
+
return;
|
|
242
657
|
}
|
|
243
|
-
|
|
658
|
+
|
|
659
|
+
if (installed) {
|
|
660
|
+
startInlineListening(intent, partialResults, call, currentSessionId, restarting);
|
|
661
|
+
return;
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
triggerOnDeviceModelDownload(intent, partialResults, call, currentSessionId, restarting);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
@Override
|
|
668
|
+
public void onError(int error) {
|
|
669
|
+
String errorCode = getErrorCode(error);
|
|
670
|
+
String message = getErrorText(error);
|
|
671
|
+
if (call != null) {
|
|
672
|
+
call.reject(message);
|
|
673
|
+
}
|
|
674
|
+
emitErrorEvent(errorCode, message, currentSessionId);
|
|
675
|
+
finishSession(currentSessionId, "error", errorCode);
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
private void triggerOnDeviceModelDownload(
|
|
682
|
+
Intent intent,
|
|
683
|
+
boolean partialResults,
|
|
684
|
+
PluginCall call,
|
|
685
|
+
long currentSessionId,
|
|
686
|
+
boolean restarting
|
|
687
|
+
) {
|
|
688
|
+
speechRecognizer.triggerModelDownload(
|
|
689
|
+
intent,
|
|
690
|
+
mainExecutor(),
|
|
691
|
+
new ModelDownloadListener() {
|
|
692
|
+
@Override
|
|
693
|
+
public void onProgress(int completedPercent) {}
|
|
694
|
+
|
|
695
|
+
@Override
|
|
696
|
+
public void onSuccess() {
|
|
697
|
+
startInlineListening(intent, partialResults, call, currentSessionId, restarting);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
@Override
|
|
701
|
+
public void onScheduled() {
|
|
702
|
+
String message = "On-device speech model download was scheduled. Try again once it finishes.";
|
|
703
|
+
if (call != null) {
|
|
704
|
+
call.reject(message);
|
|
705
|
+
}
|
|
706
|
+
emitErrorEvent("MODEL_DOWNLOAD_SCHEDULED", message, currentSessionId);
|
|
707
|
+
finishSession(currentSessionId, "error", "MODEL_DOWNLOAD_SCHEDULED");
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
@Override
|
|
711
|
+
public void onError(int error) {
|
|
712
|
+
String errorCode = getErrorCode(error);
|
|
713
|
+
String message = getErrorText(error);
|
|
714
|
+
if (call != null) {
|
|
715
|
+
call.reject(message);
|
|
716
|
+
}
|
|
717
|
+
emitErrorEvent(errorCode, message, currentSessionId);
|
|
718
|
+
finishSession(currentSessionId, "error", errorCode);
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
|
|
724
|
+
private void startInlineListening(Intent intent, boolean partialResults, PluginCall call, long currentSessionId, boolean restarting) {
|
|
725
|
+
speechRecognizer.startListening(intent);
|
|
726
|
+
listening(true);
|
|
727
|
+
state = ListeningState.STARTED;
|
|
728
|
+
emitListeningState("started", currentSessionId, restarting ? "results" : "userStart", null, "started");
|
|
729
|
+
if (partialResults && call != null) {
|
|
730
|
+
activeStartCall = null;
|
|
731
|
+
call.resolve();
|
|
244
732
|
}
|
|
245
733
|
}
|
|
246
734
|
|
|
247
|
-
private void
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
.post(() -> {
|
|
735
|
+
private void scheduleContinuousRestart(long currentSessionId) {
|
|
736
|
+
handler.postDelayed(
|
|
737
|
+
() -> {
|
|
251
738
|
try {
|
|
252
739
|
lock.lock();
|
|
253
|
-
if (
|
|
254
|
-
|
|
255
|
-
speechRecognizer.stopListening();
|
|
256
|
-
listening(false);
|
|
740
|
+
if (currentSessionId != sessionId || !continuousPTTMode || !pttButtonHeld || pendingStopReason != null) {
|
|
741
|
+
return;
|
|
257
742
|
}
|
|
743
|
+
} finally {
|
|
744
|
+
lock.unlock();
|
|
745
|
+
}
|
|
746
|
+
|
|
747
|
+
beginListening(
|
|
748
|
+
lastLanguage,
|
|
749
|
+
lastMaxResults,
|
|
750
|
+
lastPrompt,
|
|
751
|
+
lastPartialResults,
|
|
752
|
+
false,
|
|
753
|
+
null,
|
|
754
|
+
lastAllowForSilence,
|
|
755
|
+
lastUseOnDeviceRecognition,
|
|
756
|
+
currentSessionId,
|
|
757
|
+
true
|
|
758
|
+
);
|
|
759
|
+
},
|
|
760
|
+
CONTINUOUS_RESTART_DELAY_MS
|
|
761
|
+
);
|
|
762
|
+
}
|
|
763
|
+
|
|
764
|
+
private void finishSession(long finishedSessionId, String explicitReason, String errorCode) {
|
|
765
|
+
handler.post(() -> {
|
|
766
|
+
PluginCall startCallToReject = null;
|
|
767
|
+
boolean emitReady = true;
|
|
768
|
+
String reason;
|
|
769
|
+
|
|
770
|
+
try {
|
|
771
|
+
lock.lock();
|
|
772
|
+
if (finishedSessionId != sessionId) {
|
|
773
|
+
return;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
reason = explicitReason != null ? explicitReason : (pendingStopReason != null ? pendingStopReason : "unknown");
|
|
777
|
+
if (state != ListeningState.STOPPING) {
|
|
778
|
+
state = ListeningState.STOPPING;
|
|
779
|
+
emitListeningState("stoppingListening", finishedSessionId, reason, errorCode, null);
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
cancelPendingForceStopLocked();
|
|
783
|
+
listening(false);
|
|
784
|
+
if (activeStartCall != null && !lastPartialResults && ("userStop".equals(reason) || "forceStop".equals(reason))) {
|
|
785
|
+
startCallToReject = activeStartCall;
|
|
786
|
+
}
|
|
787
|
+
activeStartCall = null;
|
|
788
|
+
pendingStopReason = null;
|
|
789
|
+
continuousPTTMode = false;
|
|
790
|
+
popupSessionActive = false;
|
|
791
|
+
popupSessionCancelled = false;
|
|
792
|
+
forceStopped = false;
|
|
793
|
+
resetPartialResultsCache();
|
|
794
|
+
accumulatedResults = new StringBuilder();
|
|
795
|
+
|
|
796
|
+
destroyCurrentRecognizerLocked();
|
|
797
|
+
try {
|
|
798
|
+
recreateIdleRecognizerLocked();
|
|
258
799
|
} catch (Exception ex) {
|
|
259
|
-
|
|
800
|
+
emitReady = false;
|
|
801
|
+
Logger.error(TAG, "Failed to recreate recognizer", ex);
|
|
802
|
+
emitErrorEvent("RECREATE_FAILED", ex.getMessage(), finishedSessionId);
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
state = ListeningState.IDLE;
|
|
806
|
+
if (emitReady) {
|
|
807
|
+
notifyListeners(READY_FOR_NEXT_SESSION_EVENT, new JSObject().put("sessionId", finishedSessionId));
|
|
808
|
+
}
|
|
809
|
+
emitListeningState("stopped", finishedSessionId, reason, errorCode, "stopped");
|
|
810
|
+
} finally {
|
|
811
|
+
lock.unlock();
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
if (startCallToReject != null) {
|
|
815
|
+
startCallToReject.reject("Recognition stopped before final results were produced.");
|
|
816
|
+
}
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
private void recreateIdleRecognizerLocked() {
|
|
821
|
+
destroyCurrentRecognizerLocked();
|
|
822
|
+
speechRecognizer = SpeechRecognizer.createSpeechRecognizer(bridge.getActivity());
|
|
823
|
+
speechRecognizerUsesOnDevice = false;
|
|
824
|
+
recognizerGeneration++;
|
|
825
|
+
speechRecognizer.setRecognitionListener(new SpeechRecognitionListener(sessionId, recognizerGeneration));
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
private void destroyCurrentRecognizerLocked() {
|
|
829
|
+
if (speechRecognizer != null) {
|
|
830
|
+
try {
|
|
831
|
+
speechRecognizer.cancel();
|
|
832
|
+
} catch (Exception ignored) {}
|
|
833
|
+
try {
|
|
834
|
+
speechRecognizer.destroy();
|
|
835
|
+
} catch (Exception ignored) {}
|
|
836
|
+
speechRecognizer = null;
|
|
837
|
+
}
|
|
838
|
+
speechRecognizerUsesOnDevice = false;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
private void cancelPendingForceStopLocked() {
|
|
842
|
+
if (forceStopRunnable != null) {
|
|
843
|
+
handler.removeCallbacks(forceStopRunnable);
|
|
844
|
+
forceStopRunnable = null;
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
|
|
848
|
+
private void scheduleFinishFallbackLocked(long currentSessionId, String reason, String errorCode, int delayMs) {
|
|
849
|
+
handler.postDelayed(
|
|
850
|
+
() -> {
|
|
851
|
+
boolean shouldFinish;
|
|
852
|
+
try {
|
|
853
|
+
lock.lock();
|
|
854
|
+
shouldFinish = currentSessionId == sessionId && state != ListeningState.IDLE;
|
|
260
855
|
} finally {
|
|
261
856
|
lock.unlock();
|
|
262
857
|
}
|
|
263
|
-
|
|
858
|
+
|
|
859
|
+
if (shouldFinish) {
|
|
860
|
+
finishSession(currentSessionId, reason, errorCode);
|
|
861
|
+
}
|
|
862
|
+
},
|
|
863
|
+
delayMs
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
private JSObject buildForcedPartialResultLocked() {
|
|
868
|
+
if (previousPartialResults.length() == 0 && accumulatedResults.length() == 0) {
|
|
869
|
+
return null;
|
|
870
|
+
}
|
|
871
|
+
|
|
872
|
+
JSObject payload = new JSObject();
|
|
873
|
+
payload.put("forced", true);
|
|
874
|
+
if (previousPartialResults.length() > 0) {
|
|
875
|
+
payload.put("matches", previousPartialResults);
|
|
876
|
+
}
|
|
877
|
+
String accumulatedText = buildCurrentTranscriptTextLocked();
|
|
878
|
+
if (!accumulatedText.isEmpty()) {
|
|
879
|
+
payload.put("accumulatedText", accumulatedText);
|
|
880
|
+
}
|
|
881
|
+
return payload;
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
private String buildCurrentTranscriptTextLocked() {
|
|
885
|
+
String accumulatedText = accumulatedResults.toString().trim();
|
|
886
|
+
String latestText = previousPartialResults.length() > 0 ? previousPartialResults.optString(0, "").trim() : "";
|
|
887
|
+
|
|
888
|
+
if (accumulatedText.isEmpty()) {
|
|
889
|
+
return latestText;
|
|
890
|
+
}
|
|
891
|
+
if (latestText.isEmpty()) {
|
|
892
|
+
return accumulatedText;
|
|
893
|
+
}
|
|
894
|
+
return accumulatedText + " " + latestText;
|
|
264
895
|
}
|
|
265
896
|
|
|
266
897
|
private void listening(boolean value) {
|
|
267
|
-
|
|
898
|
+
listening = value;
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
private void resetPartialResultsCache() {
|
|
902
|
+
previousPartialResults = new JSONArray();
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
private boolean canUseOnDeviceRecognition() {
|
|
906
|
+
return (
|
|
907
|
+
Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && SpeechRecognizer.isOnDeviceRecognitionAvailable(bridge.getContext())
|
|
908
|
+
);
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
private boolean isLanguageSupported(String requestedLanguage, List<String> candidateLanguages) {
|
|
912
|
+
if (candidateLanguages == null || candidateLanguages.isEmpty()) {
|
|
913
|
+
return false;
|
|
914
|
+
}
|
|
915
|
+
|
|
916
|
+
String normalizedRequestedLanguage = normalizeLanguageTag(requestedLanguage);
|
|
917
|
+
for (String candidateLanguage : candidateLanguages) {
|
|
918
|
+
if (normalizedRequestedLanguage.equals(normalizeLanguageTag(candidateLanguage))) {
|
|
919
|
+
return true;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
return false;
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
private String normalizeLanguageTag(String language) {
|
|
927
|
+
return language == null ? "" : language.replace('_', '-').toLowerCase(Locale.US);
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
private Executor mainExecutor() {
|
|
931
|
+
return (command) -> bridge.getActivity().runOnUiThread(command);
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
private void emitListeningState(String stateValue, long currentSessionId, String reason, String errorCode, String status) {
|
|
935
|
+
JSObject payload = new JSObject();
|
|
936
|
+
payload.put("state", stateValue);
|
|
937
|
+
payload.put("sessionId", currentSessionId);
|
|
938
|
+
payload.put("reason", reason);
|
|
939
|
+
if (errorCode != null) {
|
|
940
|
+
payload.put("errorCode", errorCode);
|
|
941
|
+
}
|
|
942
|
+
if (status != null) {
|
|
943
|
+
payload.put("status", status);
|
|
944
|
+
}
|
|
945
|
+
notifyListeners(LISTENING_EVENT, payload);
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
private void emitErrorEvent(String errorCode, String message, long currentSessionId) {
|
|
949
|
+
JSObject payload = new JSObject();
|
|
950
|
+
payload.put("code", errorCode);
|
|
951
|
+
payload.put("message", message);
|
|
952
|
+
payload.put("sessionId", currentSessionId);
|
|
953
|
+
notifyListeners(ERROR_EVENT, payload);
|
|
268
954
|
}
|
|
269
955
|
|
|
270
956
|
private String permissionStateValue(PermissionState state) {
|
|
@@ -280,11 +966,36 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
280
966
|
}
|
|
281
967
|
}
|
|
282
968
|
|
|
969
|
+
@Override
|
|
970
|
+
protected void handleOnDestroy() {
|
|
971
|
+
super.handleOnDestroy();
|
|
972
|
+
handler.removeCallbacksAndMessages(null);
|
|
973
|
+
try {
|
|
974
|
+
lock.lock();
|
|
975
|
+
destroyCurrentRecognizerLocked();
|
|
976
|
+
activeStartCall = null;
|
|
977
|
+
pendingStopReason = null;
|
|
978
|
+
popupSessionActive = false;
|
|
979
|
+
popupSessionCancelled = false;
|
|
980
|
+
listening(false);
|
|
981
|
+
state = ListeningState.IDLE;
|
|
982
|
+
} finally {
|
|
983
|
+
lock.unlock();
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
|
|
283
987
|
private class SpeechRecognitionListener implements RecognitionListener {
|
|
284
988
|
|
|
989
|
+
private final long listenerSessionId;
|
|
990
|
+
private final long listenerGeneration;
|
|
285
991
|
private PluginCall call;
|
|
286
992
|
private boolean partialResults;
|
|
287
993
|
|
|
994
|
+
SpeechRecognitionListener(long listenerSessionId, long listenerGeneration) {
|
|
995
|
+
this.listenerSessionId = listenerSessionId;
|
|
996
|
+
this.listenerGeneration = listenerGeneration;
|
|
997
|
+
}
|
|
998
|
+
|
|
288
999
|
public void setCall(PluginCall call) {
|
|
289
1000
|
this.call = call;
|
|
290
1001
|
}
|
|
@@ -297,17 +1008,7 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
297
1008
|
public void onReadyForSpeech(Bundle params) {}
|
|
298
1009
|
|
|
299
1010
|
@Override
|
|
300
|
-
public void onBeginningOfSpeech() {
|
|
301
|
-
try {
|
|
302
|
-
lock.lock();
|
|
303
|
-
JSObject ret = new JSObject();
|
|
304
|
-
ret.put("status", "started");
|
|
305
|
-
notifyListeners(LISTENING_EVENT, ret);
|
|
306
|
-
Logger.debug(TAG, "Listening started");
|
|
307
|
-
} finally {
|
|
308
|
-
lock.unlock();
|
|
309
|
-
}
|
|
310
|
-
}
|
|
1011
|
+
public void onBeginningOfSpeech() {}
|
|
311
1012
|
|
|
312
1013
|
@Override
|
|
313
1014
|
public void onRmsChanged(float rmsdB) {}
|
|
@@ -316,72 +1017,154 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
316
1017
|
public void onBufferReceived(byte[] buffer) {}
|
|
317
1018
|
|
|
318
1019
|
@Override
|
|
319
|
-
public void onEndOfSpeech() {
|
|
320
|
-
bridge
|
|
321
|
-
.getWebView()
|
|
322
|
-
.post(() -> {
|
|
323
|
-
try {
|
|
324
|
-
lock.lock();
|
|
325
|
-
listening(false);
|
|
326
|
-
|
|
327
|
-
JSObject ret = new JSObject();
|
|
328
|
-
ret.put("status", "stopped");
|
|
329
|
-
notifyListeners(LISTENING_EVENT, ret);
|
|
330
|
-
} finally {
|
|
331
|
-
lock.unlock();
|
|
332
|
-
}
|
|
333
|
-
});
|
|
334
|
-
}
|
|
1020
|
+
public void onEndOfSpeech() {}
|
|
335
1021
|
|
|
336
1022
|
@Override
|
|
337
1023
|
public void onError(int error) {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
1024
|
+
if (isStale()) {
|
|
1025
|
+
return;
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
String errorCode = getErrorCode(error);
|
|
1029
|
+
String errorMessage = getErrorText(error);
|
|
1030
|
+
emitErrorEvent(errorCode, errorMessage, listenerSessionId);
|
|
1031
|
+
|
|
1032
|
+
boolean restartContinuous;
|
|
1033
|
+
try {
|
|
1034
|
+
lock.lock();
|
|
1035
|
+
restartContinuous =
|
|
1036
|
+
continuousPTTMode &&
|
|
1037
|
+
pttButtonHeld &&
|
|
1038
|
+
pendingStopReason == null &&
|
|
1039
|
+
(error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT);
|
|
1040
|
+
|
|
1041
|
+
if (restartContinuous && previousPartialResults.length() > 0) {
|
|
1042
|
+
String lastPartial = previousPartialResults.optString(0, "").trim();
|
|
1043
|
+
if (!lastPartial.isEmpty()) {
|
|
1044
|
+
if (accumulatedResults.length() > 0) {
|
|
1045
|
+
accumulatedResults.append(" ");
|
|
1046
|
+
}
|
|
1047
|
+
accumulatedResults.append(lastPartial);
|
|
1048
|
+
}
|
|
1049
|
+
resetPartialResultsCache();
|
|
1050
|
+
listening(false);
|
|
1051
|
+
}
|
|
1052
|
+
} finally {
|
|
1053
|
+
lock.unlock();
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
if (restartContinuous) {
|
|
1057
|
+
scheduleContinuousRestart(listenerSessionId);
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
341
1060
|
|
|
342
|
-
if (call != null) {
|
|
343
|
-
call.reject(
|
|
1061
|
+
if (call != null && !partialResults) {
|
|
1062
|
+
call.reject(errorMessage);
|
|
344
1063
|
}
|
|
1064
|
+
|
|
1065
|
+
String reason = pendingStopReason != null
|
|
1066
|
+
? pendingStopReason
|
|
1067
|
+
: (error == SpeechRecognizer.ERROR_NO_MATCH || error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT ? "silence" : "error");
|
|
1068
|
+
finishSession(listenerSessionId, reason, errorCode);
|
|
345
1069
|
}
|
|
346
1070
|
|
|
347
1071
|
@Override
|
|
348
1072
|
public void onResults(Bundle results) {
|
|
349
|
-
|
|
1073
|
+
if (isStale()) {
|
|
1074
|
+
return;
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
ArrayList<String> matches = buildMatchesWithUnstableText(results);
|
|
1078
|
+
if (matches == null) {
|
|
1079
|
+
matches = new ArrayList<>();
|
|
1080
|
+
}
|
|
1081
|
+
String resultText = matches.isEmpty() ? "" : matches.get(0);
|
|
350
1082
|
|
|
1083
|
+
boolean restartContinuous;
|
|
1084
|
+
JSObject restartPayload = null;
|
|
1085
|
+
JSObject finalPayload = null;
|
|
351
1086
|
try {
|
|
352
|
-
|
|
353
|
-
|
|
1087
|
+
lock.lock();
|
|
1088
|
+
cancelPendingForceStopLocked();
|
|
1089
|
+
previousPartialResults = new JSONArray(matches);
|
|
1090
|
+
restartContinuous = continuousPTTMode && pttButtonHeld && pendingStopReason == null;
|
|
354
1091
|
|
|
355
|
-
if (
|
|
356
|
-
if (!
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
1092
|
+
if (restartContinuous) {
|
|
1093
|
+
if (!resultText.trim().isEmpty()) {
|
|
1094
|
+
if (accumulatedResults.length() > 0) {
|
|
1095
|
+
accumulatedResults.append(" ");
|
|
1096
|
+
}
|
|
1097
|
+
accumulatedResults.append(resultText.trim());
|
|
1098
|
+
}
|
|
1099
|
+
restartPayload = new JSObject();
|
|
1100
|
+
restartPayload.put("matches", new JSArray(matches));
|
|
1101
|
+
restartPayload.put("accumulated", accumulatedResults.toString().trim());
|
|
1102
|
+
restartPayload.put("isRestarting", true);
|
|
1103
|
+
resetPartialResultsCache();
|
|
1104
|
+
listening(false);
|
|
1105
|
+
} else if (partialResults) {
|
|
1106
|
+
finalPayload = new JSObject();
|
|
1107
|
+
finalPayload.put("matches", new JSArray(matches));
|
|
1108
|
+
if (accumulatedResults.length() > 0) {
|
|
1109
|
+
String accumulatedText = accumulatedResults.toString().trim();
|
|
1110
|
+
if (!resultText.trim().isEmpty()) {
|
|
1111
|
+
accumulatedText = accumulatedText + " " + resultText.trim();
|
|
1112
|
+
}
|
|
1113
|
+
finalPayload.put("accumulatedText", accumulatedText.trim());
|
|
362
1114
|
}
|
|
363
1115
|
}
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
|
|
1116
|
+
} finally {
|
|
1117
|
+
lock.unlock();
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
if (restartContinuous) {
|
|
1121
|
+
if (restartPayload != null) {
|
|
1122
|
+
notifyListeners(PARTIAL_RESULTS_EVENT, restartPayload);
|
|
367
1123
|
}
|
|
1124
|
+
scheduleContinuousRestart(listenerSessionId);
|
|
1125
|
+
return;
|
|
368
1126
|
}
|
|
1127
|
+
|
|
1128
|
+
if (call != null && !partialResults) {
|
|
1129
|
+
call.resolve(new JSObject().put("status", "success").put("matches", new JSArray(matches)));
|
|
1130
|
+
} else if (finalPayload != null) {
|
|
1131
|
+
notifyListeners(PARTIAL_RESULTS_EVENT, finalPayload);
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
finishSession(listenerSessionId, pendingStopReason != null ? pendingStopReason : "results", null);
|
|
369
1135
|
}
|
|
370
1136
|
|
|
371
1137
|
@Override
|
|
372
1138
|
public void onPartialResults(Bundle partialResultsBundle) {
|
|
373
|
-
|
|
374
|
-
|
|
1139
|
+
if (isStale() || forceStopped) {
|
|
1140
|
+
return;
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
ArrayList<String> matches = buildMatchesWithUnstableText(partialResultsBundle);
|
|
1144
|
+
if (matches == null || matches.isEmpty()) {
|
|
1145
|
+
return;
|
|
1146
|
+
}
|
|
375
1147
|
|
|
1148
|
+
JSObject payload = null;
|
|
376
1149
|
try {
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
1150
|
+
lock.lock();
|
|
1151
|
+
JSArray matchesJSON = new JSArray(matches);
|
|
1152
|
+
JSONArray nextPartialResults = new JSONArray(matches);
|
|
1153
|
+
if (!previousPartialResults.toString().equals(nextPartialResults.toString())) {
|
|
1154
|
+
previousPartialResults = nextPartialResults;
|
|
1155
|
+
payload = new JSObject();
|
|
1156
|
+
payload.put("matches", matchesJSON);
|
|
1157
|
+
if (accumulatedResults.length() > 0) {
|
|
1158
|
+
payload.put("accumulated", accumulatedResults.toString().trim());
|
|
1159
|
+
}
|
|
383
1160
|
}
|
|
384
|
-
}
|
|
1161
|
+
} finally {
|
|
1162
|
+
lock.unlock();
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
if (payload != null) {
|
|
1166
|
+
notifyListeners(PARTIAL_RESULTS_EVENT, payload);
|
|
1167
|
+
}
|
|
385
1168
|
}
|
|
386
1169
|
|
|
387
1170
|
@Override
|
|
@@ -390,22 +1173,83 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
390
1173
|
if (matches == null) {
|
|
391
1174
|
return;
|
|
392
1175
|
}
|
|
393
|
-
|
|
394
|
-
JSObject ret = new JSObject();
|
|
395
|
-
ret.put("matches", new JSArray(matches));
|
|
396
|
-
notifyListeners(SEGMENT_RESULTS_EVENT, ret);
|
|
397
|
-
Logger.debug(TAG, "Segment results emitted");
|
|
398
|
-
} catch (Exception ignored) {}
|
|
1176
|
+
notifyListeners(SEGMENT_RESULTS_EVENT, new JSObject().put("matches", new JSArray(matches)));
|
|
399
1177
|
}
|
|
400
1178
|
|
|
401
1179
|
@Override
|
|
402
1180
|
public void onEndOfSegmentedSession() {
|
|
403
1181
|
notifyListeners(END_OF_SEGMENT_EVENT, new JSObject());
|
|
404
|
-
Logger.debug(TAG, "Segmented session ended");
|
|
405
1182
|
}
|
|
406
1183
|
|
|
407
1184
|
@Override
|
|
408
1185
|
public void onEvent(int eventType, Bundle params) {}
|
|
1186
|
+
|
|
1187
|
+
private boolean isStale() {
|
|
1188
|
+
try {
|
|
1189
|
+
lock.lock();
|
|
1190
|
+
return listenerSessionId != sessionId || listenerGeneration != recognizerGeneration;
|
|
1191
|
+
} finally {
|
|
1192
|
+
lock.unlock();
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
private ArrayList<String> buildMatchesWithUnstableText(Bundle resultsBundle) {
|
|
1197
|
+
ArrayList<String> matches = resultsBundle.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
|
|
1198
|
+
if (matches == null || matches.isEmpty()) {
|
|
1199
|
+
return matches;
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
String unstableText = resultsBundle.getString("android.speech.extra.UNSTABLE_TEXT");
|
|
1203
|
+
if (unstableText == null) {
|
|
1204
|
+
return matches;
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
String trimmedUnstable = unstableText.trim();
|
|
1208
|
+
if (trimmedUnstable.isEmpty()) {
|
|
1209
|
+
return matches;
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
String firstMatch = matches.get(0);
|
|
1213
|
+
if (firstMatch == null) {
|
|
1214
|
+
return matches;
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
String trimmedFirstMatch = firstMatch.trim();
|
|
1218
|
+
if (trimmedFirstMatch.equals(trimmedUnstable) || trimmedFirstMatch.endsWith(" " + trimmedUnstable)) {
|
|
1219
|
+
return matches;
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
ArrayList<String> mergedMatches = new ArrayList<>(matches);
|
|
1223
|
+
mergedMatches.set(0, trimmedFirstMatch + " " + trimmedUnstable);
|
|
1224
|
+
return mergedMatches;
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
private String getErrorCode(int errorCode) {
|
|
1229
|
+
switch (errorCode) {
|
|
1230
|
+
case SpeechRecognizer.ERROR_AUDIO:
|
|
1231
|
+
return "AUDIO";
|
|
1232
|
+
case SpeechRecognizer.ERROR_CLIENT:
|
|
1233
|
+
return "CLIENT";
|
|
1234
|
+
case SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS:
|
|
1235
|
+
return "INSUFFICIENT_PERMISSIONS";
|
|
1236
|
+
case SpeechRecognizer.ERROR_NETWORK:
|
|
1237
|
+
return "NETWORK";
|
|
1238
|
+
case SpeechRecognizer.ERROR_NETWORK_TIMEOUT:
|
|
1239
|
+
return "NETWORK_TIMEOUT";
|
|
1240
|
+
case SpeechRecognizer.ERROR_NO_MATCH:
|
|
1241
|
+
return "NO_MATCH";
|
|
1242
|
+
case SpeechRecognizer.ERROR_RECOGNIZER_BUSY:
|
|
1243
|
+
return "RECOGNIZER_BUSY";
|
|
1244
|
+
case SpeechRecognizer.ERROR_SERVER:
|
|
1245
|
+
return "SERVER";
|
|
1246
|
+
case SpeechRecognizer.ERROR_SERVER_DISCONNECTED:
|
|
1247
|
+
return "SERVER_DISCONNECTED";
|
|
1248
|
+
case SpeechRecognizer.ERROR_SPEECH_TIMEOUT:
|
|
1249
|
+
return "SPEECH_TIMEOUT";
|
|
1250
|
+
default:
|
|
1251
|
+
return "UNKNOWN_" + errorCode;
|
|
1252
|
+
}
|
|
409
1253
|
}
|
|
410
1254
|
|
|
411
1255
|
private String getErrorText(int errorCode) {
|
|
@@ -426,10 +1270,12 @@ public class SpeechRecognitionPlugin extends Plugin implements Constants {
|
|
|
426
1270
|
return "RecognitionService busy";
|
|
427
1271
|
case SpeechRecognizer.ERROR_SERVER:
|
|
428
1272
|
return "Error from server";
|
|
1273
|
+
case SpeechRecognizer.ERROR_SERVER_DISCONNECTED:
|
|
1274
|
+
return "Server disconnected";
|
|
429
1275
|
case SpeechRecognizer.ERROR_SPEECH_TIMEOUT:
|
|
430
1276
|
return "No speech input";
|
|
431
1277
|
default:
|
|
432
|
-
return "Didn't understand, please try again.";
|
|
1278
|
+
return "Didn't understand, please try again. Error code: " + errorCode;
|
|
433
1279
|
}
|
|
434
1280
|
}
|
|
435
1281
|
}
|