@agent-native/core 0.84.12 → 0.84.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +11 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/durable-background.ts +30 -0
- package/corpus/core/src/agent/production-agent.ts +34 -6
- package/corpus/core/src/agent/run-manager.ts +45 -2
- package/corpus/core/src/chat-threads/store.ts +2 -0
- package/corpus/core/src/cli/skills.ts +10 -6
- package/corpus/core/src/client/AssistantChat.tsx +40 -9
- package/corpus/core/src/client/agent-chat-adapter.ts +23 -2
- package/corpus/core/src/client/chat/run-recovery.tsx +52 -44
- package/corpus/core/src/client/chat/tool-call-display.tsx +7 -2
- package/corpus/core/src/server/agent-chat-plugin.ts +1 -0
- package/corpus/core/src/shared/streaming-text-smoothing.ts +10 -5
- package/corpus/templates/clips/app/components/library/recording-card.tsx +14 -4
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +10 -12
- package/corpus/templates/clips/app/hooks/use-library.ts +7 -4
- package/corpus/templates/clips/app/lib/recording-status.ts +32 -0
- package/corpus/templates/clips/changelog/2026-07-01-dictation-keeps-listening-through-natural-pauses-and-gives-c.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-01-redoing-a-paused-desktop-recording-no-longer-leaves-the-recording-controls-disabled.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-01-stalled-clip-uploads-now-update-in-the-library-and-surface-as-failed.md +6 -0
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +36 -18
- package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts +99 -27
- package/corpus/templates/clips/desktop/src/overlays/flow-bar.tsx +4 -43
- package/corpus/templates/clips/desktop/src/overlays/toolbar.tsx +3 -0
- package/corpus/templates/clips/desktop/src/styles.css +5 -48
- package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +4 -5
- package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +15 -6
- package/corpus/templates/design/AGENTS.md +8 -4
- package/corpus/templates/design/actions/create-design.ts +1 -0
- package/corpus/templates/design/actions/duplicate-design.ts +3 -1
- package/corpus/templates/design/actions/edit-design.ts +43 -13
- package/corpus/templates/design/actions/generate-design.ts +5 -1
- package/corpus/templates/design/actions/present-design-variants.ts +17 -12
- package/corpus/templates/design/app/hooks/use-question-flow.ts +2 -2
- package/corpus/templates/design/app/pages/DesignEditor.tsx +2 -2
- package/corpus/templates/design/changelog/2026-07-01-org-members-can-see-design-projects-created-in-their-workspace.md +6 -0
- package/corpus/templates/design/server/plugins/db.ts +9 -0
- package/dist/agent/durable-background.d.ts +3 -0
- package/dist/agent/durable-background.d.ts.map +1 -1
- package/dist/agent/durable-background.js +19 -0
- package/dist/agent/durable-background.js.map +1 -1
- package/dist/agent/production-agent.d.ts +1 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +29 -6
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +40 -2
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/chat-threads/store.d.ts +1 -0
- package/dist/chat-threads/store.d.ts.map +1 -1
- package/dist/chat-threads/store.js +2 -0
- package/dist/chat-threads/store.js.map +1 -1
- package/dist/cli/skills.d.ts.map +1 -1
- package/dist/cli/skills.js +10 -6
- package/dist/cli/skills.js.map +1 -1
- package/dist/client/AssistantChat.d.ts +10 -0
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +26 -8
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +21 -2
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/chat/run-recovery.d.ts.map +1 -1
- package/dist/client/chat/run-recovery.js +15 -10
- package/dist/client/chat/run-recovery.js.map +1 -1
- package/dist/client/chat/tool-call-display.d.ts +1 -0
- package/dist/client/chat/tool-call-display.d.ts.map +1 -1
- package/dist/client/chat/tool-call-display.js +3 -2
- package/dist/client/chat/tool-call-display.js.map +1 -1
- package/dist/collab/routes.d.ts +2 -2
- package/dist/notifications/routes.d.ts +2 -2
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +1 -1
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/shared/streaming-text-smoothing.d.ts.map +1 -1
- package/dist/shared/streaming-text-smoothing.js +8 -5
- package/dist/shared/streaming-text-smoothing.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
export const SMOOTH_STREAMING_COMMIT_INTERVAL_MS = 16;
|
|
2
2
|
export const SMOOTH_STREAMING_LONG_TEXT_THRESHOLD_GRAPHEMES = 640;
|
|
3
3
|
export const SMOOTH_STREAMING_LONG_TEXT_TAIL_GRAPHEMES = 180;
|
|
4
|
+
const SMOOTH_STREAMING_SPEED_MULTIPLIER = 2;
|
|
4
5
|
|
|
5
6
|
type SegmenterInstance = {
|
|
6
7
|
segment(input: string): Iterable<{ segment: string }>;
|
|
@@ -137,7 +138,7 @@ export function smoothStreamingRevealCount({
|
|
|
137
138
|
return 0;
|
|
138
139
|
}
|
|
139
140
|
|
|
140
|
-
const
|
|
141
|
+
const baseCharactersPerSecond = inputDone
|
|
141
142
|
? backlog > 800
|
|
142
143
|
? 900
|
|
143
144
|
: 420
|
|
@@ -148,8 +149,12 @@ export function smoothStreamingRevealCount({
|
|
|
148
149
|
: backlog > 180
|
|
149
150
|
? 190
|
|
150
151
|
: 95;
|
|
152
|
+
const charactersPerSecond =
|
|
153
|
+
baseCharactersPerSecond * SMOOTH_STREAMING_SPEED_MULTIPLIER;
|
|
151
154
|
|
|
152
|
-
const maxBurst =
|
|
155
|
+
const maxBurst =
|
|
156
|
+
(inputDone ? 160 : backlog > 1400 ? 120 : 72) *
|
|
157
|
+
SMOOTH_STREAMING_SPEED_MULTIPLIER;
|
|
153
158
|
const count = Math.floor((elapsedMs / 1000) * charactersPerSecond);
|
|
154
159
|
|
|
155
160
|
return Math.min(backlog, Math.max(1, count), maxBurst);
|
|
@@ -164,15 +169,15 @@ export function smoothStreamingPunctuationDelayMs(
|
|
|
164
169
|
}
|
|
165
170
|
|
|
166
171
|
if (grapheme === "\n") {
|
|
167
|
-
return 80;
|
|
172
|
+
return 80 / SMOOTH_STREAMING_SPEED_MULTIPLIER;
|
|
168
173
|
}
|
|
169
174
|
|
|
170
175
|
if (/[.!?)]/.test(grapheme)) {
|
|
171
|
-
return 70;
|
|
176
|
+
return 70 / SMOOTH_STREAMING_SPEED_MULTIPLIER;
|
|
172
177
|
}
|
|
173
178
|
|
|
174
179
|
if (/[,;:]/.test(grapheme)) {
|
|
175
|
-
return 35;
|
|
180
|
+
return 35 / SMOOTH_STREAMING_SPEED_MULTIPLIER;
|
|
176
181
|
}
|
|
177
182
|
|
|
178
183
|
return 0;
|
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
} from "@/components/ui/dropdown-menu";
|
|
29
29
|
import { isDefaultTitle } from "@/hooks/use-auto-title";
|
|
30
30
|
import type { RecordingSummary } from "@/hooks/use-library";
|
|
31
|
+
import { isStaleRecordingUpload } from "@/lib/recording-status";
|
|
31
32
|
import { isStorageSetupFailureReason } from "@/lib/storage-failures";
|
|
32
33
|
import { cn } from "@/lib/utils";
|
|
33
34
|
|
|
@@ -100,6 +101,12 @@ export function RecordingCard({
|
|
|
100
101
|
const waitingForStorage = isStorageSetupFailureReason(
|
|
101
102
|
recording.failureReason,
|
|
102
103
|
);
|
|
104
|
+
const staleUpload = isStaleRecordingUpload(recording);
|
|
105
|
+
const displayFailed = recording.status === "failed" || staleUpload;
|
|
106
|
+
const failureReason = staleUpload
|
|
107
|
+
? (recording.failureReason ??
|
|
108
|
+
t("recordingPage.processingStuck", { status: recording.status }))
|
|
109
|
+
: (recording.failureReason ?? t("clipsFinalRaw.removeFailedClip"));
|
|
103
110
|
const nativeUploadPaused =
|
|
104
111
|
recording.status === "failed" &&
|
|
105
112
|
/native recording|native fullscreen|screencapture|avconvert/i.test(
|
|
@@ -221,11 +228,15 @@ export function RecordingCard({
|
|
|
221
228
|
{/* Status pill for non-ready recordings */}
|
|
222
229
|
{recording.status !== "ready" && (
|
|
223
230
|
<div className="absolute top-2 end-2 rounded-full bg-black/80 px-2 py-0.5 text-[10px] font-medium text-white uppercase tracking-wide">
|
|
224
|
-
{waitingForStorage
|
|
231
|
+
{waitingForStorage
|
|
232
|
+
? "storage"
|
|
233
|
+
: staleUpload
|
|
234
|
+
? "failed"
|
|
235
|
+
: recording.status}
|
|
225
236
|
</div>
|
|
226
237
|
)}
|
|
227
238
|
|
|
228
|
-
{(
|
|
239
|
+
{(displayFailed || waitingForStorage) && (
|
|
229
240
|
<div
|
|
230
241
|
className={cn(
|
|
231
242
|
"absolute inset-x-2 bottom-2 rounded-md border bg-background/95 p-2 text-start shadow-sm backdrop-blur",
|
|
@@ -252,8 +263,7 @@ export function RecordingCard({
|
|
|
252
263
|
? t("clipsFinalRaw.connectStorageToFinish")
|
|
253
264
|
: nativeUploadPaused
|
|
254
265
|
? t("clipsFinalRaw.retryFromClipsMenu")
|
|
255
|
-
:
|
|
256
|
-
t("clipsFinalRaw.removeFailedClip"))}
|
|
266
|
+
: failureReason}
|
|
257
267
|
</div>
|
|
258
268
|
</div>
|
|
259
269
|
{!waitingForStorage && (
|
|
@@ -1145,6 +1145,9 @@ export class RecorderEngine {
|
|
|
1145
1145
|
// mid-state — the UI spinner is wired to engine state and would
|
|
1146
1146
|
// hang forever otherwise.
|
|
1147
1147
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
1148
|
+
if (e.name !== "AbortError") {
|
|
1149
|
+
this.rememberUploadFailure(e);
|
|
1150
|
+
}
|
|
1148
1151
|
this.transition("error", { message: e.message });
|
|
1149
1152
|
throw e;
|
|
1150
1153
|
} finally {
|
|
@@ -1183,6 +1186,9 @@ export class RecorderEngine {
|
|
|
1183
1186
|
return this.toFinalizeResult(result, meta);
|
|
1184
1187
|
} catch (err) {
|
|
1185
1188
|
const e = err instanceof Error ? err : new Error(String(err));
|
|
1189
|
+
if (e.name !== "AbortError") {
|
|
1190
|
+
this.rememberUploadFailure(e);
|
|
1191
|
+
}
|
|
1186
1192
|
this.transition("error", { message: e.message });
|
|
1187
1193
|
throw e;
|
|
1188
1194
|
} finally {
|
|
@@ -1595,26 +1601,18 @@ export class RecorderEngine {
|
|
|
1595
1601
|
const failure = err instanceof Error ? err : new Error(String(err));
|
|
1596
1602
|
// User-initiated cancel — cancel() already runs the abortUrl path.
|
|
1597
1603
|
if (failure.name === "AbortError") return;
|
|
1598
|
-
|
|
1604
|
+
this.rememberUploadFailure(failure);
|
|
1599
1605
|
this.emitError(failure);
|
|
1600
1606
|
}
|
|
1601
1607
|
});
|
|
1602
1608
|
}
|
|
1603
1609
|
|
|
1604
|
-
private
|
|
1610
|
+
private rememberUploadFailure(err: Error): void {
|
|
1605
1611
|
if (!this.uploadFailure) {
|
|
1606
1612
|
this.uploadFailure = err;
|
|
1607
1613
|
}
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
await fetch(this.opts.abortUrl, {
|
|
1611
|
-
method: "POST",
|
|
1612
|
-
headers: { "Content-Type": "application/json" },
|
|
1613
|
-
body: JSON.stringify({ reason: err.message }),
|
|
1614
|
-
});
|
|
1615
|
-
} catch {
|
|
1616
|
-
// ignore — the stop path will surface the original upload error.
|
|
1617
|
-
}
|
|
1614
|
+
// Do not call abortUrl for retryable upload failures. retryUpload() reuses
|
|
1615
|
+
// this recording id; cancel() owns the terminal server-side abort path.
|
|
1618
1616
|
}
|
|
1619
1617
|
|
|
1620
1618
|
private async uploadBlobInSlices(
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { useActionQuery, useActionMutation } from "@agent-native/core/client";
|
|
2
2
|
|
|
3
|
+
import { isLiveRecordingUpload } from "@/lib/recording-status";
|
|
4
|
+
|
|
3
5
|
export interface RecordingSummary {
|
|
4
6
|
id: string;
|
|
5
7
|
title: string;
|
|
@@ -72,16 +74,17 @@ export function useRecordings(args: ListRecordingsArgs = {}) {
|
|
|
72
74
|
recordings: Array.isArray(data?.recordings) ? data.recordings : [],
|
|
73
75
|
};
|
|
74
76
|
},
|
|
75
|
-
//
|
|
76
|
-
//
|
|
77
|
-
//
|
|
77
|
+
// Keep a short poll while uploads/processors are active so the library
|
|
78
|
+
// card does not get stuck if the global refresh signal is missed.
|
|
79
|
+
// Also poll for replaceable seed titles so the card upgrades promptly.
|
|
78
80
|
refetchInterval: (q) => {
|
|
79
81
|
const recs = (q.state.data as any)?.recordings as
|
|
80
82
|
| RecordingSummary[]
|
|
81
83
|
| undefined;
|
|
82
84
|
if (!recs || recs.length === 0) return false;
|
|
85
|
+
const pendingUpload = recs.some((rec) => isLiveRecordingUpload(rec));
|
|
83
86
|
const pendingTitle = recs.some(isAwaitingAutoTitle);
|
|
84
|
-
return pendingTitle ? 3000 : false;
|
|
87
|
+
return pendingUpload || pendingTitle ? 3000 : false;
|
|
85
88
|
},
|
|
86
89
|
},
|
|
87
90
|
);
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export const STALE_RECORDING_UPLOAD_MS = 30 * 60 * 1000;
|
|
2
|
+
|
|
3
|
+
type RecordingStatusLike = {
|
|
4
|
+
status?: string | null;
|
|
5
|
+
updatedAt?: string | null;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export function isActiveRecordingUploadStatus(
|
|
9
|
+
status: string | null | undefined,
|
|
10
|
+
): boolean {
|
|
11
|
+
return status === "uploading" || status === "processing";
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function isStaleRecordingUpload(
|
|
15
|
+
recording: RecordingStatusLike,
|
|
16
|
+
nowMs = Date.now(),
|
|
17
|
+
): boolean {
|
|
18
|
+
if (!isActiveRecordingUploadStatus(recording.status)) return false;
|
|
19
|
+
const updatedAtMs = Date.parse(recording.updatedAt ?? "");
|
|
20
|
+
if (!Number.isFinite(updatedAtMs)) return false;
|
|
21
|
+
return nowMs - updatedAtMs >= STALE_RECORDING_UPLOAD_MS;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function isLiveRecordingUpload(
|
|
25
|
+
recording: RecordingStatusLike,
|
|
26
|
+
nowMs = Date.now(),
|
|
27
|
+
): boolean {
|
|
28
|
+
if (!isActiveRecordingUploadStatus(recording.status)) return false;
|
|
29
|
+
const updatedAtMs = Date.parse(recording.updatedAt ?? "");
|
|
30
|
+
if (!Number.isFinite(updatedAtMs)) return true;
|
|
31
|
+
return nowMs - updatedAtMs < STALE_RECORDING_UPLOAD_MS;
|
|
32
|
+
}
|
|
@@ -1278,6 +1278,18 @@ async function trashRecording(
|
|
|
1278
1278
|
}
|
|
1279
1279
|
}
|
|
1280
1280
|
|
|
1281
|
+
async function cleanupCancelledRemoteRecording(
|
|
1282
|
+
serverUrl: string,
|
|
1283
|
+
recordingId: string,
|
|
1284
|
+
): Promise<void> {
|
|
1285
|
+
await abortRecordingUpload(
|
|
1286
|
+
serverUrl,
|
|
1287
|
+
recordingId,
|
|
1288
|
+
"Recording cancelled by user",
|
|
1289
|
+
);
|
|
1290
|
+
await trashRecording(serverUrl, recordingId);
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1281
1293
|
class CountdownCancelledError extends Error {
|
|
1282
1294
|
constructor() {
|
|
1283
1295
|
super("Recording cancelled during countdown");
|
|
@@ -2029,7 +2041,12 @@ async function startNativeFullscreenRecording(
|
|
|
2029
2041
|
}
|
|
2030
2042
|
stateUnlistens.forEach((u) => u());
|
|
2031
2043
|
stateUnlistens = [];
|
|
2032
|
-
|
|
2044
|
+
void transcriptionCapture?.cancel().catch((err) => {
|
|
2045
|
+
console.warn(
|
|
2046
|
+
"[clips-recorder] native transcription cancel failed:",
|
|
2047
|
+
err,
|
|
2048
|
+
);
|
|
2049
|
+
});
|
|
2033
2050
|
await localCameraExport?.cancel().catch(() => {});
|
|
2034
2051
|
await invoke("native_fullscreen_recording_cancel").catch((err) =>
|
|
2035
2052
|
console.warn(
|
|
@@ -2049,12 +2066,14 @@ async function startNativeFullscreenRecording(
|
|
|
2049
2066
|
streamCleanups.forEach((cleanup) => cleanup());
|
|
2050
2067
|
await invoke("hide_overlays").catch(() => {});
|
|
2051
2068
|
if (!localOnly && id) {
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2069
|
+
void cleanupCancelledRemoteRecording(params.serverUrl, id).catch(
|
|
2070
|
+
(err) => {
|
|
2071
|
+
console.warn(
|
|
2072
|
+
"[clips-recorder] cancelled recording cleanup failed:",
|
|
2073
|
+
err,
|
|
2074
|
+
);
|
|
2075
|
+
},
|
|
2056
2076
|
);
|
|
2057
|
-
await trashRecording(params.serverUrl, id);
|
|
2058
2077
|
}
|
|
2059
2078
|
})();
|
|
2060
2079
|
return cancelPromise;
|
|
@@ -3244,7 +3263,9 @@ async function startRecordingInner(
|
|
|
3244
3263
|
if (tickHandle) clearInterval(tickHandle);
|
|
3245
3264
|
stateUnlistens.forEach((u) => u());
|
|
3246
3265
|
stateUnlistens = [];
|
|
3247
|
-
transcriptionCapture?.cancel().catch(() => {
|
|
3266
|
+
void transcriptionCapture?.cancel().catch((err) => {
|
|
3267
|
+
console.warn("[clips-recorder] transcription cancel failed:", err);
|
|
3268
|
+
});
|
|
3248
3269
|
// Remove MediaRecorder's data handler so any final `ondataavailable`
|
|
3249
3270
|
// from the stop() below doesn't push a new Blob into `inflight`
|
|
3250
3271
|
// after we've decided to discard everything.
|
|
@@ -3282,18 +3303,15 @@ async function startRecordingInner(
|
|
|
3282
3303
|
inflight.clear();
|
|
3283
3304
|
await invoke("hide_recording_chrome").catch(() => {});
|
|
3284
3305
|
// Tell the server to abort the partial recording (drops chunks from
|
|
3285
|
-
// application_state, flips the recording row to 'failed')
|
|
3286
|
-
//
|
|
3287
|
-
//
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3306
|
+
// application_state, flips the recording row to 'failed'), then trash
|
|
3307
|
+
// it. This is best-effort background cleanup: redo/cancel must release
|
|
3308
|
+
// the desktop chrome immediately even if the server is slow or offline.
|
|
3309
|
+
if (id) {
|
|
3310
|
+
void cleanupCancelledRemoteRecording(params.serverUrl, id).catch(
|
|
3311
|
+
(err) => {
|
|
3312
|
+
console.warn("[clips-recorder] abort failed (non-fatal):", err);
|
|
3313
|
+
},
|
|
3293
3314
|
);
|
|
3294
|
-
await trashRecording(params.serverUrl, id);
|
|
3295
|
-
} catch (err) {
|
|
3296
|
-
console.warn("[clips-recorder] abort failed (non-fatal):", err);
|
|
3297
3315
|
}
|
|
3298
3316
|
await deleteBrowserRecordingBackup(id).catch((err) => {
|
|
3299
3317
|
console.warn("[clips-recorder] local backup cleanup failed:", err);
|
|
@@ -102,6 +102,11 @@ interface VoiceSession {
|
|
|
102
102
|
// Accumulated final transcript from interim webkit results, in case
|
|
103
103
|
// the recognition session ends before we ask it to stop.
|
|
104
104
|
browserTranscript: string;
|
|
105
|
+
// Native / Whisper engines can emit one final segment per pause. Keep
|
|
106
|
+
// committed segments separate from the current partial so long dictations
|
|
107
|
+
// survive natural pauses.
|
|
108
|
+
finalTranscriptParts: string[];
|
|
109
|
+
interimTranscript: string;
|
|
105
110
|
// browser-only: monotonic timestamp of the most recent
|
|
106
111
|
// recognition.onresult event. stop() reads this to decide whether
|
|
107
112
|
// the user was actively speaking up to Fn release (worth a tail-
|
|
@@ -229,6 +234,73 @@ function setFlowState(state: FlowState): void {
|
|
|
229
234
|
emit("voice:state-change", { state }).catch(() => {});
|
|
230
235
|
}
|
|
231
236
|
|
|
237
|
+
let lastStartPingAt = 0;
|
|
238
|
+
|
|
239
|
+
function playStartPing(): void {
|
|
240
|
+
const now = Date.now();
|
|
241
|
+
if (now - lastStartPingAt < 250) return;
|
|
242
|
+
lastStartPingAt = now;
|
|
243
|
+
try {
|
|
244
|
+
const AudioCtx = window.AudioContext || (window as any).webkitAudioContext;
|
|
245
|
+
if (!AudioCtx) return;
|
|
246
|
+
const ctx = new AudioCtx();
|
|
247
|
+
const osc = ctx.createOscillator();
|
|
248
|
+
const gain = ctx.createGain();
|
|
249
|
+
const start = ctx.currentTime;
|
|
250
|
+
osc.type = "sine";
|
|
251
|
+
osc.frequency.setValueAtTime(880, start);
|
|
252
|
+
osc.frequency.exponentialRampToValueAtTime(1174.66, start + 0.08);
|
|
253
|
+
gain.gain.setValueAtTime(0.0001, start);
|
|
254
|
+
gain.gain.exponentialRampToValueAtTime(0.08, start + 0.015);
|
|
255
|
+
gain.gain.exponentialRampToValueAtTime(0.0001, start + 0.13);
|
|
256
|
+
osc.connect(gain);
|
|
257
|
+
gain.connect(ctx.destination);
|
|
258
|
+
osc.start(start);
|
|
259
|
+
osc.stop(start + 0.14);
|
|
260
|
+
window.setTimeout(() => {
|
|
261
|
+
ctx.close().catch(() => {});
|
|
262
|
+
}, 250);
|
|
263
|
+
} catch {
|
|
264
|
+
// Sound feedback is best-effort; the moving bars remain the visual cue.
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function enterRecordingState(): void {
|
|
269
|
+
setFlowState("recording");
|
|
270
|
+
playStartPing();
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function joinedTranscript(parts: string[], interim: string): string {
|
|
274
|
+
return [...parts, interim].filter(Boolean).join(" ").trim();
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function appendFinalTranscript(session: VoiceSession, text: string): void {
|
|
278
|
+
const clean = text.trim();
|
|
279
|
+
if (!clean) return;
|
|
280
|
+
const committed = joinedTranscript(session.finalTranscriptParts, "");
|
|
281
|
+
if (!committed) {
|
|
282
|
+
session.finalTranscriptParts = [clean];
|
|
283
|
+
} else if (clean === committed || clean.startsWith(`${committed} `)) {
|
|
284
|
+
// Some recognizers send the whole dictation as their final result.
|
|
285
|
+
session.finalTranscriptParts = [clean];
|
|
286
|
+
} else if (session.finalTranscriptParts.at(-1) !== clean) {
|
|
287
|
+
session.finalTranscriptParts.push(clean);
|
|
288
|
+
}
|
|
289
|
+
session.interimTranscript = "";
|
|
290
|
+
session.browserTranscript = joinedTranscript(
|
|
291
|
+
session.finalTranscriptParts,
|
|
292
|
+
"",
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function setInterimTranscript(session: VoiceSession, text: string): void {
|
|
297
|
+
session.interimTranscript = text.trim();
|
|
298
|
+
session.browserTranscript = joinedTranscript(
|
|
299
|
+
session.finalTranscriptParts,
|
|
300
|
+
session.interimTranscript,
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
|
|
232
304
|
function stopMeter(session: VoiceSession): void {
|
|
233
305
|
if (session.raf != null) {
|
|
234
306
|
cancelAnimationFrame(session.raf);
|
|
@@ -698,7 +770,6 @@ export function installDesktopVoiceDictation(
|
|
|
698
770
|
} catch (err) {
|
|
699
771
|
console.warn("[voice-dictation] vocab learn-monitor failed:", err);
|
|
700
772
|
}
|
|
701
|
-
emit("voice:partial-transcript", { text }).catch(() => {});
|
|
702
773
|
if (target.cleanupProvider) setFlowState("idle");
|
|
703
774
|
return text;
|
|
704
775
|
};
|
|
@@ -808,7 +879,7 @@ export function installDesktopVoiceDictation(
|
|
|
808
879
|
abortPendingStart();
|
|
809
880
|
return;
|
|
810
881
|
}
|
|
811
|
-
|
|
882
|
+
enterRecordingState();
|
|
812
883
|
const mimeType = pickMimeType();
|
|
813
884
|
const recorder = new MediaRecorder(stream, { mimeType });
|
|
814
885
|
const next: VoiceSession = {
|
|
@@ -822,6 +893,8 @@ export function installDesktopVoiceDictation(
|
|
|
822
893
|
mimeType: recorder.mimeType || mimeType,
|
|
823
894
|
recognition: null,
|
|
824
895
|
browserTranscript: "",
|
|
896
|
+
finalTranscriptParts: [],
|
|
897
|
+
interimTranscript: "",
|
|
825
898
|
lastResultAt: 0,
|
|
826
899
|
startedAt: Date.now(),
|
|
827
900
|
stopping: false,
|
|
@@ -948,7 +1021,7 @@ export function installDesktopVoiceDictation(
|
|
|
948
1021
|
abortPendingStart();
|
|
949
1022
|
return;
|
|
950
1023
|
}
|
|
951
|
-
|
|
1024
|
+
enterRecordingState();
|
|
952
1025
|
// Reset any prior partial transcript display in the flow-bar.
|
|
953
1026
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
954
1027
|
const next: VoiceSession = {
|
|
@@ -962,6 +1035,8 @@ export function installDesktopVoiceDictation(
|
|
|
962
1035
|
mimeType: "",
|
|
963
1036
|
recognition: null,
|
|
964
1037
|
browserTranscript: "",
|
|
1038
|
+
finalTranscriptParts: [],
|
|
1039
|
+
interimTranscript: "",
|
|
965
1040
|
lastResultAt: 0,
|
|
966
1041
|
startedAt: Date.now(),
|
|
967
1042
|
stopping: false,
|
|
@@ -1043,7 +1118,7 @@ export function installDesktopVoiceDictation(
|
|
|
1043
1118
|
abortPendingStart();
|
|
1044
1119
|
return;
|
|
1045
1120
|
}
|
|
1046
|
-
|
|
1121
|
+
enterRecordingState();
|
|
1047
1122
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
1048
1123
|
const next: VoiceSession = {
|
|
1049
1124
|
kind: "whisper",
|
|
@@ -1056,6 +1131,8 @@ export function installDesktopVoiceDictation(
|
|
|
1056
1131
|
mimeType: "",
|
|
1057
1132
|
recognition: null,
|
|
1058
1133
|
browserTranscript: "",
|
|
1134
|
+
finalTranscriptParts: [],
|
|
1135
|
+
interimTranscript: "",
|
|
1059
1136
|
lastResultAt: 0,
|
|
1060
1137
|
startedAt: Date.now(),
|
|
1061
1138
|
stopping: false,
|
|
@@ -1112,7 +1189,7 @@ export function installDesktopVoiceDictation(
|
|
|
1112
1189
|
abortPendingStart();
|
|
1113
1190
|
return;
|
|
1114
1191
|
}
|
|
1115
|
-
|
|
1192
|
+
enterRecordingState();
|
|
1116
1193
|
// Reset any prior partial transcript display in the flow-bar.
|
|
1117
1194
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
1118
1195
|
const recognition = new Ctor();
|
|
@@ -1131,6 +1208,8 @@ export function installDesktopVoiceDictation(
|
|
|
1131
1208
|
mimeType: "",
|
|
1132
1209
|
recognition,
|
|
1133
1210
|
browserTranscript: "",
|
|
1211
|
+
finalTranscriptParts: [],
|
|
1212
|
+
interimTranscript: "",
|
|
1134
1213
|
lastResultAt: 0,
|
|
1135
1214
|
startedAt: Date.now(),
|
|
1136
1215
|
stopping: false,
|
|
@@ -1203,10 +1282,6 @@ export function installDesktopVoiceDictation(
|
|
|
1203
1282
|
// the tail because Web Speech only marks a segment as `isFinal`
|
|
1204
1283
|
// after a confidence-threshold pass.
|
|
1205
1284
|
next.browserTranscript = (finalSoFar + interim).trim();
|
|
1206
|
-
// Stream the live transcript to the flow-bar.
|
|
1207
|
-
emit("voice:partial-transcript", {
|
|
1208
|
-
text: next.browserTranscript,
|
|
1209
|
-
}).catch(() => {});
|
|
1210
1285
|
};
|
|
1211
1286
|
recognition.onerror = (ev) => {
|
|
1212
1287
|
if (ev.error !== "no-speech" && ev.error !== "aborted") {
|
|
@@ -1732,31 +1807,28 @@ export function installDesktopVoiceDictation(
|
|
|
1732
1807
|
if (!current || (current.kind !== "native" && current.kind !== "whisper"))
|
|
1733
1808
|
return;
|
|
1734
1809
|
if (current.cancelled || current.stopping) return;
|
|
1735
|
-
current
|
|
1810
|
+
setInterimTranscript(current, text);
|
|
1736
1811
|
})
|
|
1737
1812
|
.then((u) => unlistens.push(u))
|
|
1738
1813
|
.catch(() => {});
|
|
1739
1814
|
onFinalTranscript(({ text }) => {
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
1742
|
-
//
|
|
1743
|
-
//
|
|
1744
|
-
// late
|
|
1745
|
-
//
|
|
1746
|
-
const current =
|
|
1747
|
-
lingeringSession
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
? lingeringSession
|
|
1815
|
+
// Whisper emits final segments throughout a long dictation whenever the
|
|
1816
|
+
// speaker pauses. Native speech emits its final after stop, when the
|
|
1817
|
+
// stopped session is parked in lingeringSession. Native final events do not
|
|
1818
|
+
// carry a session id, so never fall back to an active native session here:
|
|
1819
|
+
// a late final from the previous stop could otherwise be appended to a new
|
|
1820
|
+
// dictation.
|
|
1821
|
+
const current = lingeringSession
|
|
1822
|
+
? lingeringSession
|
|
1823
|
+
: session && session.kind === "whisper" && !session.stopping
|
|
1824
|
+
? session
|
|
1751
1825
|
: null;
|
|
1752
1826
|
if (!current) return;
|
|
1753
1827
|
if (current.cancelled) return;
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
// finalize sequence now (paste → 1s linger → dismiss).
|
|
1759
|
-
current.onNativeFinalize?.();
|
|
1828
|
+
appendFinalTranscript(current, text);
|
|
1829
|
+
if (current === lingeringSession) {
|
|
1830
|
+
current.onNativeFinalize?.();
|
|
1831
|
+
}
|
|
1760
1832
|
})
|
|
1761
1833
|
.then((u) => unlistens.push(u))
|
|
1762
1834
|
.catch(() => {});
|
|
@@ -3,11 +3,7 @@ import { invoke } from "@tauri-apps/api/core";
|
|
|
3
3
|
import { emit, listen } from "@tauri-apps/api/event";
|
|
4
4
|
import { useEffect, useRef, useState } from "react";
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
onAudioLevel,
|
|
8
|
-
onFinalTranscript,
|
|
9
|
-
onPartialTranscript,
|
|
10
|
-
} from "../lib/transcription-engine";
|
|
6
|
+
import { onAudioLevel } from "../lib/transcription-engine";
|
|
11
7
|
|
|
12
8
|
type FlowState = "idle" | "recording" | "processing" | "complete" | "error";
|
|
13
9
|
|
|
@@ -28,7 +24,6 @@ export function FlowBar() {
|
|
|
28
24
|
// "idle" caused the bar to flash an "EN" language pill that never went
|
|
29
25
|
// away if the start event was missed.
|
|
30
26
|
const [state, setState] = useState<FlowState>("recording");
|
|
31
|
-
const [partialTranscript, setPartialTranscript] = useState("");
|
|
32
27
|
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
|
33
28
|
const levelRef = useRef(0);
|
|
34
29
|
const rafRef = useRef<number | null>(null);
|
|
@@ -63,23 +58,6 @@ export function FlowBar() {
|
|
|
63
58
|
}),
|
|
64
59
|
);
|
|
65
60
|
|
|
66
|
-
trackListen(
|
|
67
|
-
onPartialTranscript(({ text }) => {
|
|
68
|
-
// Live transcript as the user speaks — rendered above the pill.
|
|
69
|
-
// Empty payload clears the display (sent at session start/end).
|
|
70
|
-
setPartialTranscript(text);
|
|
71
|
-
}),
|
|
72
|
-
);
|
|
73
|
-
|
|
74
|
-
trackListen(
|
|
75
|
-
onFinalTranscript(({ text }) => {
|
|
76
|
-
// Final result from the recognizer (only fires after stop is
|
|
77
|
-
// requested). Show it on the bar — the last word lingers there
|
|
78
|
-
// for ~1s before voice-dictation.ts dismisses everything.
|
|
79
|
-
if (text) setPartialTranscript(text);
|
|
80
|
-
}),
|
|
81
|
-
);
|
|
82
|
-
|
|
83
61
|
return () => {
|
|
84
62
|
stopped = true;
|
|
85
63
|
unlistens.forEach((u) => {
|
|
@@ -166,29 +144,12 @@ export function FlowBar() {
|
|
|
166
144
|
}, 250);
|
|
167
145
|
};
|
|
168
146
|
|
|
169
|
-
// The transcript chip is independent of the pill — it can linger on
|
|
170
|
-
// its own after Fn release while the pill dismisses snappily. Voice-
|
|
171
|
-
// dictation.ts emits an empty payload to clear it once the linger
|
|
172
|
-
// window expires.
|
|
173
|
-
const showTranscript = partialTranscript.length > 0;
|
|
174
|
-
|
|
175
147
|
return (
|
|
176
148
|
<div className="flow-bar-root">
|
|
177
|
-
{showTranscript && (
|
|
178
|
-
<div className="flow-bar-transcript">
|
|
179
|
-
{/* <bdi> + unicode-bidi: plaintext (in CSS) keeps Latin text
|
|
180
|
-
in its natural LTR order while the parent's direction:rtl
|
|
181
|
-
clips overflow from the visual left. Without this, the
|
|
182
|
-
last few characters of the newest words were being pushed
|
|
183
|
-
past the visible right edge by bidi reordering. */}
|
|
184
|
-
<bdi>{partialTranscript}</bdi>
|
|
185
|
-
</div>
|
|
186
|
-
)}
|
|
187
149
|
{/* Pill is ALWAYS mounted — when state goes idle we fade the
|
|
188
|
-
opacity to 0 (see CSS) instead of removing it from the DOM
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
during the fade so the canvas doesn't pop. */}
|
|
150
|
+
opacity to 0 (see CSS) instead of removing it from the DOM.
|
|
151
|
+
Inner content keeps its last frame rendered during the fade
|
|
152
|
+
so the canvas doesn't pop. */}
|
|
192
153
|
<div className={`flow-bar flow-bar-${state}`}>
|
|
193
154
|
{(state === "recording" || state === "idle") && (
|
|
194
155
|
<div className="flow-bar-recording">
|
|
@@ -98,8 +98,11 @@ export function Toolbar() {
|
|
|
98
98
|
trackListen(
|
|
99
99
|
listen<boolean>("clips:toolbar-enabled", (ev) => {
|
|
100
100
|
setEnabled(!!ev.payload);
|
|
101
|
+
setPendingAction(null);
|
|
101
102
|
if (!ev.payload) {
|
|
102
103
|
setDiskSpaceLevel("ok");
|
|
104
|
+
setPaused(false);
|
|
105
|
+
setElapsed(0);
|
|
103
106
|
}
|
|
104
107
|
}),
|
|
105
108
|
);
|