@agent-native/core 0.84.12 → 0.84.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +19 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/durable-background.ts +42 -5
- package/corpus/core/src/agent/production-agent.ts +72 -10
- package/corpus/core/src/agent/run-manager.ts +84 -4
- package/corpus/core/src/agent/run-store.ts +55 -11
- package/corpus/core/src/chat-threads/store.ts +2 -0
- package/corpus/core/src/cli/skills.ts +10 -6
- package/corpus/core/src/client/AssistantChat.tsx +40 -9
- package/corpus/core/src/client/agent-chat-adapter.ts +44 -5
- package/corpus/core/src/client/blocks/library/diagram.tsx +3 -3
- package/corpus/core/src/client/chat/message-components.tsx +17 -0
- package/corpus/core/src/client/chat/run-recovery.tsx +52 -44
- package/corpus/core/src/client/chat/tool-call-display.tsx +7 -2
- package/corpus/core/src/observability/traces.ts +17 -3
- package/corpus/core/src/server/agent-chat-plugin.ts +12 -1
- package/corpus/core/src/shared/streaming-text-smoothing.ts +10 -5
- package/corpus/core/src/styles/blocks.css +44 -1
- package/corpus/templates/clips/app/components/library/recording-card.tsx +14 -4
- package/corpus/templates/clips/app/components/recorder/recorder-engine.ts +10 -12
- package/corpus/templates/clips/app/hooks/use-library.ts +7 -4
- package/corpus/templates/clips/app/lib/recording-status.ts +32 -0
- package/corpus/templates/clips/changelog/2026-07-01-chrome-extension-recordings-stream-uploads-while-recording.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-01-dictation-keeps-listening-through-natural-pauses-and-gives-c.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-01-redoing-a-paused-desktop-recording-no-longer-leaves-the-recording-controls-disabled.md +6 -0
- package/corpus/templates/clips/changelog/2026-07-01-stalled-clip-uploads-now-update-in-the-library-and-surface-as-failed.md +6 -0
- package/corpus/templates/clips/chrome-extension/src/background.ts +8 -0
- package/corpus/templates/clips/chrome-extension/src/offscreen.ts +61 -16
- package/corpus/templates/clips/desktop/src/lib/recorder.ts +36 -18
- package/corpus/templates/clips/desktop/src/lib/voice-dictation.ts +99 -27
- package/corpus/templates/clips/desktop/src/overlays/flow-bar.tsx +4 -43
- package/corpus/templates/clips/desktop/src/overlays/toolbar.tsx +3 -0
- package/corpus/templates/clips/desktop/src/styles.css +5 -48
- package/corpus/templates/clips/desktop/src-tauri/src/clips/mod.rs +4 -5
- package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +15 -6
- package/corpus/templates/design/AGENTS.md +8 -4
- package/corpus/templates/design/actions/create-design.ts +1 -0
- package/corpus/templates/design/actions/duplicate-design.ts +3 -1
- package/corpus/templates/design/actions/edit-design.ts +43 -13
- package/corpus/templates/design/actions/generate-design.ts +5 -1
- package/corpus/templates/design/actions/present-design-variants.ts +17 -12
- package/corpus/templates/design/app/hooks/use-question-flow.ts +2 -2
- package/corpus/templates/design/app/pages/DesignEditor.tsx +2 -2
- package/corpus/templates/design/changelog/2026-07-01-org-members-can-see-design-projects-created-in-their-workspace.md +6 -0
- package/corpus/templates/design/server/plugins/db.ts +9 -0
- package/corpus/templates/plan/app/components/plan/wireframe/html-artboard.css +44 -1
- package/corpus/templates/plan/changelog/2026-07-01-visual-plan-and-recap-diagram-labels-now-wrap-inside-their-b.md +6 -0
- package/dist/agent/durable-background.d.ts +3 -0
- package/dist/agent/durable-background.d.ts.map +1 -1
- package/dist/agent/durable-background.js +33 -3
- package/dist/agent/durable-background.js.map +1 -1
- package/dist/agent/production-agent.d.ts +1 -0
- package/dist/agent/production-agent.d.ts.map +1 -1
- package/dist/agent/production-agent.js +55 -9
- package/dist/agent/production-agent.js.map +1 -1
- package/dist/agent/run-manager.d.ts +2 -0
- package/dist/agent/run-manager.d.ts.map +1 -1
- package/dist/agent/run-manager.js +68 -5
- package/dist/agent/run-manager.js.map +1 -1
- package/dist/agent/run-store.d.ts +9 -0
- package/dist/agent/run-store.d.ts.map +1 -1
- package/dist/agent/run-store.js +47 -11
- package/dist/agent/run-store.js.map +1 -1
- package/dist/chat-threads/store.d.ts +1 -0
- package/dist/chat-threads/store.d.ts.map +1 -1
- package/dist/chat-threads/store.js +2 -0
- package/dist/chat-threads/store.js.map +1 -1
- package/dist/cli/skills.d.ts.map +1 -1
- package/dist/cli/skills.js +10 -6
- package/dist/cli/skills.js.map +1 -1
- package/dist/client/AssistantChat.d.ts +10 -0
- package/dist/client/AssistantChat.d.ts.map +1 -1
- package/dist/client/AssistantChat.js +26 -8
- package/dist/client/AssistantChat.js.map +1 -1
- package/dist/client/agent-chat-adapter.d.ts.map +1 -1
- package/dist/client/agent-chat-adapter.js +40 -5
- package/dist/client/agent-chat-adapter.js.map +1 -1
- package/dist/client/blocks/library/diagram.js +3 -3
- package/dist/client/blocks/library/diagram.js.map +1 -1
- package/dist/client/chat/message-components.d.ts +3 -1
- package/dist/client/chat/message-components.d.ts.map +1 -1
- package/dist/client/chat/message-components.js +16 -2
- package/dist/client/chat/message-components.js.map +1 -1
- package/dist/client/chat/run-recovery.d.ts.map +1 -1
- package/dist/client/chat/run-recovery.js +15 -10
- package/dist/client/chat/run-recovery.js.map +1 -1
- package/dist/client/chat/tool-call-display.d.ts +1 -0
- package/dist/client/chat/tool-call-display.d.ts.map +1 -1
- package/dist/client/chat/tool-call-display.js +3 -2
- package/dist/client/chat/tool-call-display.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +6 -6
- package/dist/observability/traces.d.ts +5 -0
- package/dist/observability/traces.d.ts.map +1 -1
- package/dist/observability/traces.js +9 -3
- package/dist/observability/traces.js.map +1 -1
- package/dist/resources/handlers.d.ts +2 -2
- package/dist/server/agent-chat-plugin.d.ts +2 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +8 -3
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/shared/streaming-text-smoothing.d.ts.map +1 -1
- package/dist/shared/streaming-text-smoothing.js +8 -5
- package/dist/shared/streaming-text-smoothing.js.map +1 -1
- package/dist/styles/blocks.css +44 -1
- package/package.json +1 -1
|
@@ -17,7 +17,11 @@
|
|
|
17
17
|
// app recorder via @shared/recording-core so the server contract can't drift.
|
|
18
18
|
|
|
19
19
|
import { scheduleReadyChime } from "@shared/recording-audio";
|
|
20
|
-
import {
|
|
20
|
+
import {
|
|
21
|
+
chunkUploadUrl,
|
|
22
|
+
pickMimeType,
|
|
23
|
+
type UploadMode,
|
|
24
|
+
} from "@shared/recording-core";
|
|
21
25
|
import { MAX_UPLOAD_BYTES } from "@shared/upload-limits";
|
|
22
26
|
|
|
23
27
|
import { waitForReadyRecordingAfterFinalizeError } from "./finalize-recovery";
|
|
@@ -63,6 +67,7 @@ type BeginMessage = {
|
|
|
63
67
|
sessionId: string;
|
|
64
68
|
recordingId: string;
|
|
65
69
|
uploadUrl: string;
|
|
70
|
+
uploadMode?: UploadMode;
|
|
66
71
|
hasCamera?: boolean;
|
|
67
72
|
// Pre-roll countdown delay, owned here in the offscreen document (a reliable
|
|
68
73
|
// context) rather than the service worker (which can suspend and drop timers).
|
|
@@ -114,6 +119,7 @@ type ActiveRecording = {
|
|
|
114
119
|
sessionId: string;
|
|
115
120
|
recordingId: string;
|
|
116
121
|
uploadUrl: string;
|
|
122
|
+
uploadMode: UploadMode;
|
|
117
123
|
authToken: string | null;
|
|
118
124
|
mode: CaptureMode;
|
|
119
125
|
startedAtMs: number;
|
|
@@ -134,6 +140,8 @@ type ActiveRecording = {
|
|
|
134
140
|
// Set if the recording grew past the buffer ceiling and we stopped retaining
|
|
135
141
|
// — at that point a local save can't be guaranteed, so we don't promise one.
|
|
136
142
|
localBufferOverflow: boolean;
|
|
143
|
+
pendingStreamBlobs: Blob[];
|
|
144
|
+
pendingStreamBytes: number;
|
|
137
145
|
cancelled: boolean;
|
|
138
146
|
// Set when the recorder is being torn down to start over on the same source
|
|
139
147
|
// streams, so the stop handler skips the usual track cleanup.
|
|
@@ -155,6 +163,8 @@ type ActiveRecording = {
|
|
|
155
163
|
rejectStopped: (error: Error) => void;
|
|
156
164
|
};
|
|
157
165
|
|
|
166
|
+
const GCS_CHUNK_ALIGN_BYTES = 256 * 1024;
|
|
167
|
+
const STREAM_CHUNK_BYTES = 15 * GCS_CHUNK_ALIGN_BYTES; // 3.75 MiB
|
|
158
168
|
const UPLOAD_SLICE_BYTES = 3 * 1024 * 1024;
|
|
159
169
|
|
|
160
170
|
// Don't retain more than the upload ceiling — past it the server rejects the
|
|
@@ -578,6 +588,32 @@ async function uploadBlobInSlices(
|
|
|
578
588
|
}
|
|
579
589
|
}
|
|
580
590
|
|
|
591
|
+
async function uploadStreamingBlob(
|
|
592
|
+
recording: ActiveRecording,
|
|
593
|
+
blob: Blob,
|
|
594
|
+
): Promise<void> {
|
|
595
|
+
if (blob.size === 0) return;
|
|
596
|
+
recording.pendingStreamBlobs.push(blob);
|
|
597
|
+
recording.pendingStreamBytes += blob.size;
|
|
598
|
+
|
|
599
|
+
while (recording.pendingStreamBytes >= STREAM_CHUNK_BYTES) {
|
|
600
|
+
if (recording.cancelled || recording.uploadFailure) return;
|
|
601
|
+
const combined = new Blob(recording.pendingStreamBlobs, {
|
|
602
|
+
type: recording.mimeType,
|
|
603
|
+
});
|
|
604
|
+
const head = combined.slice(0, STREAM_CHUNK_BYTES, recording.mimeType);
|
|
605
|
+
const tail = combined.slice(
|
|
606
|
+
STREAM_CHUNK_BYTES,
|
|
607
|
+
undefined,
|
|
608
|
+
recording.mimeType,
|
|
609
|
+
);
|
|
610
|
+
recording.pendingStreamBlobs = tail.size > 0 ? [tail] : [];
|
|
611
|
+
recording.pendingStreamBytes = tail.size;
|
|
612
|
+
const index = recording.chunkIndex++;
|
|
613
|
+
await uploadChunk(recording, head, index);
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
581
617
|
function stopStreams(streams: (MediaStream | null)[]): void {
|
|
582
618
|
for (const stream of streams) {
|
|
583
619
|
if (!stream) continue;
|
|
@@ -744,6 +780,7 @@ async function begin(message: BeginMessage): Promise<{
|
|
|
744
780
|
sessionId: ready.sessionId,
|
|
745
781
|
recordingId: message.recordingId,
|
|
746
782
|
uploadUrl: message.uploadUrl,
|
|
783
|
+
uploadMode: message.uploadMode ?? "buffered",
|
|
747
784
|
authToken: message.authToken ?? null,
|
|
748
785
|
mode: ready.mode,
|
|
749
786
|
startedAtMs: 0,
|
|
@@ -763,6 +800,8 @@ async function begin(message: BeginMessage): Promise<{
|
|
|
763
800
|
recordedBlobs: [],
|
|
764
801
|
recordedBytes: 0,
|
|
765
802
|
localBufferOverflow: false,
|
|
803
|
+
pendingStreamBlobs: [],
|
|
804
|
+
pendingStreamBytes: 0,
|
|
766
805
|
cancelled: false,
|
|
767
806
|
restarting: false,
|
|
768
807
|
startTimer: null,
|
|
@@ -798,7 +837,11 @@ async function begin(message: BeginMessage): Promise<{
|
|
|
798
837
|
// rejected promise that surfaces as an "Uncaught (in promise)" error (bad
|
|
799
838
|
// look in a Chrome Web Store review). finalizeStop reads recording.upload-
|
|
800
839
|
// Failure and surfaces it through the normal error path instead.
|
|
801
|
-
const upload =
|
|
840
|
+
const upload = (
|
|
841
|
+
recording.uploadMode === "streaming"
|
|
842
|
+
? uploadStreamingBlob(recording, event.data)
|
|
843
|
+
: uploadBlobInSlices(recording, event.data)
|
|
844
|
+
).catch((err) => {
|
|
802
845
|
recording.uploadFailure =
|
|
803
846
|
err instanceof Error ? err : new Error(String(err));
|
|
804
847
|
captureExtensionError(recording.uploadFailure, {
|
|
@@ -1001,20 +1044,22 @@ async function finalizeStop(recording: ActiveRecording): Promise<void> {
|
|
|
1001
1044
|
}
|
|
1002
1045
|
let result: UploadResult;
|
|
1003
1046
|
try {
|
|
1004
|
-
|
|
1005
|
-
recording
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1047
|
+
const finalBlob =
|
|
1048
|
+
recording.uploadMode === "streaming"
|
|
1049
|
+
? new Blob(recording.pendingStreamBlobs, { type: recording.mimeType })
|
|
1050
|
+
: new Blob([], { type: recording.mimeType });
|
|
1051
|
+
recording.pendingStreamBlobs = [];
|
|
1052
|
+
recording.pendingStreamBytes = 0;
|
|
1053
|
+
const finalIndex = recording.chunkIndex;
|
|
1054
|
+
result = await uploadChunk(recording, finalBlob, finalIndex, {
|
|
1055
|
+
isFinal: true,
|
|
1056
|
+
total: finalIndex + (finalBlob.size > 0 ? 1 : 0),
|
|
1057
|
+
durationMs,
|
|
1058
|
+
width: recording.dimensions.width,
|
|
1059
|
+
height: recording.dimensions.height,
|
|
1060
|
+
hasAudio: recording.hasAudio,
|
|
1061
|
+
hasCamera: recording.hasCamera,
|
|
1062
|
+
});
|
|
1018
1063
|
} catch (err) {
|
|
1019
1064
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
1020
1065
|
(error as { finalUpload?: boolean }).finalUpload = true;
|
|
@@ -1278,6 +1278,18 @@ async function trashRecording(
|
|
|
1278
1278
|
}
|
|
1279
1279
|
}
|
|
1280
1280
|
|
|
1281
|
+
async function cleanupCancelledRemoteRecording(
|
|
1282
|
+
serverUrl: string,
|
|
1283
|
+
recordingId: string,
|
|
1284
|
+
): Promise<void> {
|
|
1285
|
+
await abortRecordingUpload(
|
|
1286
|
+
serverUrl,
|
|
1287
|
+
recordingId,
|
|
1288
|
+
"Recording cancelled by user",
|
|
1289
|
+
);
|
|
1290
|
+
await trashRecording(serverUrl, recordingId);
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1281
1293
|
class CountdownCancelledError extends Error {
|
|
1282
1294
|
constructor() {
|
|
1283
1295
|
super("Recording cancelled during countdown");
|
|
@@ -2029,7 +2041,12 @@ async function startNativeFullscreenRecording(
|
|
|
2029
2041
|
}
|
|
2030
2042
|
stateUnlistens.forEach((u) => u());
|
|
2031
2043
|
stateUnlistens = [];
|
|
2032
|
-
|
|
2044
|
+
void transcriptionCapture?.cancel().catch((err) => {
|
|
2045
|
+
console.warn(
|
|
2046
|
+
"[clips-recorder] native transcription cancel failed:",
|
|
2047
|
+
err,
|
|
2048
|
+
);
|
|
2049
|
+
});
|
|
2033
2050
|
await localCameraExport?.cancel().catch(() => {});
|
|
2034
2051
|
await invoke("native_fullscreen_recording_cancel").catch((err) =>
|
|
2035
2052
|
console.warn(
|
|
@@ -2049,12 +2066,14 @@ async function startNativeFullscreenRecording(
|
|
|
2049
2066
|
streamCleanups.forEach((cleanup) => cleanup());
|
|
2050
2067
|
await invoke("hide_overlays").catch(() => {});
|
|
2051
2068
|
if (!localOnly && id) {
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2069
|
+
void cleanupCancelledRemoteRecording(params.serverUrl, id).catch(
|
|
2070
|
+
(err) => {
|
|
2071
|
+
console.warn(
|
|
2072
|
+
"[clips-recorder] cancelled recording cleanup failed:",
|
|
2073
|
+
err,
|
|
2074
|
+
);
|
|
2075
|
+
},
|
|
2056
2076
|
);
|
|
2057
|
-
await trashRecording(params.serverUrl, id);
|
|
2058
2077
|
}
|
|
2059
2078
|
})();
|
|
2060
2079
|
return cancelPromise;
|
|
@@ -3244,7 +3263,9 @@ async function startRecordingInner(
|
|
|
3244
3263
|
if (tickHandle) clearInterval(tickHandle);
|
|
3245
3264
|
stateUnlistens.forEach((u) => u());
|
|
3246
3265
|
stateUnlistens = [];
|
|
3247
|
-
transcriptionCapture?.cancel().catch(() => {
|
|
3266
|
+
void transcriptionCapture?.cancel().catch((err) => {
|
|
3267
|
+
console.warn("[clips-recorder] transcription cancel failed:", err);
|
|
3268
|
+
});
|
|
3248
3269
|
// Remove MediaRecorder's data handler so any final `ondataavailable`
|
|
3249
3270
|
// from the stop() below doesn't push a new Blob into `inflight`
|
|
3250
3271
|
// after we've decided to discard everything.
|
|
@@ -3282,18 +3303,15 @@ async function startRecordingInner(
|
|
|
3282
3303
|
inflight.clear();
|
|
3283
3304
|
await invoke("hide_recording_chrome").catch(() => {});
|
|
3284
3305
|
// Tell the server to abort the partial recording (drops chunks from
|
|
3285
|
-
// application_state, flips the recording row to 'failed')
|
|
3286
|
-
//
|
|
3287
|
-
//
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3306
|
+
// application_state, flips the recording row to 'failed'), then trash
|
|
3307
|
+
// it. This is best-effort background cleanup: redo/cancel must release
|
|
3308
|
+
// the desktop chrome immediately even if the server is slow or offline.
|
|
3309
|
+
if (id) {
|
|
3310
|
+
void cleanupCancelledRemoteRecording(params.serverUrl, id).catch(
|
|
3311
|
+
(err) => {
|
|
3312
|
+
console.warn("[clips-recorder] abort failed (non-fatal):", err);
|
|
3313
|
+
},
|
|
3293
3314
|
);
|
|
3294
|
-
await trashRecording(params.serverUrl, id);
|
|
3295
|
-
} catch (err) {
|
|
3296
|
-
console.warn("[clips-recorder] abort failed (non-fatal):", err);
|
|
3297
3315
|
}
|
|
3298
3316
|
await deleteBrowserRecordingBackup(id).catch((err) => {
|
|
3299
3317
|
console.warn("[clips-recorder] local backup cleanup failed:", err);
|
|
@@ -102,6 +102,11 @@ interface VoiceSession {
|
|
|
102
102
|
// Accumulated final transcript from interim webkit results, in case
|
|
103
103
|
// the recognition session ends before we ask it to stop.
|
|
104
104
|
browserTranscript: string;
|
|
105
|
+
// Native / Whisper engines can emit one final segment per pause. Keep
|
|
106
|
+
// committed segments separate from the current partial so long dictations
|
|
107
|
+
// survive natural pauses.
|
|
108
|
+
finalTranscriptParts: string[];
|
|
109
|
+
interimTranscript: string;
|
|
105
110
|
// browser-only: monotonic timestamp of the most recent
|
|
106
111
|
// recognition.onresult event. stop() reads this to decide whether
|
|
107
112
|
// the user was actively speaking up to Fn release (worth a tail-
|
|
@@ -229,6 +234,73 @@ function setFlowState(state: FlowState): void {
|
|
|
229
234
|
emit("voice:state-change", { state }).catch(() => {});
|
|
230
235
|
}
|
|
231
236
|
|
|
237
|
+
let lastStartPingAt = 0;
|
|
238
|
+
|
|
239
|
+
function playStartPing(): void {
|
|
240
|
+
const now = Date.now();
|
|
241
|
+
if (now - lastStartPingAt < 250) return;
|
|
242
|
+
lastStartPingAt = now;
|
|
243
|
+
try {
|
|
244
|
+
const AudioCtx = window.AudioContext || (window as any).webkitAudioContext;
|
|
245
|
+
if (!AudioCtx) return;
|
|
246
|
+
const ctx = new AudioCtx();
|
|
247
|
+
const osc = ctx.createOscillator();
|
|
248
|
+
const gain = ctx.createGain();
|
|
249
|
+
const start = ctx.currentTime;
|
|
250
|
+
osc.type = "sine";
|
|
251
|
+
osc.frequency.setValueAtTime(880, start);
|
|
252
|
+
osc.frequency.exponentialRampToValueAtTime(1174.66, start + 0.08);
|
|
253
|
+
gain.gain.setValueAtTime(0.0001, start);
|
|
254
|
+
gain.gain.exponentialRampToValueAtTime(0.08, start + 0.015);
|
|
255
|
+
gain.gain.exponentialRampToValueAtTime(0.0001, start + 0.13);
|
|
256
|
+
osc.connect(gain);
|
|
257
|
+
gain.connect(ctx.destination);
|
|
258
|
+
osc.start(start);
|
|
259
|
+
osc.stop(start + 0.14);
|
|
260
|
+
window.setTimeout(() => {
|
|
261
|
+
ctx.close().catch(() => {});
|
|
262
|
+
}, 250);
|
|
263
|
+
} catch {
|
|
264
|
+
// Sound feedback is best-effort; the moving bars remain the visual cue.
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function enterRecordingState(): void {
|
|
269
|
+
setFlowState("recording");
|
|
270
|
+
playStartPing();
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function joinedTranscript(parts: string[], interim: string): string {
|
|
274
|
+
return [...parts, interim].filter(Boolean).join(" ").trim();
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function appendFinalTranscript(session: VoiceSession, text: string): void {
|
|
278
|
+
const clean = text.trim();
|
|
279
|
+
if (!clean) return;
|
|
280
|
+
const committed = joinedTranscript(session.finalTranscriptParts, "");
|
|
281
|
+
if (!committed) {
|
|
282
|
+
session.finalTranscriptParts = [clean];
|
|
283
|
+
} else if (clean === committed || clean.startsWith(`${committed} `)) {
|
|
284
|
+
// Some recognizers send the whole dictation as their final result.
|
|
285
|
+
session.finalTranscriptParts = [clean];
|
|
286
|
+
} else if (session.finalTranscriptParts.at(-1) !== clean) {
|
|
287
|
+
session.finalTranscriptParts.push(clean);
|
|
288
|
+
}
|
|
289
|
+
session.interimTranscript = "";
|
|
290
|
+
session.browserTranscript = joinedTranscript(
|
|
291
|
+
session.finalTranscriptParts,
|
|
292
|
+
"",
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
function setInterimTranscript(session: VoiceSession, text: string): void {
|
|
297
|
+
session.interimTranscript = text.trim();
|
|
298
|
+
session.browserTranscript = joinedTranscript(
|
|
299
|
+
session.finalTranscriptParts,
|
|
300
|
+
session.interimTranscript,
|
|
301
|
+
);
|
|
302
|
+
}
|
|
303
|
+
|
|
232
304
|
function stopMeter(session: VoiceSession): void {
|
|
233
305
|
if (session.raf != null) {
|
|
234
306
|
cancelAnimationFrame(session.raf);
|
|
@@ -698,7 +770,6 @@ export function installDesktopVoiceDictation(
|
|
|
698
770
|
} catch (err) {
|
|
699
771
|
console.warn("[voice-dictation] vocab learn-monitor failed:", err);
|
|
700
772
|
}
|
|
701
|
-
emit("voice:partial-transcript", { text }).catch(() => {});
|
|
702
773
|
if (target.cleanupProvider) setFlowState("idle");
|
|
703
774
|
return text;
|
|
704
775
|
};
|
|
@@ -808,7 +879,7 @@ export function installDesktopVoiceDictation(
|
|
|
808
879
|
abortPendingStart();
|
|
809
880
|
return;
|
|
810
881
|
}
|
|
811
|
-
|
|
882
|
+
enterRecordingState();
|
|
812
883
|
const mimeType = pickMimeType();
|
|
813
884
|
const recorder = new MediaRecorder(stream, { mimeType });
|
|
814
885
|
const next: VoiceSession = {
|
|
@@ -822,6 +893,8 @@ export function installDesktopVoiceDictation(
|
|
|
822
893
|
mimeType: recorder.mimeType || mimeType,
|
|
823
894
|
recognition: null,
|
|
824
895
|
browserTranscript: "",
|
|
896
|
+
finalTranscriptParts: [],
|
|
897
|
+
interimTranscript: "",
|
|
825
898
|
lastResultAt: 0,
|
|
826
899
|
startedAt: Date.now(),
|
|
827
900
|
stopping: false,
|
|
@@ -948,7 +1021,7 @@ export function installDesktopVoiceDictation(
|
|
|
948
1021
|
abortPendingStart();
|
|
949
1022
|
return;
|
|
950
1023
|
}
|
|
951
|
-
|
|
1024
|
+
enterRecordingState();
|
|
952
1025
|
// Reset any prior partial transcript display in the flow-bar.
|
|
953
1026
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
954
1027
|
const next: VoiceSession = {
|
|
@@ -962,6 +1035,8 @@ export function installDesktopVoiceDictation(
|
|
|
962
1035
|
mimeType: "",
|
|
963
1036
|
recognition: null,
|
|
964
1037
|
browserTranscript: "",
|
|
1038
|
+
finalTranscriptParts: [],
|
|
1039
|
+
interimTranscript: "",
|
|
965
1040
|
lastResultAt: 0,
|
|
966
1041
|
startedAt: Date.now(),
|
|
967
1042
|
stopping: false,
|
|
@@ -1043,7 +1118,7 @@ export function installDesktopVoiceDictation(
|
|
|
1043
1118
|
abortPendingStart();
|
|
1044
1119
|
return;
|
|
1045
1120
|
}
|
|
1046
|
-
|
|
1121
|
+
enterRecordingState();
|
|
1047
1122
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
1048
1123
|
const next: VoiceSession = {
|
|
1049
1124
|
kind: "whisper",
|
|
@@ -1056,6 +1131,8 @@ export function installDesktopVoiceDictation(
|
|
|
1056
1131
|
mimeType: "",
|
|
1057
1132
|
recognition: null,
|
|
1058
1133
|
browserTranscript: "",
|
|
1134
|
+
finalTranscriptParts: [],
|
|
1135
|
+
interimTranscript: "",
|
|
1059
1136
|
lastResultAt: 0,
|
|
1060
1137
|
startedAt: Date.now(),
|
|
1061
1138
|
stopping: false,
|
|
@@ -1112,7 +1189,7 @@ export function installDesktopVoiceDictation(
|
|
|
1112
1189
|
abortPendingStart();
|
|
1113
1190
|
return;
|
|
1114
1191
|
}
|
|
1115
|
-
|
|
1192
|
+
enterRecordingState();
|
|
1116
1193
|
// Reset any prior partial transcript display in the flow-bar.
|
|
1117
1194
|
emit("voice:partial-transcript", { text: "" }).catch(() => {});
|
|
1118
1195
|
const recognition = new Ctor();
|
|
@@ -1131,6 +1208,8 @@ export function installDesktopVoiceDictation(
|
|
|
1131
1208
|
mimeType: "",
|
|
1132
1209
|
recognition,
|
|
1133
1210
|
browserTranscript: "",
|
|
1211
|
+
finalTranscriptParts: [],
|
|
1212
|
+
interimTranscript: "",
|
|
1134
1213
|
lastResultAt: 0,
|
|
1135
1214
|
startedAt: Date.now(),
|
|
1136
1215
|
stopping: false,
|
|
@@ -1203,10 +1282,6 @@ export function installDesktopVoiceDictation(
|
|
|
1203
1282
|
// the tail because Web Speech only marks a segment as `isFinal`
|
|
1204
1283
|
// after a confidence-threshold pass.
|
|
1205
1284
|
next.browserTranscript = (finalSoFar + interim).trim();
|
|
1206
|
-
// Stream the live transcript to the flow-bar.
|
|
1207
|
-
emit("voice:partial-transcript", {
|
|
1208
|
-
text: next.browserTranscript,
|
|
1209
|
-
}).catch(() => {});
|
|
1210
1285
|
};
|
|
1211
1286
|
recognition.onerror = (ev) => {
|
|
1212
1287
|
if (ev.error !== "no-speech" && ev.error !== "aborted") {
|
|
@@ -1732,31 +1807,28 @@ export function installDesktopVoiceDictation(
|
|
|
1732
1807
|
if (!current || (current.kind !== "native" && current.kind !== "whisper"))
|
|
1733
1808
|
return;
|
|
1734
1809
|
if (current.cancelled || current.stopping) return;
|
|
1735
|
-
current
|
|
1810
|
+
setInterimTranscript(current, text);
|
|
1736
1811
|
})
|
|
1737
1812
|
.then((u) => unlistens.push(u))
|
|
1738
1813
|
.catch(() => {});
|
|
1739
1814
|
onFinalTranscript(({ text }) => {
|
|
1740
|
-
//
|
|
1741
|
-
//
|
|
1742
|
-
//
|
|
1743
|
-
//
|
|
1744
|
-
// late
|
|
1745
|
-
//
|
|
1746
|
-
const current =
|
|
1747
|
-
lingeringSession
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
? lingeringSession
|
|
1815
|
+
// Whisper emits final segments throughout a long dictation whenever the
|
|
1816
|
+
// speaker pauses. Native speech emits its final after stop, when the
|
|
1817
|
+
// stopped session is parked in lingeringSession. Native final events do not
|
|
1818
|
+
// carry a session id, so never fall back to an active native session here:
|
|
1819
|
+
// a late final from the previous stop could otherwise be appended to a new
|
|
1820
|
+
// dictation.
|
|
1821
|
+
const current = lingeringSession
|
|
1822
|
+
? lingeringSession
|
|
1823
|
+
: session && session.kind === "whisper" && !session.stopping
|
|
1824
|
+
? session
|
|
1751
1825
|
: null;
|
|
1752
1826
|
if (!current) return;
|
|
1753
1827
|
if (current.cancelled) return;
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
// finalize sequence now (paste → 1s linger → dismiss).
|
|
1759
|
-
current.onNativeFinalize?.();
|
|
1828
|
+
appendFinalTranscript(current, text);
|
|
1829
|
+
if (current === lingeringSession) {
|
|
1830
|
+
current.onNativeFinalize?.();
|
|
1831
|
+
}
|
|
1760
1832
|
})
|
|
1761
1833
|
.then((u) => unlistens.push(u))
|
|
1762
1834
|
.catch(() => {});
|
|
@@ -3,11 +3,7 @@ import { invoke } from "@tauri-apps/api/core";
|
|
|
3
3
|
import { emit, listen } from "@tauri-apps/api/event";
|
|
4
4
|
import { useEffect, useRef, useState } from "react";
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
onAudioLevel,
|
|
8
|
-
onFinalTranscript,
|
|
9
|
-
onPartialTranscript,
|
|
10
|
-
} from "../lib/transcription-engine";
|
|
6
|
+
import { onAudioLevel } from "../lib/transcription-engine";
|
|
11
7
|
|
|
12
8
|
type FlowState = "idle" | "recording" | "processing" | "complete" | "error";
|
|
13
9
|
|
|
@@ -28,7 +24,6 @@ export function FlowBar() {
|
|
|
28
24
|
// "idle" caused the bar to flash an "EN" language pill that never went
|
|
29
25
|
// away if the start event was missed.
|
|
30
26
|
const [state, setState] = useState<FlowState>("recording");
|
|
31
|
-
const [partialTranscript, setPartialTranscript] = useState("");
|
|
32
27
|
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
|
33
28
|
const levelRef = useRef(0);
|
|
34
29
|
const rafRef = useRef<number | null>(null);
|
|
@@ -63,23 +58,6 @@ export function FlowBar() {
|
|
|
63
58
|
}),
|
|
64
59
|
);
|
|
65
60
|
|
|
66
|
-
trackListen(
|
|
67
|
-
onPartialTranscript(({ text }) => {
|
|
68
|
-
// Live transcript as the user speaks — rendered above the pill.
|
|
69
|
-
// Empty payload clears the display (sent at session start/end).
|
|
70
|
-
setPartialTranscript(text);
|
|
71
|
-
}),
|
|
72
|
-
);
|
|
73
|
-
|
|
74
|
-
trackListen(
|
|
75
|
-
onFinalTranscript(({ text }) => {
|
|
76
|
-
// Final result from the recognizer (only fires after stop is
|
|
77
|
-
// requested). Show it on the bar — the last word lingers there
|
|
78
|
-
// for ~1s before voice-dictation.ts dismisses everything.
|
|
79
|
-
if (text) setPartialTranscript(text);
|
|
80
|
-
}),
|
|
81
|
-
);
|
|
82
|
-
|
|
83
61
|
return () => {
|
|
84
62
|
stopped = true;
|
|
85
63
|
unlistens.forEach((u) => {
|
|
@@ -166,29 +144,12 @@ export function FlowBar() {
|
|
|
166
144
|
}, 250);
|
|
167
145
|
};
|
|
168
146
|
|
|
169
|
-
// The transcript chip is independent of the pill — it can linger on
|
|
170
|
-
// its own after Fn release while the pill dismisses snappily. Voice-
|
|
171
|
-
// dictation.ts emits an empty payload to clear it once the linger
|
|
172
|
-
// window expires.
|
|
173
|
-
const showTranscript = partialTranscript.length > 0;
|
|
174
|
-
|
|
175
147
|
return (
|
|
176
148
|
<div className="flow-bar-root">
|
|
177
|
-
{showTranscript && (
|
|
178
|
-
<div className="flow-bar-transcript">
|
|
179
|
-
{/* <bdi> + unicode-bidi: plaintext (in CSS) keeps Latin text
|
|
180
|
-
in its natural LTR order while the parent's direction:rtl
|
|
181
|
-
clips overflow from the visual left. Without this, the
|
|
182
|
-
last few characters of the newest words were being pushed
|
|
183
|
-
past the visible right edge by bidi reordering. */}
|
|
184
|
-
<bdi>{partialTranscript}</bdi>
|
|
185
|
-
</div>
|
|
186
|
-
)}
|
|
187
149
|
{/* Pill is ALWAYS mounted — when state goes idle we fade the
|
|
188
|
-
opacity to 0 (see CSS) instead of removing it from the DOM
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
during the fade so the canvas doesn't pop. */}
|
|
150
|
+
opacity to 0 (see CSS) instead of removing it from the DOM.
|
|
151
|
+
Inner content keeps its last frame rendered during the fade
|
|
152
|
+
so the canvas doesn't pop. */}
|
|
192
153
|
<div className={`flow-bar flow-bar-${state}`}>
|
|
193
154
|
{(state === "recording" || state === "idle") && (
|
|
194
155
|
<div className="flow-bar-recording">
|
|
@@ -98,8 +98,11 @@ export function Toolbar() {
|
|
|
98
98
|
trackListen(
|
|
99
99
|
listen<boolean>("clips:toolbar-enabled", (ev) => {
|
|
100
100
|
setEnabled(!!ev.payload);
|
|
101
|
+
setPendingAction(null);
|
|
101
102
|
if (!ev.payload) {
|
|
102
103
|
setDiskSpaceLevel("ok");
|
|
104
|
+
setPaused(false);
|
|
105
|
+
setElapsed(0);
|
|
103
106
|
}
|
|
104
107
|
}),
|
|
105
108
|
);
|
|
@@ -3663,57 +3663,18 @@ body[data-clips-route="recording-pill"] #root {
|
|
|
3663
3663
|
|
|
3664
3664
|
/* Wispr Flow-style dictation pill: small capsule anchored to the bottom
|
|
3665
3665
|
of its (transparent) overlay window. The window itself is positioned
|
|
3666
|
-
bottom-center on the primary display by show_flow_bar in Rust.
|
|
3667
|
-
stack the live transcript above the pill in a column so the user sees
|
|
3668
|
-
the words being recognized in real time. */
|
|
3666
|
+
bottom-center on the primary display by show_flow_bar in Rust. */
|
|
3669
3667
|
.flow-bar-root {
|
|
3670
3668
|
position: fixed;
|
|
3671
3669
|
inset: var(--overlay-shadow-gutter);
|
|
3672
3670
|
display: flex;
|
|
3673
|
-
flex-direction: column;
|
|
3674
3671
|
align-items: center;
|
|
3675
3672
|
justify-content: flex-end;
|
|
3676
3673
|
padding-bottom: 8px;
|
|
3677
|
-
gap: 6px;
|
|
3678
3674
|
background: transparent;
|
|
3679
3675
|
pointer-events: none;
|
|
3680
3676
|
}
|
|
3681
3677
|
|
|
3682
|
-
/* Live partial-transcript chip floating above the pill. Same dark glass
|
|
3683
|
-
look as the pill, smaller, only there when transcript is non-empty.
|
|
3684
|
-
Caps width and clips from the left so the newest words stay visible
|
|
3685
|
-
when a long sentence reaches the edge of the transparent window. */
|
|
3686
|
-
.flow-bar-transcript {
|
|
3687
|
-
max-width: 360px;
|
|
3688
|
-
padding: 4px 10px;
|
|
3689
|
-
border-radius: 12px;
|
|
3690
|
-
background: rgba(18, 18, 20, 0.92);
|
|
3691
|
-
backdrop-filter: blur(20px);
|
|
3692
|
-
-webkit-backdrop-filter: blur(20px);
|
|
3693
|
-
border: 1px solid rgba(255, 255, 255, 0.06);
|
|
3694
|
-
color: rgba(255, 255, 255, 0.92);
|
|
3695
|
-
font-size: 12px;
|
|
3696
|
-
line-height: 1.35;
|
|
3697
|
-
text-align: left;
|
|
3698
|
-
white-space: nowrap;
|
|
3699
|
-
overflow: hidden;
|
|
3700
|
-
text-overflow: ellipsis;
|
|
3701
|
-
/* RTL on the OUTER box so overflow clips from the visual left
|
|
3702
|
-
(older text), keeping the newest characters pinned to the right.
|
|
3703
|
-
`unicode-bidi: plaintext` on the inner <bdi> makes the actual
|
|
3704
|
-
transcript text follow its own (LTR) direction — without it,
|
|
3705
|
-
bidi reordering pushes the last few Latin characters past the
|
|
3706
|
-
visible right edge and they get clipped a few chars short. */
|
|
3707
|
-
direction: rtl;
|
|
3708
|
-
pointer-events: none;
|
|
3709
|
-
animation: flow-bar-in 100ms ease-out;
|
|
3710
|
-
}
|
|
3711
|
-
|
|
3712
|
-
.flow-bar-transcript > bdi {
|
|
3713
|
-
unicode-bidi: plaintext;
|
|
3714
|
-
direction: ltr;
|
|
3715
|
-
}
|
|
3716
|
-
|
|
3717
3678
|
.flow-bar {
|
|
3718
3679
|
display: flex;
|
|
3719
3680
|
align-items: center;
|
|
@@ -3729,17 +3690,13 @@ body[data-clips-route="recording-pill"] #root {
|
|
|
3729
3690
|
/* Subtle fade-in so the pill doesn't hard-pop on appear. */
|
|
3730
3691
|
animation: flow-bar-in 140ms ease-out;
|
|
3731
3692
|
/* Fade out to opacity 0 when state goes idle (after Fn release).
|
|
3732
|
-
Keeping the pill in the DOM
|
|
3733
|
-
doesn't reflow when the pill "goes away" — important for the
|
|
3734
|
-
linger UX where the chip stays put while the pill dissolves. */
|
|
3693
|
+
Keeping the pill in the DOM lets the waveform fade instead of pop. */
|
|
3735
3694
|
transition: opacity 220ms ease-out;
|
|
3736
3695
|
}
|
|
3737
3696
|
|
|
3738
|
-
/* When the dictation engine reports idle
|
|
3739
|
-
|
|
3740
|
-
|
|
3741
|
-
smoothly fade rather than pop into nothing. Pointer-events go off
|
|
3742
|
-
so the now-invisible chip can't catch clicks. */
|
|
3697
|
+
/* When the dictation engine reports idle, fade the pill out without
|
|
3698
|
+
unmounting it. Pointer-events go off so the invisible pill cannot
|
|
3699
|
+
catch clicks. */
|
|
3743
3700
|
.flow-bar-idle {
|
|
3744
3701
|
opacity: 0;
|
|
3745
3702
|
pointer-events: none;
|
|
@@ -1210,11 +1210,10 @@ pub async fn show_flow_bar(app: AppHandle) -> Result<(), String> {
|
|
|
1210
1210
|
|
|
1211
1211
|
let (mx, my, mw, mh) = tray_monitor_physical_rect(&app);
|
|
1212
1212
|
let scale = overlay_scale_factor(&app);
|
|
1213
|
-
//
|
|
1214
|
-
//
|
|
1215
|
-
|
|
1216
|
-
let
|
|
1217
|
-
let content_h: u32 = (120.0 * scale).round() as u32;
|
|
1213
|
+
// Compact Wispr-style pill window: just enough transparent canvas for
|
|
1214
|
+
// the bottom-centered waveform bar and its shadow gutter.
|
|
1215
|
+
let content_w: u32 = (160.0 * scale).round() as u32;
|
|
1216
|
+
let content_h: u32 = (56.0 * scale).round() as u32;
|
|
1218
1217
|
let bottom_margin: i32 = (14.0 * scale).round() as i32;
|
|
1219
1218
|
let gutter = overlay_shadow_gutter_physical(&app);
|
|
1220
1219
|
let w: u32 = content_w + gutter * 2;
|