@agent-native/core 0.132.1 → 0.132.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +37 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/chat-threads/store.ts +42 -0
- package/corpus/core/src/client/use-chat-threads.ts +76 -16
- package/corpus/core/src/server/agent-chat-plugin.ts +16 -1
- package/corpus/templates/clips/changelog/2026-07-30-meeting-microphone-transcription-works-reliably-from-the-fir.md +6 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +13 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen.rs +2 -0
- package/corpus/templates/clips/desktop/src-tauri/src/system_audio.rs +25 -132
- package/corpus/templates/design/.generated/bridge/editor-chrome.generated.ts +5 -0
- package/corpus/templates/design/app/components/design/KeyboardShortcutsPanel.tsx +5 -3
- package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +23 -0
- package/corpus/templates/design/app/components/design/keyboard-shortcuts.ts +3 -0
- package/corpus/templates/design/app/hooks/useDesignHotkeys.ts +11 -7
- package/corpus/templates/design/changelog/2026-07-30-fixed-the-keyboard-shortcuts-panel-not-opening-with-ctrl-shi.md +6 -0
- package/dist/chat-threads/store.d.ts +13 -0
- package/dist/chat-threads/store.d.ts.map +1 -1
- package/dist/chat-threads/store.js +35 -0
- package/dist/chat-threads/store.js.map +1 -1
- package/dist/client/use-chat-threads.d.ts.map +1 -1
- package/dist/client/use-chat-threads.js +66 -16
- package/dist/client/use-chat-threads.js.map +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/mcp/screen-memory-stdio.d.ts +7 -7
- package/dist/mcp/screen-memory-stdio.d.ts.map +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +1 -1
- package/dist/server/agent-chat-plugin.d.ts.map +1 -1
- package/dist/server/agent-chat-plugin.js +13 -2
- package/dist/server/agent-chat-plugin.js.map +1 -1
- package/package.json +1 -1
- package/src/chat-threads/store.ts +42 -0
- package/src/client/use-chat-threads.ts +76 -16
- package/src/server/agent-chat-plugin.ts +16 -1
package/corpus/README.md
CHANGED
package/corpus/core/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
# @agent-native/core
|
|
2
2
|
|
|
3
|
+
## 0.132.2
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 3aa3c49: Keep each resource's agent chat to itself instead of showing one chat everywhere.
|
|
8
|
+
|
|
9
|
+
A chat thread's `scope` carried two meanings at once: "general chat, visible in
|
|
10
|
+
every resource" and "nobody has told the server this thread's scope yet". Because
|
|
11
|
+
those were indistinguishable, a thread that lost its scope silently became a
|
|
12
|
+
permanent global chat — it followed the user into every design/deck/form, and
|
|
13
|
+
because an unscoped chat is allowed to stay visible, no per-resource chat was ever
|
|
14
|
+
started.
|
|
15
|
+
|
|
16
|
+
Two paths dropped the scope. The server created the row on the first message
|
|
17
|
+
without one (`persistSubmittedUserMessage`), even though the client already sends
|
|
18
|
+
it and `production-agent` had already normalized it onto
|
|
19
|
+
`RequestRunContext.chatScope` — nothing read that field. The client then asserted
|
|
20
|
+
`scope: null` on every save for any thread missing from its local list, which the
|
|
21
|
+
`PUT` applies unconditionally, cementing the null.
|
|
22
|
+
|
|
23
|
+
Now the run's scope is used when the row is created, a thread with no scope adopts
|
|
24
|
+
the scope of the resource it is used in (`resolveRunThreadScope`, which never
|
|
25
|
+
retags or clears an already-scoped thread), and the client only mirrors a scope it
|
|
26
|
+
actually knows. Adoption also heals threads already stored with `scope: null`, and
|
|
27
|
+
claims the row with a compare-and-set on the unscoped state so two workers racing
|
|
28
|
+
to adopt the same legacy thread cannot retag it to the wrong resource.
|
|
29
|
+
|
|
30
|
+
Scope now rides only on thread creation: a periodic save no longer sends it, so a
|
|
31
|
+
stale client guess cannot move an existing thread between resources, and
|
|
32
|
+
`detachThread` is the only client path that clears one. A restored active-chat
|
|
33
|
+
pointer is checked against the thread's real scope on a direct mount as well as
|
|
34
|
+
when moving between resources — and because the thread list is one page, a pointer
|
|
35
|
+
naming a thread the page did not reach is resolved by id rather than assumed to be
|
|
36
|
+
a never-messaged local tab.
|
|
37
|
+
|
|
38
|
+
Genuinely general chats are unaffected until they are used inside a resource.
|
|
39
|
+
|
|
3
40
|
## 0.132.1
|
|
4
41
|
|
|
5
42
|
### Patch Changes
|
package/corpus/core/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-native/core",
|
|
3
|
-
"version": "0.132.
|
|
3
|
+
"version": "0.132.2",
|
|
4
4
|
"description": "Framework for agent-native application development — where AI agents and UI share SQL state, actions, and context",
|
|
5
5
|
"homepage": "https://github.com/BuilderIO/agent-native#readme",
|
|
6
6
|
"bugs": {
|
|
@@ -766,6 +766,48 @@ export async function searchThreads(
|
|
|
766
766
|
.filter((r): r is ChatThreadSummary => r !== null);
|
|
767
767
|
}
|
|
768
768
|
|
|
769
|
+
/**
|
|
770
|
+
* Scope a thread should carry after a run inside a resource: adopt when it has
|
|
771
|
+
* none, otherwise keep what it has. An unscoped thread reads as general, and a
|
|
772
|
+
* general chat renders inside every resource — so never retag, never clear.
|
|
773
|
+
*/
|
|
774
|
+
export function resolveRunThreadScope(
|
|
775
|
+
existing: ChatThreadScope | null,
|
|
776
|
+
incoming: ChatThreadScope | null | undefined,
|
|
777
|
+
): ChatThreadScope | null {
|
|
778
|
+
if (existing) return existing;
|
|
779
|
+
return incoming ?? null;
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* Claim an unscoped thread for `scope`, returning the scope it actually ends up
|
|
784
|
+
* with. `withThreadDataLock` only serializes one process, so two workers can
|
|
785
|
+
* both read the same unscoped row; the `scope_type IS NULL` guard makes the
|
|
786
|
+
* first writer win and the loser reports the winner instead of retagging.
|
|
787
|
+
*/
|
|
788
|
+
export async function adoptThreadScopeIfUnscoped(
|
|
789
|
+
id: string,
|
|
790
|
+
scope: ChatThreadScope,
|
|
791
|
+
): Promise<ChatThreadScope | null> {
|
|
792
|
+
await ensureTable();
|
|
793
|
+
const client = getDbExec();
|
|
794
|
+
const result = await client.execute({
|
|
795
|
+
sql: `UPDATE chat_threads SET scope_type = ?, scope_id = ?, scope_label = ?, updated_at = ? WHERE id = ? AND scope_type IS NULL`,
|
|
796
|
+
args: [
|
|
797
|
+
scope.type,
|
|
798
|
+
scope.id,
|
|
799
|
+
scope.label ?? null,
|
|
800
|
+
Math.max(Date.now(), 1),
|
|
801
|
+
id,
|
|
802
|
+
],
|
|
803
|
+
});
|
|
804
|
+
if (result.rowsAffected > 0) {
|
|
805
|
+
emitChatThreadChange(id);
|
|
806
|
+
return scope;
|
|
807
|
+
}
|
|
808
|
+
return (await getThread(id))?.scope ?? null;
|
|
809
|
+
}
|
|
810
|
+
|
|
769
811
|
/**
|
|
770
812
|
* Detach or rebind a chat's scope. Used by the UI's "Detach from <resource>"
|
|
771
813
|
* action and by templates that need to retag a chat after a rename. Pass
|
|
@@ -91,6 +91,25 @@ async function fetchThreadListPage(
|
|
|
91
91
|
});
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
/**
|
|
95
|
+
* Look up one thread the list page did not carry. Distinguishes the three states
|
|
96
|
+
* the caller must not collapse: the thread (found), `null` (the server denies it
|
|
97
|
+
* exists), and `undefined` (unreachable — nothing was learned).
|
|
98
|
+
*/
|
|
99
|
+
async function fetchThreadById(
|
|
100
|
+
apiUrl: string,
|
|
101
|
+
id: string,
|
|
102
|
+
): Promise<ChatThreadSummary | null | undefined> {
|
|
103
|
+
try {
|
|
104
|
+
const res = await fetch(`${apiUrl}/threads/${encodeURIComponent(id)}`);
|
|
105
|
+
if (res.status === 404) return null;
|
|
106
|
+
if (!res.ok) return undefined;
|
|
107
|
+
return (await res.json()) as ChatThreadSummary;
|
|
108
|
+
} catch {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
94
113
|
function emitThreadsUpdated() {
|
|
95
114
|
if (typeof window === "undefined") return;
|
|
96
115
|
window.dispatchEvent(new CustomEvent(THREADS_UPDATED_EVENT));
|
|
@@ -411,6 +430,17 @@ export function useChatThreads(
|
|
|
411
430
|
} catch {
|
|
412
431
|
nextActiveThreadId = null;
|
|
413
432
|
}
|
|
433
|
+
// Only a known mismatch disqualifies the pointer — an unresolved scope
|
|
434
|
+
// must not be read as "belongs here".
|
|
435
|
+
if (nextActiveThreadId) {
|
|
436
|
+
const savedScope = readKnownThreadScope(nextActiveThreadId);
|
|
437
|
+
if (
|
|
438
|
+
savedScope !== undefined &&
|
|
439
|
+
!threadCanStayVisibleInScope(savedScope, scopeRef.current)
|
|
440
|
+
) {
|
|
441
|
+
nextActiveThreadId = null;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
414
444
|
if (!nextActiveThreadId && autoCreate) {
|
|
415
445
|
nextActiveThreadId = createLocalThreadId();
|
|
416
446
|
newlyCreatedRef.current.add(nextActiveThreadId);
|
|
@@ -583,7 +613,7 @@ export function useChatThreads(
|
|
|
583
613
|
|
|
584
614
|
(async () => {
|
|
585
615
|
const loadedThreads = await fetchThreads();
|
|
586
|
-
const
|
|
616
|
+
const restoredId = activeThreadIdRef.current;
|
|
587
617
|
if (loadedThreads === undefined) {
|
|
588
618
|
// Thread-list fetch failed. Do not reclassify a saved id as a new
|
|
589
619
|
// optimistic tab; AssistantChat should still get a chance to restore
|
|
@@ -591,8 +621,40 @@ export function useChatThreads(
|
|
|
591
621
|
setIsLoading(false);
|
|
592
622
|
return;
|
|
593
623
|
}
|
|
624
|
+
// Exempts route-owned threads (the URL names what the user asked for) and
|
|
625
|
+
// ids this client generated, which have never reached the server.
|
|
626
|
+
const lookupRestored = Boolean(
|
|
627
|
+
restoredId &&
|
|
628
|
+
!routeControlsActiveThread &&
|
|
629
|
+
!newlyCreatedRef.current.has(restoredId),
|
|
630
|
+
);
|
|
631
|
+
const restoredOnPage = restoredId
|
|
632
|
+
? loadedThreads.find((t) => t.id === restoredId)
|
|
633
|
+
: undefined;
|
|
634
|
+
// One page, so absence from it is not absence from the server — this is what
|
|
635
|
+
// separates an older real thread from the ghost tab reclassified below.
|
|
636
|
+
const restoredThread =
|
|
637
|
+
lookupRestored && !restoredOnPage
|
|
638
|
+
? await fetchThreadById(apiUrl, restoredId!)
|
|
639
|
+
: restoredOnPage;
|
|
640
|
+
if (restoredThread === undefined && lookupRestored && !restoredOnPage) {
|
|
641
|
+
// Lookup unreachable. Reclassifying now would stamp this thread with the
|
|
642
|
+
// current scope on a guess; leave it untouched for the next mount.
|
|
643
|
+
setIsLoading(false);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
const restoredBelongsElsewhere = Boolean(
|
|
647
|
+
restoredThread &&
|
|
648
|
+
!threadCanStayVisibleInScope(
|
|
649
|
+
restoredThread.scope ?? null,
|
|
650
|
+
scopeRef.current,
|
|
651
|
+
),
|
|
652
|
+
);
|
|
653
|
+
if (restoredBelongsElsewhere) setActiveThreadId(null);
|
|
654
|
+
const savedId = restoredBelongsElsewhere ? null : restoredId;
|
|
594
655
|
const loadedHasSavedId = Boolean(
|
|
595
|
-
savedId &&
|
|
656
|
+
savedId &&
|
|
657
|
+
(restoredThread || loadedThreads.some((t) => t.id === savedId)),
|
|
596
658
|
);
|
|
597
659
|
const savedIdCameFromRoute =
|
|
598
660
|
Boolean(savedId) &&
|
|
@@ -646,6 +708,7 @@ export function useChatThreads(
|
|
|
646
708
|
setIsLoading(false);
|
|
647
709
|
})();
|
|
648
710
|
}, [
|
|
711
|
+
apiUrl,
|
|
649
712
|
fetchThreads,
|
|
650
713
|
addOptimisticThread,
|
|
651
714
|
autoCreate,
|
|
@@ -948,12 +1011,9 @@ export function useChatThreads(
|
|
|
948
1011
|
[apiUrl, clearUserRenamedThread, createThread],
|
|
949
1012
|
);
|
|
950
1013
|
|
|
951
|
-
//
|
|
952
|
-
//
|
|
953
|
-
//
|
|
954
|
-
// detachThread / fetchThreads — saveThreadData just mirrors it on
|
|
955
|
-
// every save so the server eventually catches up after
|
|
956
|
-
// persistSubmittedUserMessage creates the row sans scope.
|
|
1014
|
+
// Reads scope through refs so this callback survives every setThreads. Scope
|
|
1015
|
+
// rides only on creation: a periodic save must never move an existing thread
|
|
1016
|
+
// between resources, however stale this client's guess is.
|
|
957
1017
|
const saveThreadData = useCallback(
|
|
958
1018
|
async (
|
|
959
1019
|
id: string,
|
|
@@ -968,7 +1028,7 @@ export function useChatThreads(
|
|
|
968
1028
|
try {
|
|
969
1029
|
const { titleSource, ...threadDataPayload } = data;
|
|
970
1030
|
const localThread = threadsRef.current.find((t) => t.id === id);
|
|
971
|
-
const
|
|
1031
|
+
const knownScope = readKnownThreadScope(id) ?? null;
|
|
972
1032
|
const preserveUserTitle = userRenamedThreadIdsRef.current.has(id);
|
|
973
1033
|
const title = nextThreadTitle(
|
|
974
1034
|
localThread?.title,
|
|
@@ -977,11 +1037,7 @@ export function useChatThreads(
|
|
|
977
1037
|
titleSource,
|
|
978
1038
|
{ preserveUserTitle },
|
|
979
1039
|
);
|
|
980
|
-
const payload = {
|
|
981
|
-
...threadDataPayload,
|
|
982
|
-
title,
|
|
983
|
-
scope: localScope,
|
|
984
|
-
};
|
|
1040
|
+
const payload = { ...threadDataPayload, title };
|
|
985
1041
|
let response = await fetch(
|
|
986
1042
|
`${apiUrl}/threads/${encodeURIComponent(id)}`,
|
|
987
1043
|
{
|
|
@@ -997,7 +1053,11 @@ export function useChatThreads(
|
|
|
997
1053
|
const created = await fetch(`${apiUrl}/threads`, {
|
|
998
1054
|
method: "POST",
|
|
999
1055
|
headers: { "Content-Type": "application/json" },
|
|
1000
|
-
body: JSON.stringify({
|
|
1056
|
+
body: JSON.stringify({
|
|
1057
|
+
id,
|
|
1058
|
+
title,
|
|
1059
|
+
...(knownScope ? { scope: knownScope } : {}),
|
|
1060
|
+
}),
|
|
1001
1061
|
});
|
|
1002
1062
|
if (!created.ok) return;
|
|
1003
1063
|
response = await fetch(
|
|
@@ -1059,7 +1119,7 @@ export function useChatThreads(
|
|
|
1059
1119
|
});
|
|
1060
1120
|
} catch {}
|
|
1061
1121
|
},
|
|
1062
|
-
[apiUrl],
|
|
1122
|
+
[apiUrl, readKnownThreadScope],
|
|
1063
1123
|
);
|
|
1064
1124
|
|
|
1065
1125
|
const generateTitle = useCallback(
|
|
@@ -107,10 +107,12 @@ import type {
|
|
|
107
107
|
import { readAppStateForCurrentTab } from "../application-state/script-helpers.js";
|
|
108
108
|
import { runChatThreadDataMigrations } from "../chat-threads/migrations.js";
|
|
109
109
|
import {
|
|
110
|
+
adoptThreadScopeIfUnscoped,
|
|
110
111
|
createThread,
|
|
111
112
|
forkThread,
|
|
112
113
|
getThread,
|
|
113
114
|
registerChatThreadsShareable,
|
|
115
|
+
resolveRunThreadScope,
|
|
114
116
|
resolveThreadAccess,
|
|
115
117
|
listThreads,
|
|
116
118
|
searchThreads,
|
|
@@ -2538,11 +2540,16 @@ export function createAgentChatPlugin(
|
|
|
2538
2540
|
getRequestRunContext()?.owner ?? getRequestUserEmail();
|
|
2539
2541
|
if (!ownerEmail) return;
|
|
2540
2542
|
|
|
2543
|
+
const runScope = getRequestRunContext()?.chatScope ?? null;
|
|
2544
|
+
|
|
2541
2545
|
await withThreadDataLock(threadId, async () => {
|
|
2542
2546
|
let thread = await getThread(threadId);
|
|
2543
2547
|
if (!thread) {
|
|
2544
2548
|
try {
|
|
2545
|
-
thread = await createThread(ownerEmail, {
|
|
2549
|
+
thread = await createThread(ownerEmail, {
|
|
2550
|
+
id: threadId,
|
|
2551
|
+
scope: runScope,
|
|
2552
|
+
});
|
|
2546
2553
|
} catch {
|
|
2547
2554
|
thread = await getThread(threadId);
|
|
2548
2555
|
}
|
|
@@ -2566,6 +2573,14 @@ export function createAgentChatPlugin(
|
|
|
2566
2573
|
});
|
|
2567
2574
|
}
|
|
2568
2575
|
|
|
2576
|
+
const nextScope = resolveRunThreadScope(thread.scope, runScope);
|
|
2577
|
+
if (nextScope && nextScope !== thread.scope) {
|
|
2578
|
+
thread = {
|
|
2579
|
+
...thread,
|
|
2580
|
+
scope: await adoptThreadScopeIfUnscoped(threadId, nextScope),
|
|
2581
|
+
};
|
|
2582
|
+
}
|
|
2583
|
+
|
|
2569
2584
|
let repo: any;
|
|
2570
2585
|
try {
|
|
2571
2586
|
repo = JSON.parse(thread.threadData || "{}");
|
|
@@ -2743,6 +2743,19 @@ fn extract_interleaved_stereo(
|
|
|
2743
2743
|
Some((out, pts_seconds))
|
|
2744
2744
|
}
|
|
2745
2745
|
|
|
2746
|
+
pub(crate) fn extract_mono_audio(
|
|
2747
|
+
sample: &screencapturekit::cm::CMSampleBuffer,
|
|
2748
|
+
label: &str,
|
|
2749
|
+
) -> Option<Vec<f32>> {
|
|
2750
|
+
let (interleaved, _) = extract_interleaved_stereo(sample, label)?;
|
|
2751
|
+
Some(
|
|
2752
|
+
interleaved
|
|
2753
|
+
.chunks_exact(2)
|
|
2754
|
+
.map(|channels| (channels[0] + channels[1]) * 0.5)
|
|
2755
|
+
.collect(),
|
|
2756
|
+
)
|
|
2757
|
+
}
|
|
2758
|
+
|
|
2746
2759
|
/// Record whether decoded source PCM contains a real signal before it reaches
|
|
2747
2760
|
/// the timeline mixer. This distinguishes capture/format failures from mixer
|
|
2748
2761
|
/// or writer failures without persisting any audio content.
|
|
@@ -128,6 +128,8 @@ const NATIVE_CAPTURE_FPS: u32 = 24;
|
|
|
128
128
|
#[cfg(target_os = "macos")]
|
|
129
129
|
mod custom_capture;
|
|
130
130
|
#[cfg(target_os = "macos")]
|
|
131
|
+
pub(crate) use custom_capture::extract_mono_audio;
|
|
132
|
+
#[cfg(target_os = "macos")]
|
|
131
133
|
use custom_capture::{
|
|
132
134
|
prepare_clip_sink, start_custom_screencapturekit_backend_at, ClosedSegmentFile,
|
|
133
135
|
CustomCaptureResume, CustomScreenCaptureWriter, PreparedClipSink, SegmentFence,
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
//! renderer's `LiveTranscript` tagged `source: "system"`.
|
|
12
12
|
//!
|
|
13
13
|
//! Uses the safe `screencapturekit` Rust crate. Its `SCStreamOutputTrait`
|
|
14
|
-
//! callback hands us a `CMSampleBuffer` per audio frame;
|
|
15
|
-
//!
|
|
14
|
+
//! callback hands us a `CMSampleBuffer` per audio frame; each source is decoded
|
|
15
|
+
//! from its advertised format, normalized to 48 kHz, and mono-mixed.
|
|
16
16
|
//!
|
|
17
17
|
//! ## Tauri commands
|
|
18
18
|
//!
|
|
@@ -163,9 +163,6 @@ pub(crate) mod macos {
|
|
|
163
163
|
use std::sync::Arc;
|
|
164
164
|
use std::time::Duration;
|
|
165
165
|
|
|
166
|
-
use objc2::rc::Retained;
|
|
167
|
-
use objc2::AnyThread;
|
|
168
|
-
use objc2_avf_audio::{AVAudioFormat, AVAudioPCMBuffer};
|
|
169
166
|
use objc2_foundation::NSProcessInfo;
|
|
170
167
|
use serde::Serialize;
|
|
171
168
|
use tauri::{AppHandle, Emitter};
|
|
@@ -260,7 +257,6 @@ pub(crate) mod macos {
|
|
|
260
257
|
// ----------------------------------------------------------------------
|
|
261
258
|
struct RawAudioForwarder {
|
|
262
259
|
on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
|
|
263
|
-
speech_format: Retained<AVAudioFormat>,
|
|
264
260
|
app: AppHandle,
|
|
265
261
|
cancelled: Arc<AtomicBool>,
|
|
266
262
|
level_tick: Arc<AtomicU32>,
|
|
@@ -268,10 +264,8 @@ pub(crate) mod macos {
|
|
|
268
264
|
source: &'static str,
|
|
269
265
|
}
|
|
270
266
|
|
|
271
|
-
// SAFETY:
|
|
272
|
-
//
|
|
273
|
-
// SCK calls our handler from its own dispatch queue; we never alias
|
|
274
|
-
// these via `&` across threads.
|
|
267
|
+
// SAFETY: SCK calls this handler from its dispatch queue; all shared
|
|
268
|
+
// fields are immutable handles or atomics.
|
|
275
269
|
unsafe impl Send for RawAudioForwarder {}
|
|
276
270
|
unsafe impl Sync for RawAudioForwarder {}
|
|
277
271
|
|
|
@@ -287,30 +281,30 @@ pub(crate) mod macos {
|
|
|
287
281
|
if self.cancelled.load(Ordering::SeqCst) {
|
|
288
282
|
return;
|
|
289
283
|
}
|
|
290
|
-
let Some(
|
|
284
|
+
let Some(samples) =
|
|
285
|
+
crate::native_screen::extract_mono_audio(&sample_buffer, self.source)
|
|
291
286
|
else {
|
|
292
287
|
return;
|
|
293
288
|
};
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
}
|
|
289
|
+
if samples.is_empty() {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
(self.on_samples)(&samples);
|
|
293
|
+
let n = self.level_tick.fetch_add(1, Ordering::Relaxed);
|
|
294
|
+
if n % 3 == 0 {
|
|
295
|
+
let level = samples
|
|
296
|
+
.iter()
|
|
297
|
+
.copied()
|
|
298
|
+
.map(f32::abs)
|
|
299
|
+
.fold(0.0_f32, f32::max)
|
|
300
|
+
.min(1.0);
|
|
301
|
+
let _ = self.app.emit(
|
|
302
|
+
"voice:audio-level",
|
|
303
|
+
AudioLevelPayload {
|
|
304
|
+
level,
|
|
305
|
+
source: self.source,
|
|
306
|
+
},
|
|
307
|
+
);
|
|
314
308
|
}
|
|
315
309
|
}
|
|
316
310
|
}
|
|
@@ -448,19 +442,11 @@ pub(crate) mod macos {
|
|
|
448
442
|
);
|
|
449
443
|
}
|
|
450
444
|
|
|
451
|
-
// Mono float32 @ 48 kHz destination format for the mono-mix.
|
|
452
|
-
let speech_format = unsafe {
|
|
453
|
-
let allocated = AVAudioFormat::alloc();
|
|
454
|
-
AVAudioFormat::initStandardFormatWithSampleRate_channels(allocated, 48000.0, 1)
|
|
455
|
-
}
|
|
456
|
-
.ok_or_else(|| "AVAudioFormat init failed for raw system capture".to_string())?;
|
|
457
|
-
|
|
458
445
|
let cancelled = Arc::new(AtomicBool::new(false));
|
|
459
446
|
let mut stream = SCStream::new(&filter, &config);
|
|
460
447
|
if let Some(on_samples) = on_system_samples {
|
|
461
448
|
let forwarder = RawAudioForwarder {
|
|
462
449
|
on_samples,
|
|
463
|
-
speech_format: speech_format.clone(),
|
|
464
450
|
app: app.clone(),
|
|
465
451
|
cancelled: cancelled.clone(),
|
|
466
452
|
level_tick: Arc::new(AtomicU32::new(0)),
|
|
@@ -472,7 +458,6 @@ pub(crate) mod macos {
|
|
|
472
458
|
if let Some(on_samples) = on_mic_samples {
|
|
473
459
|
let forwarder = RawAudioForwarder {
|
|
474
460
|
on_samples,
|
|
475
|
-
speech_format,
|
|
476
461
|
app: app.clone(),
|
|
477
462
|
cancelled: cancelled.clone(),
|
|
478
463
|
level_tick: Arc::new(AtomicU32::new(0)),
|
|
@@ -495,96 +480,4 @@ pub(crate) mod macos {
|
|
|
495
480
|
level: f32,
|
|
496
481
|
source: &'static str,
|
|
497
482
|
}
|
|
498
|
-
|
|
499
|
-
/// Pull the PCM bytes out of a SCK CMSampleBuffer and copy them into a
|
|
500
|
-
/// freshly-allocated AVAudioPCMBuffer matching `speech_format`
|
|
501
|
-
/// (single-channel float32 at the SCK sample rate). SCK delivers stereo
|
|
502
|
-
/// non-interleaved float32 by default — we mono-mix by averaging the
|
|
503
|
-
/// two channels. Returns `None` if the sample buffer's audio layout
|
|
504
|
-
/// can't be interpreted (rare; only happens if SCK changes its output
|
|
505
|
-
/// shape mid-stream).
|
|
506
|
-
fn build_pcm_buffer_from_sample(
|
|
507
|
-
sample: &CMSampleBuffer,
|
|
508
|
-
speech_format: &AVAudioFormat,
|
|
509
|
-
) -> Option<Retained<AVAudioPCMBuffer>> {
|
|
510
|
-
let num_samples = sample.num_samples();
|
|
511
|
-
if num_samples == 0 {
|
|
512
|
-
return None;
|
|
513
|
-
}
|
|
514
|
-
let abl = sample.audio_buffer_list()?;
|
|
515
|
-
let n_buffers = abl.num_buffers();
|
|
516
|
-
if n_buffers == 0 {
|
|
517
|
-
return None;
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
// Allocate the destination buffer.
|
|
521
|
-
// SAFETY: standard AVAudioPCMBuffer init; we control the format and
|
|
522
|
-
// capacity.
|
|
523
|
-
#[allow(clippy::cast_possible_truncation)]
|
|
524
|
-
let frame_capacity = num_samples as u32;
|
|
525
|
-
let allocated = AVAudioPCMBuffer::alloc();
|
|
526
|
-
let dest = unsafe {
|
|
527
|
-
AVAudioPCMBuffer::initWithPCMFormat_frameCapacity(
|
|
528
|
-
allocated,
|
|
529
|
-
speech_format,
|
|
530
|
-
frame_capacity,
|
|
531
|
-
)
|
|
532
|
-
}?;
|
|
533
|
-
unsafe { dest.setFrameLength(frame_capacity) };
|
|
534
|
-
|
|
535
|
-
// SAFETY: the format is the one we constructed below — float, mono,
|
|
536
|
-
// non-interleaved — so `floatChannelData` is non-null and points at
|
|
537
|
-
// `channelCount=1` pointers, each to `frame_capacity` floats.
|
|
538
|
-
let dest_ch_ptr = unsafe { dest.floatChannelData() };
|
|
539
|
-
if dest_ch_ptr.is_null() {
|
|
540
|
-
return None;
|
|
541
|
-
}
|
|
542
|
-
let dest_slice =
|
|
543
|
-
unsafe { std::slice::from_raw_parts_mut((*dest_ch_ptr).as_ptr(), num_samples) };
|
|
544
|
-
|
|
545
|
-
if n_buffers >= 2 {
|
|
546
|
-
// Stereo non-interleaved — average the two channels.
|
|
547
|
-
let l = abl.get(0)?;
|
|
548
|
-
let r = abl.get(1)?;
|
|
549
|
-
let l_bytes = l.data();
|
|
550
|
-
let r_bytes = r.data();
|
|
551
|
-
// Treat as f32 little-endian (host byte order on every Apple
|
|
552
|
-
// platform we ship).
|
|
553
|
-
let l_floats = bytes_as_f32(l_bytes);
|
|
554
|
-
let r_floats = bytes_as_f32(r_bytes);
|
|
555
|
-
let n = num_samples.min(l_floats.len()).min(r_floats.len());
|
|
556
|
-
for i in 0..n {
|
|
557
|
-
dest_slice[i] = 0.5 * (l_floats[i] + r_floats[i]);
|
|
558
|
-
}
|
|
559
|
-
for v in dest_slice.iter_mut().take(num_samples).skip(n) {
|
|
560
|
-
*v = 0.0;
|
|
561
|
-
}
|
|
562
|
-
} else {
|
|
563
|
-
// Mono — just copy.
|
|
564
|
-
let only = abl.get(0)?;
|
|
565
|
-
let src = bytes_as_f32(only.data());
|
|
566
|
-
let n = num_samples.min(src.len());
|
|
567
|
-
dest_slice[..n].copy_from_slice(&src[..n]);
|
|
568
|
-
for v in dest_slice.iter_mut().take(num_samples).skip(n) {
|
|
569
|
-
*v = 0.0;
|
|
570
|
-
}
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
Some(dest)
|
|
574
|
-
}
|
|
575
|
-
|
|
576
|
-
/// Reinterpret a `&[u8]` as `&[f32]`. Length is rounded down to the
|
|
577
|
-
/// nearest multiple of 4. Safe because `f32` has no invalid
|
|
578
|
-
/// bit-patterns and the caller only uses the elements they're
|
|
579
|
-
/// indexing into.
|
|
580
|
-
fn bytes_as_f32(b: &[u8]) -> &[f32] {
|
|
581
|
-
let n = b.len() / 4;
|
|
582
|
-
if n == 0 {
|
|
583
|
-
return &[];
|
|
584
|
-
}
|
|
585
|
-
// SAFETY: `f32` is plain old data with alignment 4; CoreAudio's
|
|
586
|
-
// AudioBuffer pointers are 16-byte aligned in practice. We cap the
|
|
587
|
-
// length at `n` so we never read past the end.
|
|
588
|
-
unsafe { std::slice::from_raw_parts(b.as_ptr().cast::<f32>(), n) }
|
|
589
|
-
}
|
|
590
483
|
}
|
|
@@ -3808,7 +3808,12 @@ export const editorChromeBridgeScript: string = `"use strict";
|
|
|
3808
3808
|
'input, textarea, select, [contenteditable], [role="textbox"], [data-agent-native-text-editing]'
|
|
3809
3809
|
);
|
|
3810
3810
|
}
|
|
3811
|
+
function isShowShortcutsChord(e) {
|
|
3812
|
+
if (!(e.metaKey || e.ctrlKey) || !e.shiftKey || e.altKey) return false;
|
|
3813
|
+
return e.key === "?" || e.key === "/";
|
|
3814
|
+
}
|
|
3811
3815
|
function shouldForwardDesignHotkey(e) {
|
|
3816
|
+
if (isShowShortcutsChord(e)) return true;
|
|
3812
3817
|
if (readOnly) return false;
|
|
3813
3818
|
if (activeTextEditEl || isEditorTypingTarget(e.target) || e.isComposing)
|
|
3814
3819
|
return false;
|
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
} from "@/components/design/keyboard-shortcuts";
|
|
26
26
|
import { Button } from "@/components/ui/button";
|
|
27
27
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
|
28
|
+
import { isApplePlatform } from "@/hooks/useDesignHotkeys";
|
|
28
29
|
|
|
29
30
|
interface KeyboardShortcutsPanelProps {
|
|
30
31
|
onClose: () => void;
|
|
@@ -113,9 +114,10 @@ function KeycapGroup({
|
|
|
113
114
|
|
|
114
115
|
function ShortcutBindings({ bindings }: { bindings: readonly string[] }) {
|
|
115
116
|
const t = useT();
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
// Shared with the hotkey matcher: navigator.platform alone is deprecated and
|
|
118
|
+
// blank in some browsers, which labelled Ctrl on Macs that the hotkey layer
|
|
119
|
+
// already treated as Apple.
|
|
120
|
+
const applePlatform = isApplePlatform();
|
|
119
121
|
const accessibleBindings = bindings.map((binding) =>
|
|
120
122
|
binding
|
|
121
123
|
.split("+")
|
|
@@ -5101,7 +5101,23 @@ declare var __SELECTED_LAYER_DRAG_PRIORITY__: boolean;
|
|
|
5101
5101
|
);
|
|
5102
5102
|
}
|
|
5103
5103
|
|
|
5104
|
+
function isShowShortcutsChord(e) {
|
|
5105
|
+
if (!(e.metaKey || e.ctrlKey) || !e.shiftKey || e.altKey) return false;
|
|
5106
|
+
// macOS delivers Control+Shift+/ as "/" — Control suppresses the shifted
|
|
5107
|
+
// character — while Windows sends "?". Match both; see
|
|
5108
|
+
// isShowKeyboardShortcutsHotkey in useDesignHotkeys.ts.
|
|
5109
|
+
return e.key === "?" || e.key === "/";
|
|
5110
|
+
}
|
|
5111
|
+
|
|
5104
5112
|
function shouldForwardDesignHotkey(e) {
|
|
5113
|
+
// Shortcut help is not an editing affordance, so it forwards ahead of the
|
|
5114
|
+
// read-only and typing guards below — the host matcher is deliberately
|
|
5115
|
+
// global for the same reason. Without this the chord never escapes the
|
|
5116
|
+
// canvas iframe, which is where focus lands the moment you click a frame.
|
|
5117
|
+
// Not reached during a live text-edit session: that block returns before
|
|
5118
|
+
// this function runs, deliberately — see the activeTextEditEl guard in
|
|
5119
|
+
// the keydown listener.
|
|
5120
|
+
if (isShowShortcutsChord(e)) return true;
|
|
5105
5121
|
// Read-only surfaces (e.g. background/inactive board screens) must never
|
|
5106
5122
|
// forward edit hotkeys or preventDefault() native browser shortcuts —
|
|
5107
5123
|
// Escape/Enter/Tab/Delete/arrow-key/undo-redo forwarding is an editing
|
|
@@ -11092,6 +11108,13 @@ declare var __SELECTED_LAYER_DRAG_PRIORITY__: boolean;
|
|
|
11092
11108
|
}
|
|
11093
11109
|
// While a live session exists, never forward hotkeys to the host
|
|
11094
11110
|
// (matches shouldForwardDesignHotkey's activeTextEditEl guard).
|
|
11111
|
+
// The shortcut-help chord is deliberately included in that exclusion:
|
|
11112
|
+
// the panel lives in the parent document, so opening it moves focus
|
|
11113
|
+
// out of the iframe, blurs the editable and commits the in-progress
|
|
11114
|
+
// edit. Ending someone's text entry to show help is a worse trade
|
|
11115
|
+
// than help being unavailable for the duration of a typing session;
|
|
11116
|
+
// it stays available everywhere else, including while a text layer is
|
|
11117
|
+
// merely selected.
|
|
11095
11118
|
return;
|
|
11096
11119
|
}
|
|
11097
11120
|
if (!shouldForwardDesignHotkey(e)) return;
|
|
@@ -49,6 +49,9 @@ export const DESIGN_SHORTCUTS: readonly DesignShortcutDefinition[] = [
|
|
|
49
49
|
shortcut({
|
|
50
50
|
id: "show-shortcuts",
|
|
51
51
|
category: "essential",
|
|
52
|
+
// Literal ctrl, not $mod: on macOS ⌘⇧? is the system Help-menu shortcut and
|
|
53
|
+
// the browser consumes it before the page sees it, so ⌃⇧? is the only
|
|
54
|
+
// pressable binding there. Do not "fix" this to $mod.
|
|
52
55
|
bindings: ["ctrl+shift+?"],
|
|
53
56
|
labelKey: "designEditor.keyboardShortcuts.commands.showShortcuts",
|
|
54
57
|
handler: "onShowKeyboardShortcuts",
|