@agent-native/core 0.90.5 → 0.90.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -7
- package/corpus/README.md +1 -1
- package/corpus/core/CHANGELOG.md +13 -0
- package/corpus/core/docs/content/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/ar-SA/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/de-DE/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/es-ES/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/fr-FR/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/hi-IN/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/ja-JP/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/ko-KR/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/pt-BR/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/zh-CN/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/locales/zh-TW/agent-native-toolkit.mdx +1 -1
- package/corpus/core/docs/content/toolkit-agent-ux.mdx +1 -1
- package/corpus/core/docs/content/toolkit-app-adapters.mdx +1 -1
- package/corpus/core/docs/content/toolkit-collaboration-ui.mdx +1 -1
- package/corpus/core/docs/content/toolkit-collaboration.mdx +1 -1
- package/corpus/core/docs/content/toolkit-command-navigation.mdx +2 -2
- package/corpus/core/docs/content/toolkit-comments-review.mdx +1 -1
- package/corpus/core/docs/content/toolkit-history.mdx +1 -1
- package/corpus/core/docs/content/toolkit-observability.mdx +1 -1
- package/corpus/core/docs/content/toolkit-org-team.mdx +1 -1
- package/corpus/core/docs/content/toolkit-resources.mdx +1 -1
- package/corpus/core/docs/content/toolkit-settings.mdx +1 -1
- package/corpus/core/docs/content/toolkit-setup-connections.mdx +1 -1
- package/corpus/core/docs/content/toolkit-sharing-ui.mdx +1 -1
- package/corpus/core/docs/content/toolkit-sharing.mdx +1 -1
- package/corpus/core/docs/content/toolkit-shell-hooks.mdx +1 -1
- package/corpus/core/docs/content/toolkit-ui.mdx +1 -1
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/client/use-db-sync.ts +99 -1
- package/corpus/core/src/collab/client.ts +49 -4
- package/corpus/core/src/deploy/build.ts +103 -0
- package/corpus/core/src/server/core-routes-plugin.ts +137 -8
- package/corpus/templates/clips/changelog/2026-07-07-faster-local-whisper-dictation-start.md +6 -0
- package/corpus/templates/clips/desktop/src-tauri/src/native_speech.rs +311 -31
- package/corpus/templates/clips/desktop/src-tauri/src/whisper_speech.rs +2 -0
- package/corpus/templates/design/.agents/skills/design-generation/SKILL.md +9 -0
- package/corpus/templates/design/actions/read-source-file.ts +28 -0
- package/corpus/templates/design/actions/update-file.ts +76 -7
- package/corpus/templates/design/app/components/design/DesignCanvas.tsx +181 -1
- package/corpus/templates/design/app/components/design/EditPanel.tsx +4 -0
- package/corpus/templates/design/app/components/design/MultiScreenCanvas.tsx +387 -25
- package/corpus/templates/design/app/components/design/bridge/editor-chrome.bridge.ts +1110 -88
- package/corpus/templates/design/app/components/design/bridge/hit-test.bridge.ts +368 -16
- package/corpus/templates/design/app/components/design/edit-panel/fill-gradient-helpers.ts +24 -0
- package/corpus/templates/design/app/components/design/edit-panel/panel-primitives.tsx +27 -6
- package/corpus/templates/design/app/components/design/edit-panel/position-helpers.ts +18 -0
- package/corpus/templates/design/app/components/design/edit-panel/stroke-properties.tsx +5 -4
- package/corpus/templates/design/app/components/design/multi-screen/handle-hit-zones.ts +124 -0
- package/corpus/templates/design/app/components/design/multi-screen/types.ts +23 -0
- package/corpus/templates/design/app/i18n/zh-TW.ts +1 -0
- package/corpus/templates/design/app/i18n-data.ts +19 -0
- package/corpus/templates/design/changelog/2026-07-07-dragging-a-free-floating-element-into-a-layout-column-or-row.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-dragging-a-repeated-list-item-like-a-to-do-card-generated-fr.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-dragging-an-element-onto-a-button-or-chip-s-middle-no-longer.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-elements-dropped-into-another-board-shape-now-stay-exactly-w.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-drag-reordering-elements-within-a-screen-no-longer-sil.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-dragging-an-element-from-a-screen-onto-the-canvas-now-.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-dragging-elements-between-screens-into-ai-generated-co.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-moving-elements-between-screens-no-longer-bakes-intern.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-the-cross-screen-drop-indicator-now-points-at-the-corr.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-fixed-the-zoom-menu-no-longer-leaves-the-canvas-unresponsive.md +6 -0
- package/corpus/templates/design/changelog/2026-07-07-text-drawn-on-the-canvas-is-now-readable-white-instead-of-in.md +6 -0
- package/corpus/templates/design/server/source-workspace.ts +27 -3
- package/corpus/templates/design/shared/board-file.ts +359 -0
- package/corpus/templates/design/shared/breakpoint-media.ts +21 -3
- package/corpus/templates/design/shared/code-layer.ts +145 -10
- package/dist/client/use-db-sync.d.ts +18 -0
- package/dist/client/use-db-sync.d.ts.map +1 -1
- package/dist/client/use-db-sync.js +88 -1
- package/dist/client/use-db-sync.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/collab/client.d.ts.map +1 -1
- package/dist/collab/client.js +43 -4
- package/dist/collab/client.js.map +1 -1
- package/dist/deploy/build.d.ts +1 -0
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +57 -0
- package/dist/deploy/build.js.map +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/server/core-routes-plugin.d.ts +10 -0
- package/dist/server/core-routes-plugin.d.ts.map +1 -1
- package/dist/server/core-routes-plugin.js +103 -9
- package/dist/server/core-routes-plugin.js.map +1 -1
- package/docs/content/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/ar-SA/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/de-DE/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/es-ES/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/fr-FR/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/hi-IN/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/ja-JP/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/ko-KR/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/pt-BR/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/zh-CN/agent-native-toolkit.mdx +1 -1
- package/docs/content/locales/zh-TW/agent-native-toolkit.mdx +1 -1
- package/docs/content/toolkit-agent-ux.mdx +1 -1
- package/docs/content/toolkit-app-adapters.mdx +1 -1
- package/docs/content/toolkit-collaboration-ui.mdx +1 -1
- package/docs/content/toolkit-collaboration.mdx +1 -1
- package/docs/content/toolkit-command-navigation.mdx +2 -2
- package/docs/content/toolkit-comments-review.mdx +1 -1
- package/docs/content/toolkit-history.mdx +1 -1
- package/docs/content/toolkit-observability.mdx +1 -1
- package/docs/content/toolkit-org-team.mdx +1 -1
- package/docs/content/toolkit-resources.mdx +1 -1
- package/docs/content/toolkit-settings.mdx +1 -1
- package/docs/content/toolkit-setup-connections.mdx +1 -1
- package/docs/content/toolkit-sharing-ui.mdx +1 -1
- package/docs/content/toolkit-sharing.mdx +1 -1
- package/docs/content/toolkit-shell-hooks.mdx +1 -1
- package/docs/content/toolkit-ui.mdx +1 -1
- package/package.json +1 -1
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
setCookie,
|
|
9
9
|
deleteCookie,
|
|
10
10
|
getRequestURL,
|
|
11
|
+
getRequestIP,
|
|
11
12
|
} from "h3";
|
|
12
13
|
import type { H3Event } from "h3";
|
|
13
14
|
import { readMultipartFormData } from "h3";
|
|
@@ -285,9 +286,16 @@ const BUILDER_WAITLIST_DEFAULT_USE_CASE = "builder_agent_background_coding";
|
|
|
285
286
|
const BUILDER_WAITLIST_USE_CASES = new Set([
|
|
286
287
|
BUILDER_WAITLIST_DEFAULT_USE_CASE,
|
|
287
288
|
"design_publish_app",
|
|
289
|
+
"docs_build_online_waitlist",
|
|
288
290
|
]);
|
|
289
291
|
const BUILDER_WAITLIST_FORM_TIMEOUT_MS = 8000;
|
|
290
292
|
const BUILDER_WAITLIST_TEXT_LIMIT = 4000;
|
|
293
|
+
const BUILDER_WAITLIST_RATE_LIMIT_WINDOW_MS = 60_000;
|
|
294
|
+
const BUILDER_WAITLIST_RATE_LIMIT_MAX = 5;
|
|
295
|
+
const builderWaitlistRateLimitHits = new Map<
|
|
296
|
+
string,
|
|
297
|
+
{ count: number; resetAt: number }
|
|
298
|
+
>();
|
|
291
299
|
|
|
292
300
|
interface BuilderWaitlistFormTarget {
|
|
293
301
|
formId: string;
|
|
@@ -295,6 +303,7 @@ interface BuilderWaitlistFormTarget {
|
|
|
295
303
|
}
|
|
296
304
|
|
|
297
305
|
export interface BuilderWaitlistBody {
|
|
306
|
+
email?: unknown;
|
|
298
307
|
prompt?: unknown;
|
|
299
308
|
orgName?: unknown;
|
|
300
309
|
appUrl?: unknown;
|
|
@@ -332,6 +341,106 @@ function normalizeBuilderWaitlistUseCase(value: unknown): string {
|
|
|
332
341
|
: BUILDER_WAITLIST_DEFAULT_USE_CASE;
|
|
333
342
|
}
|
|
334
343
|
|
|
344
|
+
function isValidWaitlistEmail(email: string): boolean {
|
|
345
|
+
return /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email);
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
export function isAnonymousWaitlistSessionEmail(email: string): boolean {
|
|
349
|
+
return email.startsWith("anon-") && email.endsWith("@agent-native.com");
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
export function resolveWaitlistEmail(
|
|
353
|
+
sessionEmail: string | undefined,
|
|
354
|
+
bodyEmail: unknown,
|
|
355
|
+
): string | null {
|
|
356
|
+
const provided = cleanBuilderWaitlistText(bodyEmail, 320);
|
|
357
|
+
if (provided && isValidWaitlistEmail(provided)) return provided;
|
|
358
|
+
if (sessionEmail && !isAnonymousWaitlistSessionEmail(sessionEmail)) {
|
|
359
|
+
return sessionEmail;
|
|
360
|
+
}
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
function normalizeWaitlistRateLimitPart(value: string): string {
|
|
365
|
+
return value.trim().toLowerCase();
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function getBuilderWaitlistClientIp(event: H3Event): string | undefined {
|
|
369
|
+
const trusted =
|
|
370
|
+
getHeader(event, "x-nf-client-connection-ip") ??
|
|
371
|
+
getHeader(event, "cf-connecting-ip") ??
|
|
372
|
+
getHeader(event, "true-client-ip") ??
|
|
373
|
+
getHeader(event, "x-real-ip");
|
|
374
|
+
if (trusted && trusted.trim()) return trusted.trim();
|
|
375
|
+
|
|
376
|
+
const forwardedFor = getHeader(event, "x-forwarded-for");
|
|
377
|
+
const forwardedClientIp = forwardedFor?.split(",")[0]?.trim();
|
|
378
|
+
if (forwardedClientIp) return forwardedClientIp;
|
|
379
|
+
|
|
380
|
+
try {
|
|
381
|
+
return getRequestIP(event) ?? undefined;
|
|
382
|
+
} catch {
|
|
383
|
+
return undefined;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function getBuilderWaitlistRateLimitKeys(
|
|
388
|
+
event: H3Event,
|
|
389
|
+
email: string,
|
|
390
|
+
): string[] {
|
|
391
|
+
const clientIp = getBuilderWaitlistClientIp(event);
|
|
392
|
+
return [
|
|
393
|
+
`email:${normalizeWaitlistRateLimitPart(email)}`,
|
|
394
|
+
`ip:${normalizeWaitlistRateLimitPart(clientIp ?? "unknown")}`,
|
|
395
|
+
];
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export function checkBuilderWaitlistRateLimit(
|
|
399
|
+
event: H3Event,
|
|
400
|
+
email: string,
|
|
401
|
+
now = Date.now(),
|
|
402
|
+
): { ok: true } | { ok: false; retryAfterSeconds: number } {
|
|
403
|
+
const keys = getBuilderWaitlistRateLimitKeys(event, email);
|
|
404
|
+
let retryAfterMs = 0;
|
|
405
|
+
|
|
406
|
+
for (const key of keys) {
|
|
407
|
+
const entry = builderWaitlistRateLimitHits.get(key);
|
|
408
|
+
if (!entry) continue;
|
|
409
|
+
if (entry.resetAt <= now) {
|
|
410
|
+
builderWaitlistRateLimitHits.delete(key);
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
if (entry.count >= BUILDER_WAITLIST_RATE_LIMIT_MAX) {
|
|
414
|
+
retryAfterMs = Math.max(retryAfterMs, entry.resetAt - now);
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
if (retryAfterMs > 0) {
|
|
419
|
+
return {
|
|
420
|
+
ok: false,
|
|
421
|
+
retryAfterSeconds: Math.max(1, Math.ceil(retryAfterMs / 1000)),
|
|
422
|
+
};
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
for (const key of keys) {
|
|
426
|
+
const entry = builderWaitlistRateLimitHits.get(key);
|
|
427
|
+
if (!entry || entry.resetAt <= now) {
|
|
428
|
+
builderWaitlistRateLimitHits.set(key, {
|
|
429
|
+
count: 1,
|
|
430
|
+
resetAt: now + BUILDER_WAITLIST_RATE_LIMIT_WINDOW_MS,
|
|
431
|
+
});
|
|
432
|
+
} else {
|
|
433
|
+
entry.count += 1;
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
return { ok: true };
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
export function resetBuilderWaitlistRateLimitForTests() {
|
|
441
|
+
builderWaitlistRateLimitHits.clear();
|
|
442
|
+
}
|
|
443
|
+
|
|
335
444
|
function normalizeHttpOrigin(value: string): string | null {
|
|
336
445
|
try {
|
|
337
446
|
const url = new URL(value);
|
|
@@ -1869,15 +1978,35 @@ export function createCoreRoutesPlugin(
|
|
|
1869
1978
|
return { error: "Method not allowed" };
|
|
1870
1979
|
}
|
|
1871
1980
|
const session = await getSession(event).catch(() => null);
|
|
1872
|
-
if (!session?.email) {
|
|
1873
|
-
setResponseStatus(event, 401);
|
|
1874
|
-
return { error: "Authentication required" };
|
|
1875
|
-
}
|
|
1876
1981
|
const body = ((await readBody(event).catch(() => ({}))) ??
|
|
1877
1982
|
{}) as BuilderWaitlistBody;
|
|
1983
|
+
const waitlistEmail = resolveWaitlistEmail(
|
|
1984
|
+
session?.email,
|
|
1985
|
+
body.email,
|
|
1986
|
+
);
|
|
1987
|
+
if (!waitlistEmail) {
|
|
1988
|
+
setResponseStatus(event, 400);
|
|
1989
|
+
return { error: "Valid email required" };
|
|
1990
|
+
}
|
|
1991
|
+
const waitlistRateLimit = checkBuilderWaitlistRateLimit(
|
|
1992
|
+
event,
|
|
1993
|
+
waitlistEmail,
|
|
1994
|
+
);
|
|
1995
|
+
if (!waitlistRateLimit.ok) {
|
|
1996
|
+
setResponseStatus(event, 429);
|
|
1997
|
+
setResponseHeader(
|
|
1998
|
+
event,
|
|
1999
|
+
"Retry-After",
|
|
2000
|
+
String(waitlistRateLimit.retryAfterSeconds),
|
|
2001
|
+
);
|
|
2002
|
+
return {
|
|
2003
|
+
error:
|
|
2004
|
+
"Too many waitlist requests. Please try again in a minute.",
|
|
2005
|
+
};
|
|
2006
|
+
}
|
|
1878
2007
|
const waitlistPayload = buildBuilderWaitlistFormPayload(
|
|
1879
2008
|
event,
|
|
1880
|
-
|
|
2009
|
+
waitlistEmail,
|
|
1881
2010
|
body,
|
|
1882
2011
|
);
|
|
1883
2012
|
const waitlistSource = waitlistPayload.data.source;
|
|
@@ -1886,14 +2015,14 @@ export function createCoreRoutesPlugin(
|
|
|
1886
2015
|
try {
|
|
1887
2016
|
formSubmission = await submitBuilderWaitlistForm(
|
|
1888
2017
|
event,
|
|
1889
|
-
|
|
2018
|
+
waitlistEmail,
|
|
1890
2019
|
body,
|
|
1891
2020
|
);
|
|
1892
2021
|
} catch (err) {
|
|
1893
2022
|
await trackBuilderLifecycle(
|
|
1894
2023
|
event,
|
|
1895
2024
|
"builder branch waitlist form failed",
|
|
1896
|
-
|
|
2025
|
+
waitlistEmail,
|
|
1897
2026
|
{
|
|
1898
2027
|
reason:
|
|
1899
2028
|
err instanceof Error ? err.message : "unknown_waitlist_error",
|
|
@@ -1911,7 +2040,7 @@ export function createCoreRoutesPlugin(
|
|
|
1911
2040
|
await trackBuilderLifecycle(
|
|
1912
2041
|
event,
|
|
1913
2042
|
"builder branch waitlist joined",
|
|
1914
|
-
|
|
2043
|
+
waitlistEmail,
|
|
1915
2044
|
{
|
|
1916
2045
|
formId: formSubmission.formId ?? null,
|
|
1917
2046
|
formSubmitted: formSubmission.submitted,
|
|
@@ -124,7 +124,7 @@ pub(crate) mod macos {
|
|
|
124
124
|
use std::ffi::c_void;
|
|
125
125
|
use std::mem::size_of;
|
|
126
126
|
use std::ptr::NonNull;
|
|
127
|
-
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
|
127
|
+
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering};
|
|
128
128
|
use std::sync::{Arc, Mutex, OnceLock};
|
|
129
129
|
|
|
130
130
|
use block2::{RcBlock, StackBlock};
|
|
@@ -134,7 +134,7 @@ pub(crate) mod macos {
|
|
|
134
134
|
kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, AudioUnitSetProperty,
|
|
135
135
|
};
|
|
136
136
|
use objc2_avf_audio::{
|
|
137
|
-
AVAudioEngine, AVAudioPCMBuffer, AVAudioTime,
|
|
137
|
+
AVAudioEngine, AVAudioInputNode, AVAudioPCMBuffer, AVAudioTime,
|
|
138
138
|
AVAudioVoiceProcessingOtherAudioDuckingConfiguration,
|
|
139
139
|
AVAudioVoiceProcessingOtherAudioDuckingLevel,
|
|
140
140
|
};
|
|
@@ -211,6 +211,51 @@ pub(crate) mod macos {
|
|
|
211
211
|
// inner pointers across threads — the session is moved through a Mutex.
|
|
212
212
|
unsafe impl Send for SpeechSession {}
|
|
213
213
|
|
|
214
|
+
#[derive(Clone, PartialEq, Eq)]
|
|
215
|
+
struct RawMicDeviceKey {
|
|
216
|
+
id: Option<String>,
|
|
217
|
+
label: Option<String>,
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
impl RawMicDeviceKey {
|
|
221
|
+
fn from_selection(device_id: Option<&str>, device_label: Option<&str>) -> Self {
|
|
222
|
+
Self {
|
|
223
|
+
id: device_id
|
|
224
|
+
.map(str::trim)
|
|
225
|
+
.filter(|value| !value.is_empty())
|
|
226
|
+
.map(ToOwned::to_owned),
|
|
227
|
+
label: device_label
|
|
228
|
+
.map(str::trim)
|
|
229
|
+
.filter(|value| !value.is_empty())
|
|
230
|
+
.map(ToOwned::to_owned),
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
#[derive(Clone)]
|
|
236
|
+
struct RawMicTapTarget {
|
|
237
|
+
app: AppHandle,
|
|
238
|
+
on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
|
|
239
|
+
level_tick: Arc<AtomicU32>,
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
struct WarmedRawMicEngine {
|
|
243
|
+
engine: Retained<AVAudioEngine>,
|
|
244
|
+
input_node: Retained<AVAudioInputNode>,
|
|
245
|
+
device_key: RawMicDeviceKey,
|
|
246
|
+
target: Arc<Mutex<Option<RawMicTapTarget>>>,
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// SAFETY: Same argument as `SpeechSession` / `RawMicCapture`: AVAudioEngine
|
|
250
|
+
// and AVAudioInputNode are refcounted ObjC objects that we move through
|
|
251
|
+
// Mutex ownership, without sharing Rust references across threads.
|
|
252
|
+
unsafe impl Send for WarmedRawMicEngine {}
|
|
253
|
+
|
|
254
|
+
struct RawMicWarmCache {
|
|
255
|
+
device_key: RawMicDeviceKey,
|
|
256
|
+
target: Arc<Mutex<Option<RawMicTapTarget>>>,
|
|
257
|
+
}
|
|
258
|
+
|
|
214
259
|
/// Process-global session slot. We only allow one dictation at a time —
|
|
215
260
|
/// starting a new one while another is in flight cancels the old one
|
|
216
261
|
/// first.
|
|
@@ -219,6 +264,11 @@ pub(crate) mod macos {
|
|
|
219
264
|
SLOT.get_or_init(|| Mutex::new(None))
|
|
220
265
|
}
|
|
221
266
|
|
|
267
|
+
fn warmed_raw_mic_engine_slot() -> &'static Mutex<Option<WarmedRawMicEngine>> {
|
|
268
|
+
static SLOT: OnceLock<Mutex<Option<WarmedRawMicEngine>>> = OnceLock::new();
|
|
269
|
+
SLOT.get_or_init(|| Mutex::new(None))
|
|
270
|
+
}
|
|
271
|
+
|
|
222
272
|
/// Bumped at the start of every `native_speech_start_impl` call. The
|
|
223
273
|
/// auto-restart thread captures the value at decision time and skips the
|
|
224
274
|
/// restart if a newer session has since been requested.
|
|
@@ -574,6 +624,160 @@ pub(crate) mod macos {
|
|
|
574
624
|
Ok(())
|
|
575
625
|
}
|
|
576
626
|
|
|
627
|
+
fn enable_voice_processing(input_node: &AVAudioInputNode) {
|
|
628
|
+
if let Err(err) = unsafe { input_node.setVoiceProcessingEnabled_error(true) } {
|
|
629
|
+
eprintln!(
|
|
630
|
+
"[whisper-mic] voice processing enable failed: {} — continuing without AEC",
|
|
631
|
+
ns_error_message(&err)
|
|
632
|
+
);
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
fn disable_voice_processing_ducking(input_node: &AVAudioInputNode) {
|
|
637
|
+
// Disable other-audio ducking so the separately-captured system audio
|
|
638
|
+
// isn't dropped to near-silent while the mic VPIO runs.
|
|
639
|
+
unsafe {
|
|
640
|
+
let responds: bool = objc2::msg_send![
|
|
641
|
+
input_node,
|
|
642
|
+
respondsToSelector: objc2::sel!(setVoiceProcessingOtherAudioDuckingConfiguration:)
|
|
643
|
+
];
|
|
644
|
+
if responds {
|
|
645
|
+
input_node.setVoiceProcessingOtherAudioDuckingConfiguration(
|
|
646
|
+
AVAudioVoiceProcessingOtherAudioDuckingConfiguration {
|
|
647
|
+
enableAdvancedDucking: objc2::runtime::Bool::NO,
|
|
648
|
+
duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingLevel::Min,
|
|
649
|
+
},
|
|
650
|
+
);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
fn take_warmed_raw_mic_engine(device_key: &RawMicDeviceKey) -> Option<WarmedRawMicEngine> {
|
|
656
|
+
let mut slot = warmed_raw_mic_engine_slot().lock().ok()?;
|
|
657
|
+
let should_reuse = slot
|
|
658
|
+
.as_ref()
|
|
659
|
+
.map(|cached| &cached.device_key == device_key)
|
|
660
|
+
.unwrap_or(false);
|
|
661
|
+
if should_reuse {
|
|
662
|
+
slot.take()
|
|
663
|
+
} else {
|
|
664
|
+
// A selected mic changed while the engine was warm. Drop the stale
|
|
665
|
+
// engine so the next start configures the requested device.
|
|
666
|
+
if let Some(stale) = slot.take() {
|
|
667
|
+
discard_warmed_raw_mic_engine(stale);
|
|
668
|
+
}
|
|
669
|
+
None
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
fn store_warmed_raw_mic_engine(engine: WarmedRawMicEngine) {
|
|
674
|
+
if let Ok(mut slot) = warmed_raw_mic_engine_slot().lock() {
|
|
675
|
+
if let Some(previous) = slot.replace(engine) {
|
|
676
|
+
discard_warmed_raw_mic_engine(previous);
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
fn clear_warmed_raw_mic_engine() {
|
|
682
|
+
if let Ok(mut slot) = warmed_raw_mic_engine_slot().lock() {
|
|
683
|
+
if let Some(engine) = slot.take() {
|
|
684
|
+
discard_warmed_raw_mic_engine(engine);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
fn discard_warmed_raw_mic_engine(engine: WarmedRawMicEngine) {
|
|
690
|
+
let WarmedRawMicEngine {
|
|
691
|
+
engine,
|
|
692
|
+
input_node,
|
|
693
|
+
device_key: _,
|
|
694
|
+
target,
|
|
695
|
+
} = engine;
|
|
696
|
+
if let Ok(mut current) = target.lock() {
|
|
697
|
+
*current = None;
|
|
698
|
+
}
|
|
699
|
+
unsafe {
|
|
700
|
+
if engine.isRunning() {
|
|
701
|
+
engine.stop();
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
let _ = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
|
|
705
|
+
input_node.removeTapOnBus(0);
|
|
706
|
+
}));
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
fn install_reusable_raw_mic_tap(
|
|
710
|
+
input_node: &AVAudioInputNode,
|
|
711
|
+
target: Arc<Mutex<Option<RawMicTapTarget>>>,
|
|
712
|
+
) -> Result<(), String> {
|
|
713
|
+
let target_for_tap = target.clone();
|
|
714
|
+
let tap_block = StackBlock::new(
|
|
715
|
+
move |buffer: std::ptr::NonNull<AVAudioPCMBuffer>,
|
|
716
|
+
_when: std::ptr::NonNull<AVAudioTime>| {
|
|
717
|
+
let Some(target) = target_for_tap
|
|
718
|
+
.try_lock()
|
|
719
|
+
.ok()
|
|
720
|
+
.and_then(|guard| guard.as_ref().cloned())
|
|
721
|
+
else {
|
|
722
|
+
return;
|
|
723
|
+
};
|
|
724
|
+
let buf = unsafe { buffer.as_ref() };
|
|
725
|
+
let mono = mono_mix_pcm(buf);
|
|
726
|
+
if !mono.is_empty() {
|
|
727
|
+
(target.on_samples)(&mono);
|
|
728
|
+
}
|
|
729
|
+
let n = target.level_tick.fetch_add(1, Ordering::Relaxed);
|
|
730
|
+
if n % 2 == 0 {
|
|
731
|
+
let level = peak_level_for_pcm(buf);
|
|
732
|
+
let _ = target.app.emit(
|
|
733
|
+
"voice:audio-level",
|
|
734
|
+
AudioLevelPayload {
|
|
735
|
+
level,
|
|
736
|
+
source: "mic",
|
|
737
|
+
},
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
},
|
|
741
|
+
)
|
|
742
|
+
.copy();
|
|
743
|
+
let block_ptr: *mut block2::Block<
|
|
744
|
+
dyn Fn(std::ptr::NonNull<AVAudioPCMBuffer>, std::ptr::NonNull<AVAudioTime>) + 'static,
|
|
745
|
+
> = (&*tap_block) as *const _ as *mut _;
|
|
746
|
+
objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
|
|
747
|
+
input_node.installTapOnBus_bufferSize_format_block(
|
|
748
|
+
0, 1024,
|
|
749
|
+
None, // use hardware's current format — avoids SCO↔A2DP stale-format race
|
|
750
|
+
block_ptr,
|
|
751
|
+
);
|
|
752
|
+
}))
|
|
753
|
+
.map_err(|e| format!("installTapOnBus threw: {e:?}"))
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
fn build_warmed_raw_mic_engine(
|
|
757
|
+
mic_device_id: Option<&str>,
|
|
758
|
+
mic_device_label: Option<&str>,
|
|
759
|
+
device_key: RawMicDeviceKey,
|
|
760
|
+
) -> Result<WarmedRawMicEngine, String> {
|
|
761
|
+
let engine: Retained<AVAudioEngine> = unsafe { AVAudioEngine::new() };
|
|
762
|
+
configure_engine_input_device(&engine, mic_device_id, mic_device_label)?;
|
|
763
|
+
let input_node: Retained<AVAudioInputNode> = unsafe { engine.inputNode() };
|
|
764
|
+
enable_voice_processing(&input_node);
|
|
765
|
+
// Finalize input format negotiation once. Dictation-owned Whisper starts
|
|
766
|
+
// can then reuse this stopped VPIO engine instead of paying the
|
|
767
|
+
// per-press VoiceProcessingIO setup cost.
|
|
768
|
+
objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe { engine.prepare() }))
|
|
769
|
+
.map_err(|e| format!("AVAudioEngine prepare threw: {e:?}"))?;
|
|
770
|
+
let target = Arc::new(Mutex::new(None));
|
|
771
|
+
install_reusable_raw_mic_tap(&input_node, target.clone())?;
|
|
772
|
+
disable_voice_processing_ducking(&input_node);
|
|
773
|
+
Ok(WarmedRawMicEngine {
|
|
774
|
+
engine,
|
|
775
|
+
input_node,
|
|
776
|
+
device_key,
|
|
777
|
+
target,
|
|
778
|
+
})
|
|
779
|
+
}
|
|
780
|
+
|
|
577
781
|
/// Tear down whatever session is currently running. Called from `start`
|
|
578
782
|
/// to guarantee a fresh slate, and from `cancel` / `stop` for explicit
|
|
579
783
|
/// teardown.
|
|
@@ -983,12 +1187,13 @@ pub(crate) mod macos {
|
|
|
983
1187
|
Ok(())
|
|
984
1188
|
}
|
|
985
1189
|
|
|
986
|
-
/// Handle for a running raw mic capture. `stop()`
|
|
987
|
-
/// the
|
|
1190
|
+
/// Handle for a running raw mic capture. `stop()` either tears down the tap
|
|
1191
|
+
/// or parks a dictation-owned VPIO engine for the next press.
|
|
988
1192
|
pub(crate) struct RawMicCapture {
|
|
989
1193
|
engine: Retained<AVAudioEngine>,
|
|
990
|
-
input_node: Retained<
|
|
1194
|
+
input_node: Retained<AVAudioInputNode>,
|
|
991
1195
|
sample_rate: f64,
|
|
1196
|
+
warm_cache: Option<RawMicWarmCache>,
|
|
992
1197
|
}
|
|
993
1198
|
|
|
994
1199
|
// SAFETY: same argument as `SpeechSession` — refcounted ObjC objects that
|
|
@@ -1003,15 +1208,103 @@ pub(crate) mod macos {
|
|
|
1003
1208
|
}
|
|
1004
1209
|
|
|
1005
1210
|
pub(crate) fn stop(self) {
|
|
1211
|
+
let RawMicCapture {
|
|
1212
|
+
engine,
|
|
1213
|
+
input_node,
|
|
1214
|
+
sample_rate: _,
|
|
1215
|
+
warm_cache,
|
|
1216
|
+
} = self;
|
|
1217
|
+
if let Some(cache) = warm_cache {
|
|
1218
|
+
unsafe {
|
|
1219
|
+
if engine.isRunning() {
|
|
1220
|
+
engine.stop();
|
|
1221
|
+
}
|
|
1222
|
+
}
|
|
1223
|
+
if let Ok(mut target) = cache.target.lock() {
|
|
1224
|
+
*target = None;
|
|
1225
|
+
}
|
|
1226
|
+
store_warmed_raw_mic_engine(WarmedRawMicEngine {
|
|
1227
|
+
engine,
|
|
1228
|
+
input_node,
|
|
1229
|
+
device_key: cache.device_key,
|
|
1230
|
+
target: cache.target,
|
|
1231
|
+
});
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1006
1234
|
unsafe {
|
|
1007
|
-
|
|
1008
|
-
if
|
|
1009
|
-
|
|
1235
|
+
input_node.removeTapOnBus(0);
|
|
1236
|
+
if engine.isRunning() {
|
|
1237
|
+
engine.stop();
|
|
1010
1238
|
}
|
|
1011
1239
|
}
|
|
1012
1240
|
}
|
|
1013
1241
|
}
|
|
1014
1242
|
|
|
1243
|
+
fn start_warmed_raw_mic_capture(
|
|
1244
|
+
app: AppHandle,
|
|
1245
|
+
mic_device_id: Option<String>,
|
|
1246
|
+
mic_device_label: Option<String>,
|
|
1247
|
+
on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
|
|
1248
|
+
) -> Result<RawMicCapture, String> {
|
|
1249
|
+
let device_key =
|
|
1250
|
+
RawMicDeviceKey::from_selection(mic_device_id.as_deref(), mic_device_label.as_deref());
|
|
1251
|
+
let warmed = match take_warmed_raw_mic_engine(&device_key) {
|
|
1252
|
+
Some(warmed) => warmed,
|
|
1253
|
+
None => build_warmed_raw_mic_engine(
|
|
1254
|
+
mic_device_id.as_deref(),
|
|
1255
|
+
mic_device_label.as_deref(),
|
|
1256
|
+
device_key.clone(),
|
|
1257
|
+
)?,
|
|
1258
|
+
};
|
|
1259
|
+
if let Ok(mut target) = warmed.target.lock() {
|
|
1260
|
+
*target = Some(RawMicTapTarget {
|
|
1261
|
+
app: app.clone(),
|
|
1262
|
+
on_samples,
|
|
1263
|
+
level_tick: Arc::new(AtomicU32::new(0)),
|
|
1264
|
+
});
|
|
1265
|
+
} else {
|
|
1266
|
+
return Err("warmed mic target lock poisoned".into());
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
let start_result = objc2::exception::catch(std::panic::AssertUnwindSafe(|| unsafe {
|
|
1270
|
+
warmed.engine.startAndReturnError()
|
|
1271
|
+
}))
|
|
1272
|
+
.map_err(|e| format!("AVAudioEngine start threw: {e:?}"))
|
|
1273
|
+
.and_then(|r| {
|
|
1274
|
+
r.map_err(|e| format!("AVAudioEngine start failed: {}", ns_error_message(&e)))
|
|
1275
|
+
});
|
|
1276
|
+
if let Err(msg) = start_result {
|
|
1277
|
+
if let Ok(mut target) = warmed.target.lock() {
|
|
1278
|
+
*target = None;
|
|
1279
|
+
}
|
|
1280
|
+
return Err(msg);
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
// Read the format only after the engine has started — at this point the
|
|
1284
|
+
// engine has committed to its I/O configuration and the sample rate
|
|
1285
|
+
// matches what the tap will actually deliver.
|
|
1286
|
+
let format = unsafe { warmed.input_node.outputFormatForBus(0) };
|
|
1287
|
+
let sample_rate = unsafe { format.sampleRate() };
|
|
1288
|
+
eprintln!(
|
|
1289
|
+
"[whisper-mic] reused warm VPIO tap format: {} Hz, {} ch",
|
|
1290
|
+
sample_rate as u32,
|
|
1291
|
+
unsafe { format.channelCount() }
|
|
1292
|
+
);
|
|
1293
|
+
|
|
1294
|
+
let WarmedRawMicEngine {
|
|
1295
|
+
engine,
|
|
1296
|
+
input_node,
|
|
1297
|
+
device_key,
|
|
1298
|
+
target,
|
|
1299
|
+
} = warmed;
|
|
1300
|
+
Ok(RawMicCapture {
|
|
1301
|
+
engine,
|
|
1302
|
+
input_node,
|
|
1303
|
+
sample_rate,
|
|
1304
|
+
warm_cache: Some(RawMicWarmCache { device_key, target }),
|
|
1305
|
+
})
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1015
1308
|
/// Start mic capture and forward a mono mix of every available channel to
|
|
1016
1309
|
/// `on_samples`. Recording transcription can disable VPIO so the tap does
|
|
1017
1310
|
/// not precondition the shared mic before ScreenCaptureKit opens it.
|
|
@@ -1020,23 +1313,24 @@ pub(crate) mod macos {
|
|
|
1020
1313
|
mic_device_id: Option<String>,
|
|
1021
1314
|
mic_device_label: Option<String>,
|
|
1022
1315
|
voice_processing: bool,
|
|
1316
|
+
reuse_voice_processing_engine: bool,
|
|
1023
1317
|
on_samples: Arc<dyn Fn(&[f32]) + Send + Sync>,
|
|
1024
1318
|
) -> Result<RawMicCapture, String> {
|
|
1319
|
+
if voice_processing && reuse_voice_processing_engine {
|
|
1320
|
+
return start_warmed_raw_mic_capture(app, mic_device_id, mic_device_label, on_samples);
|
|
1321
|
+
}
|
|
1322
|
+
clear_warmed_raw_mic_engine();
|
|
1323
|
+
|
|
1025
1324
|
let engine: Retained<AVAudioEngine> = unsafe { AVAudioEngine::new() };
|
|
1026
1325
|
configure_engine_input_device(
|
|
1027
1326
|
&engine,
|
|
1028
1327
|
mic_device_id.as_deref(),
|
|
1029
1328
|
mic_device_label.as_deref(),
|
|
1030
1329
|
)?;
|
|
1031
|
-
let input_node: Retained<
|
|
1330
|
+
let input_node: Retained<AVAudioInputNode> = unsafe { engine.inputNode() };
|
|
1032
1331
|
|
|
1033
1332
|
if voice_processing {
|
|
1034
|
-
|
|
1035
|
-
eprintln!(
|
|
1036
|
-
"[whisper-mic] voice processing enable failed: {} — continuing without AEC",
|
|
1037
|
-
ns_error_message(&err)
|
|
1038
|
-
);
|
|
1039
|
-
}
|
|
1333
|
+
enable_voice_processing(&input_node);
|
|
1040
1334
|
} else {
|
|
1041
1335
|
eprintln!("[whisper-mic] voice processing disabled for shared mic capture");
|
|
1042
1336
|
}
|
|
@@ -1102,28 +1396,14 @@ pub(crate) mod macos {
|
|
|
1102
1396
|
);
|
|
1103
1397
|
|
|
1104
1398
|
if voice_processing {
|
|
1105
|
-
|
|
1106
|
-
// isn't dropped to near-silent while the mic VPIO runs.
|
|
1107
|
-
unsafe {
|
|
1108
|
-
let responds: bool = objc2::msg_send![
|
|
1109
|
-
&*input_node,
|
|
1110
|
-
respondsToSelector: objc2::sel!(setVoiceProcessingOtherAudioDuckingConfiguration:)
|
|
1111
|
-
];
|
|
1112
|
-
if responds {
|
|
1113
|
-
input_node.setVoiceProcessingOtherAudioDuckingConfiguration(
|
|
1114
|
-
AVAudioVoiceProcessingOtherAudioDuckingConfiguration {
|
|
1115
|
-
enableAdvancedDucking: objc2::runtime::Bool::NO,
|
|
1116
|
-
duckingLevel: AVAudioVoiceProcessingOtherAudioDuckingLevel::Min,
|
|
1117
|
-
},
|
|
1118
|
-
);
|
|
1119
|
-
}
|
|
1120
|
-
}
|
|
1399
|
+
disable_voice_processing_ducking(&input_node);
|
|
1121
1400
|
}
|
|
1122
1401
|
|
|
1123
1402
|
Ok(RawMicCapture {
|
|
1124
1403
|
engine,
|
|
1125
1404
|
input_node,
|
|
1126
1405
|
sample_rate,
|
|
1406
|
+
warm_cache: None,
|
|
1127
1407
|
})
|
|
1128
1408
|
}
|
|
1129
1409
|
|
|
@@ -656,11 +656,13 @@ mod macos {
|
|
|
656
656
|
session_start,
|
|
657
657
|
);
|
|
658
658
|
let mic_for_cb = mic_stream.clone();
|
|
659
|
+
let reuse_voice_processing_engine = owner == SessionOwner::Dictation && !capture_system;
|
|
659
660
|
let mic_cap = start_raw_mic_capture(
|
|
660
661
|
app.clone(),
|
|
661
662
|
mic_device_id,
|
|
662
663
|
mic_device_label,
|
|
663
664
|
voice_processing,
|
|
665
|
+
reuse_voice_processing_engine,
|
|
664
666
|
Arc::new(move |s: &[f32]| mic_for_cb.push(s)),
|
|
665
667
|
)
|
|
666
668
|
.map_err(|e| {
|
|
@@ -643,6 +643,15 @@ regeneration is slow, expensive, and regresses unrelated parts.
|
|
|
643
643
|
`data-screen`, or `#` anchors (see Multi-screen prototypes & navigation).
|
|
644
644
|
- Never hardcode colors — always reference CSS custom properties (no raw
|
|
645
645
|
`text-white` / `bg-black` / hex literals in markup).
|
|
646
|
+
- Never define or rely on a CSS custom property starting with `--agent-native-`
|
|
647
|
+
in generated design content. That prefix (plus `--design-editor-`) is
|
|
648
|
+
reserved for editor-internal state (selection chrome, editor-chrome scale
|
|
649
|
+
compensation, clipboard/surface tokens) — the editor strips any property
|
|
650
|
+
under those prefixes before persisting a cross-screen style capture
|
|
651
|
+
(`isEditorInternalCssVar` in `app/pages/DesignEditor.tsx`), so a design that
|
|
652
|
+
defines its own theming under that namespace would silently lose those
|
|
653
|
+
values on a cross-screen move. Use `--color-*`, `--font-*`, or another
|
|
654
|
+
design-owned prefix for tokens/tweaks instead.
|
|
646
655
|
- Never use the v3 `bg-gradient-to-*` classes — use v4 `bg-linear-to-*`.
|
|
647
656
|
- Never use `<script>` blocks with raw DOM manipulation — use Alpine.js directives
|
|
648
657
|
- Never inline `onclick="..."` handlers — use `@click`
|