@agent-native/core 0.144.2 → 0.145.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/corpus/README.md +1 -1
  2. package/corpus/templates/analytics/app/pages/DataSources.tsx +28 -4
  3. package/corpus/templates/analytics/server/jobs/dashboard-report.ts +7 -3
  4. package/corpus/templates/analytics/server/plugins/dashboard-report-jobs.ts +13 -3
  5. package/corpus/templates/calendar/app/components/calendar/GoogleConnectBanner.tsx +64 -23
  6. package/corpus/templates/calendar/app/hooks/use-google-auth.ts +69 -46
  7. package/corpus/templates/calendar/app/hooks/use-view-preferences.ts +35 -6
  8. package/corpus/templates/calendar/app/hooks/use-zoom-auth.ts +24 -3
  9. package/corpus/templates/clips/app/components/player/transcript-panel.tsx +45 -6
  10. package/corpus/templates/clips/app/components/recorder/storage-setup-card.tsx +38 -5
  11. package/corpus/templates/clips/changelog/2026-08-06-desktop-full-screen-recordings-now-normalize-microphone-leve.md +6 -0
  12. package/corpus/templates/clips/desktop/src/app.tsx +146 -45
  13. package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +185 -69
  14. package/corpus/templates/clips/server/jobs/brain-export.ts +13 -6
  15. package/corpus/templates/clips/server/jobs/builder-media-compression.ts +10 -3
  16. package/corpus/templates/clips/server/jobs/meeting-reminders.ts +10 -3
  17. package/corpus/templates/clips/server/jobs/poll-calendars.ts +23 -21
  18. package/corpus/templates/clips/server/jobs/stale-meeting-sweeper.ts +10 -3
  19. package/corpus/templates/clips/server/jobs/transactional-emails.ts +10 -3
  20. package/corpus/templates/content/app/components/sidebar/NotionButton.tsx +1 -0
  21. package/corpus/templates/design/app/components/design/FusionAppBanner.tsx +15 -2
  22. package/corpus/templates/design/app/components/design/MultiScreenCanvas.tsx +47 -4
  23. package/corpus/templates/forms/server/plugins/agent-chat.ts +1 -1
  24. package/corpus/templates/mail/app/components/GoogleConnectBanner.tsx +82 -39
  25. package/corpus/templates/mail/app/components/layout/AppLayout.tsx +31 -9
  26. package/corpus/templates/mail/server/plugins/mail-jobs.ts +36 -17
  27. package/corpus/templates/slides/server/plugins/agent-chat.ts +1 -1
  28. package/dist/agent/production-agent.d.ts +6 -0
  29. package/dist/client/AssistantChat.js +73 -43
  30. package/dist/client/MultiTabAssistantChat.d.ts +3 -1
  31. package/dist/client/MultiTabAssistantChat.js +79 -92
  32. package/dist/client/browser-session-bridge.js +97 -54
  33. package/dist/client/db-admin/useAgentSync.js +34 -43
  34. package/dist/client/dynamic-suggestions.d.ts +3 -0
  35. package/dist/client/dynamic-suggestions.js +54 -52
  36. package/dist/client/hooks/index.d.ts +1 -0
  37. package/dist/client/hooks/index.js +1 -0
  38. package/dist/client/i18n.d.ts +7 -0
  39. package/dist/client/i18n.js +12 -3
  40. package/dist/client/integrations/useIntegrationStatus.d.ts +1 -1
  41. package/dist/client/integrations/useIntegrationStatus.js +9 -22
  42. package/dist/client/notifications/NotificationsBell.js +14 -9
  43. package/dist/client/progress/RunsTray.js +6 -5
  44. package/dist/client/settings/useBuilderStatus.js +95 -73
  45. package/dist/client/transcription/BuilderTranscriptionCta.js +27 -31
  46. package/dist/client/use-agent-engine-configured.js +21 -3
  47. package/dist/client/use-pausing-interval.d.ts +6 -0
  48. package/dist/client/use-pausing-interval.js +6 -0
  49. package/dist/client/use-poll-loop.d.ts +39 -0
  50. package/dist/client/use-poll-loop.js +73 -0
  51. package/dist/client/use-run-stuck-detection.js +41 -13
  52. package/dist/file-upload/actions/upload-image.d.ts +1 -1
  53. package/dist/framework-tools.d.ts +126 -0
  54. package/dist/framework-tools.js +114 -0
  55. package/dist/integrations/google-docs-poller.js +26 -7
  56. package/dist/integrations/pending-tasks-retry-job.js +17 -23
  57. package/dist/integrations/plugin.js +16 -15
  58. package/dist/integrations/remote-push-delivery-job.js +16 -25
  59. package/dist/integrations/remote-retry-job.js +19 -23
  60. package/dist/mcp/screen-memory-stdio.d.ts +7 -7
  61. package/dist/notifications/routes.d.ts +3 -3
  62. package/dist/oauth-tokens/google-refresh.js +12 -19
  63. package/dist/observability/routes.d.ts +3 -3
  64. package/dist/progress/routes.d.ts +1 -1
  65. package/dist/provider-api/actions/custom-provider-registration.d.ts +9 -9
  66. package/dist/provider-api/actions/provider-api.d.ts +11 -11
  67. package/dist/resources/handlers.d.ts +1 -1
  68. package/dist/secrets/routes.d.ts +3 -3
  69. package/dist/server/action-discovery.d.ts +27 -0
  70. package/dist/server/action-discovery.js +89 -0
  71. package/dist/server/agent-chat/action-filters-a2a.d.ts +11 -0
  72. package/dist/server/agent-chat/action-filters-a2a.js +17 -1
  73. package/dist/server/agent-chat/framework-prompts.d.ts +2 -0
  74. package/dist/server/agent-chat/plugin-options.d.ts +30 -0
  75. package/dist/server/agent-chat/prompt-resources.d.ts +4 -1
  76. package/dist/server/agent-chat/prompt-resources.js +8 -2
  77. package/dist/server/agent-chat-plugin.js +148 -100
  78. package/dist/server/index.d.ts +1 -0
  79. package/dist/server/index.js +1 -0
  80. package/dist/server/interval-job.d.ts +25 -0
  81. package/dist/server/interval-job.js +19 -0
  82. package/dist/server/prompts/framework-core-compact.d.ts +5 -0
  83. package/dist/server/prompts/framework-core-compact.js +28 -12
  84. package/dist/server/prompts/framework-core.d.ts +5 -0
  85. package/dist/server/prompts/framework-core.js +33 -16
  86. package/dist/server/prompts/index.d.ts +1 -1
  87. package/dist/server/prompts/index.js +1 -1
  88. package/dist/server/prompts/shared-rules.d.ts +12 -4
  89. package/dist/server/prompts/shared-rules.js +14 -4
  90. package/dist/server/realtime-token.d.ts +1 -1
  91. package/dist/shared/index.d.ts +1 -0
  92. package/dist/shared/index.js +1 -0
  93. package/dist/shared/poll-engine.d.ts +49 -0
  94. package/dist/shared/poll-engine.js +148 -0
  95. package/dist/templates/default/.agents/skills/internationalization/SKILL.md +5 -0
  96. package/dist/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -3
  97. package/dist/templates/workspace-core/.agents/skills/internationalization/SKILL.md +5 -0
  98. package/dist/tracking/providers.js +10 -1
  99. package/docs/content/deployment.mdx +47 -13
  100. package/docs/content/key-concepts.mdx +13 -6
  101. package/package.json +2 -1
  102. package/src/templates/default/.agents/skills/internationalization/SKILL.md +5 -0
  103. package/src/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -3
  104. package/src/templates/workspace-core/.agents/skills/internationalization/SKILL.md +5 -0
@@ -61,6 +61,18 @@ export function StorageSetupCard({
61
61
  const [err, setErr] = useState<string | null>(null);
62
62
  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
63
63
  const mountedRef = useRef(true);
64
+ const inFlightRef = useRef(false);
65
+ const visibilityHandlerRef = useRef<(() => void) | null>(null);
66
+
67
+ const stopVisibilityHandler = useCallback(() => {
68
+ if (visibilityHandlerRef.current) {
69
+ document.removeEventListener(
70
+ "visibilitychange",
71
+ visibilityHandlerRef.current,
72
+ );
73
+ visibilityHandlerRef.current = null;
74
+ }
75
+ }, []);
64
76
 
65
77
  useEffect(() => {
66
78
  mountedRef.current = true;
@@ -70,14 +82,17 @@ export function StorageSetupCard({
70
82
  clearInterval(pollRef.current);
71
83
  pollRef.current = null;
72
84
  }
85
+ stopVisibilityHandler();
73
86
  };
74
- }, []);
87
+ }, [stopVisibilityHandler]);
75
88
 
76
89
  const handleConnect = useCallback(() => {
77
90
  if (pollRef.current) {
78
91
  clearInterval(pollRef.current);
79
92
  pollRef.current = null;
80
93
  }
94
+ stopVisibilityHandler();
95
+ inFlightRef.current = false;
81
96
  setConnecting(true);
82
97
  setErr(null);
83
98
 
@@ -93,14 +108,23 @@ export function StorageSetupCard({
93
108
  clearInterval(pollRef.current);
94
109
  pollRef.current = null;
95
110
  }
111
+ stopVisibilityHandler();
96
112
  };
97
- pollRef.current = setInterval(async () => {
113
+ const tick = async () => {
114
+ if (document.hidden || inFlightRef.current) return;
115
+ inFlightRef.current = true;
116
+ const controller = new AbortController();
117
+ const abortTimer = setTimeout(
118
+ () => controller.abort(),
119
+ Math.max(10_000, 2000 * 4),
120
+ );
98
121
  try {
99
122
  const r = await fetch(
100
123
  new URL(
101
124
  agentNativePath("/_agent-native/builder/status"),
102
125
  window.location.origin,
103
126
  ).toString(),
127
+ { signal: controller.signal },
104
128
  );
105
129
  if (!r.ok) return;
106
130
  const s = (await r.json()) as { configured: boolean };
@@ -119,10 +143,19 @@ export function StorageSetupCard({
119
143
  setErr(t("storageSetup.builderTimeout"));
120
144
  }
121
145
  } catch {
122
- // transient poll error
146
+ // coercion-ok: a transient poll error keeps the loop running; the
147
+ // timeoutMs bound above surfaces builderTimeout if it never succeeds.
148
+ } finally {
149
+ clearTimeout(abortTimer);
150
+ inFlightRef.current = false;
123
151
  }
124
- }, 2000);
125
- }, [onConfigured, connectSource, connectFlow]);
152
+ };
153
+ pollRef.current = setInterval(() => void tick(), 2000);
154
+ visibilityHandlerRef.current = () => {
155
+ if (!document.hidden) void tick();
156
+ };
157
+ document.addEventListener("visibilitychange", visibilityHandlerRef.current);
158
+ }, [onConfigured, connectSource, connectFlow, stopVisibilityHandler, t]);
126
159
 
127
160
  return (
128
161
  <div className="mx-auto flex w-full max-w-md flex-col gap-5 rounded-2xl border border-border bg-card p-6 shadow-lg">
@@ -0,0 +1,6 @@
1
+ ---
2
+ type: fixed
3
+ date: 2026-08-06
4
+ ---
5
+
6
+ Desktop full-screen recordings now normalize microphone level while recording, so clips are no longer quieter than the rest of your library.
@@ -427,6 +427,24 @@ function serverUrlForPendingUpload(
427
427
  // already-connected user to the setup flow.
428
428
  type VideoStorageProbe = "configured" | "missing" | "unknown";
429
429
 
430
+ // Poll cadence for the caller's re-check loop is 5s; bound each probe request
431
+ // well above that so a hung request can't wedge the poll's in-flight guard.
432
+ const VIDEO_STORAGE_PROBE_TIMEOUT_MS = Math.max(10_000, 5000 * 4);
433
+
434
+ async function fetchWithAbortTimeout(
435
+ url: string,
436
+ init: RequestInit,
437
+ timeoutMs: number,
438
+ ): Promise<Response> {
439
+ const controller = new AbortController();
440
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
441
+ try {
442
+ return await fetch(url, { ...init, signal: controller.signal });
443
+ } finally {
444
+ clearTimeout(timer);
445
+ }
446
+ }
447
+
430
448
  async function hasConfiguredVideoStorage(
431
449
  serverUrl: string,
432
450
  ): Promise<VideoStorageProbe> {
@@ -437,12 +455,13 @@ async function hasConfiguredVideoStorage(
437
455
  let sawDefinitiveAnswer = false;
438
456
 
439
457
  try {
440
- const uploadStatus = await fetch(
458
+ const uploadStatus = await fetchWithAbortTimeout(
441
459
  `${base}/_agent-native/file-upload/status`,
442
460
  {
443
461
  credentials: "include",
444
462
  cache: "no-store",
445
463
  },
464
+ VIDEO_STORAGE_PROBE_TIMEOUT_MS,
446
465
  );
447
466
  if (uploadStatus.ok) {
448
467
  const body = (await uploadStatus.json().catch(() => null)) as {
@@ -458,10 +477,14 @@ async function hasConfiguredVideoStorage(
458
477
  }
459
478
 
460
479
  try {
461
- const builderStatus = await fetch(`${base}/_agent-native/builder/status`, {
462
- credentials: "include",
463
- cache: "no-store",
464
- });
480
+ const builderStatus = await fetchWithAbortTimeout(
481
+ `${base}/_agent-native/builder/status`,
482
+ {
483
+ credentials: "include",
484
+ cache: "no-store",
485
+ },
486
+ VIDEO_STORAGE_PROBE_TIMEOUT_MS,
487
+ );
465
488
  if (builderStatus.ok) {
466
489
  const body = (await builderStatus.json().catch(() => null)) as {
467
490
  configured?: boolean;
@@ -1143,10 +1166,23 @@ export function App() {
1143
1166
  return;
1144
1167
  }
1145
1168
 
1146
- const interval = window.setInterval(() => {
1147
- void refreshVideoStorageStatus();
1148
- }, 5000);
1149
- return () => window.clearInterval(interval);
1169
+ let inFlight = false;
1170
+ const tick = () => {
1171
+ if (document.hidden || inFlight) return;
1172
+ inFlight = true;
1173
+ void refreshVideoStorageStatus().finally(() => {
1174
+ inFlight = false;
1175
+ });
1176
+ };
1177
+ const interval = window.setInterval(tick, 5000);
1178
+ const onVisibilityChange = () => {
1179
+ if (!document.hidden) tick();
1180
+ };
1181
+ document.addEventListener("visibilitychange", onVisibilityChange);
1182
+ return () => {
1183
+ window.clearInterval(interval);
1184
+ document.removeEventListener("visibilitychange", onVisibilityChange);
1185
+ };
1150
1186
  }, [
1151
1187
  authStatus,
1152
1188
  localRecordingMode,
@@ -1630,22 +1666,45 @@ export function App() {
1630
1666
  return;
1631
1667
  }
1632
1668
  let cancelled = false;
1669
+ let inFlight = false;
1633
1670
  const poll = async () => {
1634
- const result = await callClipsAction<{
1635
- requests?: RewindExtensionRequest[];
1636
- }>("list-rewind-extension-requests", {}, { method: "GET" }).catch(
1637
- () => null,
1671
+ if (document.hidden || inFlight) return;
1672
+ inFlight = true;
1673
+ const controller = new AbortController();
1674
+ const abortTimer = setTimeout(
1675
+ () => controller.abort(),
1676
+ Math.max(10_000, 3_000 * 4),
1638
1677
  );
1639
- if (cancelled) return;
1640
- for (const request of result?.requests ?? []) {
1641
- void processRewindExtension(request);
1678
+ try {
1679
+ const result = await callClipsAction<{
1680
+ requests?: RewindExtensionRequest[];
1681
+ }>(
1682
+ "list-rewind-extension-requests",
1683
+ {},
1684
+ { method: "GET", signal: controller.signal },
1685
+ )
1686
+ // coercion-ok: nothing to process this sweep either way; the next
1687
+ // tick re-reads the pending requests.
1688
+ .catch(() => null);
1689
+ if (cancelled) return;
1690
+ for (const request of result?.requests ?? []) {
1691
+ void processRewindExtension(request);
1692
+ }
1693
+ } finally {
1694
+ clearTimeout(abortTimer);
1695
+ inFlight = false;
1642
1696
  }
1643
1697
  };
1644
1698
  void poll();
1645
1699
  const timer = window.setInterval(() => void poll(), 3_000);
1700
+ const onVisibilityChange = () => {
1701
+ if (!document.hidden) void poll();
1702
+ };
1703
+ document.addEventListener("visibilitychange", onVisibilityChange);
1646
1704
  return () => {
1647
1705
  cancelled = true;
1648
1706
  window.clearInterval(timer);
1707
+ document.removeEventListener("visibilitychange", onVisibilityChange);
1649
1708
  };
1650
1709
  }, [
1651
1710
  authStatus,
@@ -1702,41 +1761,63 @@ export function App() {
1702
1761
  useEffect(() => {
1703
1762
  if (featureConfig?.screenMemory?.enabled !== true) return;
1704
1763
  let cancelled = false;
1764
+ let inFlight = false;
1705
1765
  const sweep = async () => {
1706
- const due = await invoke<DueRewindAgentHandoff[]>(
1707
- "screen_memory_due_agent_handoffs",
1708
- ).catch(() => []);
1709
- if (cancelled) return;
1710
- for (const item of due) {
1711
- try {
1712
- const cleanup = await callClipsAction<{
1713
- deleted: boolean;
1714
- reason: string;
1715
- }>("delete-agent-recording-if-unpromoted", {
1716
- id: item.recordingId,
1717
- });
1718
- if (cleanup.deleted) {
1719
- await invoke("screen_memory_mark_agent_handoff_deleted", {
1720
- requestId: item.requestId,
1721
- });
1722
- } else if (cleanup.reason === "promoted") {
1723
- await invoke("screen_memory_cancel_agent_handoff_cleanup", {
1724
- requestId: item.requestId,
1725
- });
1766
+ if (document.hidden || inFlight) return;
1767
+ inFlight = true;
1768
+ try {
1769
+ const due = await invoke<DueRewindAgentHandoff[]>(
1770
+ "screen_memory_due_agent_handoffs",
1771
+ )
1772
+ // coercion-ok: handoffs stay due in the local store, so an empty
1773
+ // sweep and a failed one both retry on the next tick.
1774
+ .catch(() => []);
1775
+ if (cancelled) return;
1776
+ for (const item of due) {
1777
+ try {
1778
+ const controller = new AbortController();
1779
+ const abortTimer = setTimeout(
1780
+ () => controller.abort(),
1781
+ Math.max(10_000, 60_000 * 4),
1782
+ );
1783
+ const cleanup = await callClipsAction<{
1784
+ deleted: boolean;
1785
+ reason: string;
1786
+ }>(
1787
+ "delete-agent-recording-if-unpromoted",
1788
+ { id: item.recordingId },
1789
+ { signal: controller.signal },
1790
+ ).finally(() => clearTimeout(abortTimer));
1791
+ if (cleanup.deleted) {
1792
+ await invoke("screen_memory_mark_agent_handoff_deleted", {
1793
+ requestId: item.requestId,
1794
+ });
1795
+ } else if (cleanup.reason === "promoted") {
1796
+ await invoke("screen_memory_cancel_agent_handoff_cleanup", {
1797
+ requestId: item.requestId,
1798
+ });
1799
+ }
1800
+ } catch (error) {
1801
+ console.warn(
1802
+ "[clips-tray] agent-created Clip cleanup failed:",
1803
+ error,
1804
+ );
1726
1805
  }
1727
- } catch (error) {
1728
- console.warn(
1729
- "[clips-tray] agent-created Clip cleanup failed:",
1730
- error,
1731
- );
1732
1806
  }
1807
+ } finally {
1808
+ inFlight = false;
1733
1809
  }
1734
1810
  };
1735
1811
  void sweep();
1736
1812
  const timer = window.setInterval(() => void sweep(), 60_000);
1813
+ const onVisibilityChange = () => {
1814
+ if (!document.hidden) void sweep();
1815
+ };
1816
+ document.addEventListener("visibilitychange", onVisibilityChange);
1737
1817
  return () => {
1738
1818
  cancelled = true;
1739
1819
  window.clearInterval(timer);
1820
+ document.removeEventListener("visibilitychange", onVisibilityChange);
1740
1821
  };
1741
1822
  }, [callClipsAction, featureConfig?.screenMemory?.enabled]);
1742
1823
 
@@ -1967,11 +2048,18 @@ export function App() {
1967
2048
  if (signInInflightRef.current) return;
1968
2049
  signInInflightRef.current = true;
1969
2050
 
2051
+ let tickInFlight = false;
2052
+ let onVisibilityChange: (() => void) | null = null;
2053
+
1970
2054
  function stopPolling() {
1971
2055
  if (pollIntervalRef.current !== null) {
1972
2056
  clearInterval(pollIntervalRef.current);
1973
2057
  pollIntervalRef.current = null;
1974
2058
  }
2059
+ if (onVisibilityChange) {
2060
+ document.removeEventListener("visibilitychange", onVisibilityChange);
2061
+ onVisibilityChange = null;
2062
+ }
1975
2063
  }
1976
2064
 
1977
2065
  function finishWithError(message: string) {
@@ -2000,11 +2088,16 @@ export function App() {
2000
2088
  // Poll the exchange endpoint for the session token.
2001
2089
  const start = Date.now();
2002
2090
  const TIMEOUT_MS = 180_000; // 3 minutes
2003
- pollIntervalRef.current = setInterval(async () => {
2091
+ const POLL_ABORT_MS = Math.max(10_000, 1500 * 4);
2092
+ const tick = async () => {
2093
+ if (document.hidden || tickInFlight) return;
2094
+ tickInFlight = true;
2095
+ const controller = new AbortController();
2096
+ const abortTimer = setTimeout(() => controller.abort(), POLL_ABORT_MS);
2004
2097
  try {
2005
2098
  const xr = await fetch(
2006
2099
  `${base}/_agent-native/auth/desktop-exchange?flow_id=${flowId}`,
2007
- { credentials: "include" },
2100
+ { credentials: "include", signal: controller.signal },
2008
2101
  );
2009
2102
  if (!xr.ok) {
2010
2103
  if (Date.now() - start > TIMEOUT_MS) {
@@ -2030,7 +2123,7 @@ export function App() {
2030
2123
  // bearer token above is the reliable desktop auth path.
2031
2124
  await fetch(
2032
2125
  `${base}/_agent-native/auth/session?_session=${xd.token}`,
2033
- { credentials: "include" },
2126
+ { credentials: "include", signal: controller.signal },
2034
2127
  );
2035
2128
  signInInflightRef.current = false;
2036
2129
  setSignInPending(false);
@@ -2047,8 +2140,16 @@ export function App() {
2047
2140
  if (Date.now() - start > TIMEOUT_MS) {
2048
2141
  finishWithError("Google sign-in timed out. Please try again.");
2049
2142
  }
2143
+ } finally {
2144
+ clearTimeout(abortTimer);
2145
+ tickInFlight = false;
2050
2146
  }
2051
- }, 1500);
2147
+ };
2148
+ pollIntervalRef.current = setInterval(() => void tick(), 1500);
2149
+ onVisibilityChange = () => {
2150
+ if (!document.hidden) void tick();
2151
+ };
2152
+ document.addEventListener("visibilitychange", onVisibilityChange);
2052
2153
  } catch (err) {
2053
2154
  console.error("[clips-tray] signInExternal failed:", err);
2054
2155
  signInInflightRef.current = false;
@@ -2145,6 +2145,175 @@ const MIX_SOURCE_GRACE: Duration = Duration::from_millis(2000);
2145
2145
  const AUDIO_FORMAT_LPCM: u32 = 0x6C70_636D; // 'lpcm'
2146
2146
  const AUDIO_FORMAT_FLAGS_FLOAT_PACKED: u32 = 1 | 8; // float + packed, interleaved, little-endian
2147
2147
 
2148
+ /// Speech RMS the mic auto-gain aims for (~-20 dBFS), and how far it may
2149
+ /// travel to get there. It never attenuates: a hot mic keeps its own level
2150
+ /// and the limiter below owns peak safety.
2151
+ const MIC_AGC_TARGET_RMS: f32 = 0.1;
2152
+ const MIC_AGC_MIN_GAIN: f32 = 1.0;
2153
+ const MIC_AGC_MAX_GAIN: f32 = 8.0; // +18 dB
2154
+ /// Below this RMS the mic is between words: hold the current gain rather than
2155
+ /// chasing the target, or every pause swells room tone to speech level.
2156
+ ///
2157
+ /// Set at ~-60 dBFS, not the -50 dBFS that room tone alone would justify. A
2158
+ /// quiet built-in MacBook mic — the whole reason this stage exists — can carry
2159
+ /// speech at -45 dBFS RMS, and a floor set just under that would gate out the
2160
+ /// exact signal it is supposed to lift. Worst case here is room tone amplified
2161
+ /// to 0.001 × MIC_AGC_MAX_GAIN, still inaudible.
2162
+ const MIC_AGC_NOISE_FLOOR_RMS: f32 = 0.001;
2163
+ const MIC_AGC_ENVELOPE_SECONDS: f32 = 0.15;
2164
+ /// Asymmetric: back off quickly when the mic gets loud (the limiter should be
2165
+ /// a backstop, not the thing shaping the sound), return slowly so a boost
2166
+ /// doesn't audibly pump across sentences.
2167
+ const MIC_AGC_DUCK_SECONDS: f32 = 0.08;
2168
+ const MIC_AGC_BOOST_SECONDS: f32 = 1.5;
2169
+ /// Until the first speech has been levelled, converge at this rate instead of
2170
+ /// `MIC_AGC_BOOST_SECONDS`. Gain starts at unity, so a quiet mic needs to
2171
+ /// travel most of its range; at the steady-state rate that takes several
2172
+ /// seconds and the opening sentence audibly swells.
2173
+ const MIC_AGC_WARMUP_SECONDS: f32 = 0.2;
2174
+ /// How close to the target counts as levelled and ends the warmup.
2175
+ const MIC_AGC_CONVERGED_TOLERANCE: f32 = 0.05;
2176
+ /// -1.0 dBFS, near the true-peak ceiling the offline loudnorm chain targets.
2177
+ const MIX_LIMIT_CEILING: f32 = 0.891;
2178
+ const MIX_LIMIT_RELEASE_SECONDS: f32 = 0.15;
2179
+
2180
+ /// One-pole smoothing coefficient for a time constant at the mixer's rate.
2181
+ ///
2182
+ /// `exp_m1`, not `1.0 - exp(x)`: at 48 kHz the exponent is ~1e-5, so the
2183
+ /// subtraction form cancels away most of the f32 mantissa and the longer time
2184
+ /// constants come out with about three significant digits.
2185
+ fn one_pole_coefficient(tau_seconds: f32, sample_rate: i32) -> f32 {
2186
+ if tau_seconds <= 0.0 || sample_rate <= 0 {
2187
+ return 1.0;
2188
+ }
2189
+ -(-1.0 / (tau_seconds * sample_rate as f32)).exp_m1()
2190
+ }
2191
+
2192
+ /// Causal mic auto-gain.
2193
+ ///
2194
+ /// ScreenCaptureKit captures without AGC — unlike the browser path, which gets
2195
+ /// one from `autoGainControl` — so a MacBook mic lands far below the -16 LUFS
2196
+ /// the offline chain in `native_screen.rs` normalizes to. Live-uploaded clips
2197
+ /// never reach that chain: their bytes stream to the server while the writer
2198
+ /// is still producing them, so nothing downstream can re-read the file. Gain
2199
+ /// has to be decided here, from what has already been heard.
2200
+ struct MicAutoGain {
2201
+ mean_square: f32,
2202
+ gain: f32,
2203
+ /// False until the first speech has been levelled; see
2204
+ /// `MIC_AGC_WARMUP_SECONDS`.
2205
+ levelled: bool,
2206
+ envelope_coefficient: f32,
2207
+ warmup_coefficient: f32,
2208
+ duck_coefficient: f32,
2209
+ boost_coefficient: f32,
2210
+ }
2211
+
2212
+ impl MicAutoGain {
2213
+ fn new(sample_rate: i32) -> Self {
2214
+ Self {
2215
+ mean_square: 0.0,
2216
+ gain: 1.0,
2217
+ levelled: false,
2218
+ envelope_coefficient: one_pole_coefficient(MIC_AGC_ENVELOPE_SECONDS, sample_rate),
2219
+ warmup_coefficient: one_pole_coefficient(MIC_AGC_WARMUP_SECONDS, sample_rate),
2220
+ duck_coefficient: one_pole_coefficient(MIC_AGC_DUCK_SECONDS, sample_rate),
2221
+ boost_coefficient: one_pole_coefficient(MIC_AGC_BOOST_SECONDS, sample_rate),
2222
+ }
2223
+ }
2224
+
2225
+ fn next_gain(&mut self, left: f32, right: f32) -> f32 {
2226
+ let peak = left.abs().max(right.abs());
2227
+ self.mean_square += (peak * peak - self.mean_square) * self.envelope_coefficient;
2228
+ let rms = self.mean_square.max(0.0).sqrt();
2229
+ if rms >= MIC_AGC_NOISE_FLOOR_RMS {
2230
+ let target = (MIC_AGC_TARGET_RMS / rms).clamp(MIC_AGC_MIN_GAIN, MIC_AGC_MAX_GAIN);
2231
+ let coefficient = if !self.levelled {
2232
+ self.warmup_coefficient
2233
+ } else if target < self.gain {
2234
+ self.duck_coefficient
2235
+ } else {
2236
+ self.boost_coefficient
2237
+ };
2238
+ self.gain += (target - self.gain) * coefficient;
2239
+ if !self.levelled && (self.gain - target).abs() <= target * MIC_AGC_CONVERGED_TOLERANCE
2240
+ {
2241
+ self.levelled = true;
2242
+ // Once per recording. A "clip is too quiet" report is only
2243
+ // diagnosable if the log says what the mic measured and how
2244
+ // much was added; `gain` pinned at MIC_AGC_MAX_GAIN means the
2245
+ // cap, not the target, decided the level.
2246
+ crate::logfile::diagnostic(&format!(
2247
+ "[capture-health] mic auto-gain levelled: rms={rms:.6} gain={:.2} capped={}",
2248
+ self.gain,
2249
+ self.gain >= MIC_AGC_MAX_GAIN * 0.99
2250
+ ));
2251
+ }
2252
+ }
2253
+ self.gain
2254
+ }
2255
+ }
2256
+
2257
+ /// Zero-latency peak limiter over the summed mix.
2258
+ ///
2259
+ /// This replaces a flat 0.5x weight on each source, which bought clip safety
2260
+ /// by discarding 6 dB whether or not anything was near full scale. Lookahead
2261
+ /// would buy cleaner transients but costs an output delay, and the live
2262
+ /// uploader has already streamed the frames preceding it.
2263
+ struct PeakLimiter {
2264
+ gain: f32,
2265
+ release_coefficient: f32,
2266
+ }
2267
+
2268
+ impl PeakLimiter {
2269
+ fn new(sample_rate: i32) -> Self {
2270
+ Self {
2271
+ gain: 1.0,
2272
+ release_coefficient: one_pole_coefficient(MIX_LIMIT_RELEASE_SECONDS, sample_rate),
2273
+ }
2274
+ }
2275
+
2276
+ fn next_gain(&mut self, peak: f32) -> f32 {
2277
+ self.gain += (1.0 - self.gain) * self.release_coefficient;
2278
+ if peak > MIX_LIMIT_CEILING {
2279
+ self.gain = self.gain.min(MIX_LIMIT_CEILING / peak);
2280
+ }
2281
+ self.gain
2282
+ }
2283
+ }
2284
+
2285
+ /// The mixer's whole gain stage: auto-gain the mic, sum with system audio at
2286
+ /// unity, limit the result. Kept separate from `LiveAudioMixer` so the mix rule
2287
+ /// is reachable without constructing a `CMFormatDescription` — the reported bug
2288
+ /// was a wrong per-frame weight, so that arithmetic is what needs covering.
2289
+ struct MixGainStage {
2290
+ mic: MicAutoGain,
2291
+ limiter: PeakLimiter,
2292
+ }
2293
+
2294
+ impl MixGainStage {
2295
+ fn new(sample_rate: i32) -> Self {
2296
+ Self {
2297
+ mic: MicAutoGain::new(sample_rate),
2298
+ limiter: PeakLimiter::new(sample_rate),
2299
+ }
2300
+ }
2301
+
2302
+ /// One output frame. Neither source is pre-attenuated: system audio passes
2303
+ /// at unity and the mic is only ever boosted, with the limiter owning peak
2304
+ /// safety for the sum.
2305
+ fn frame(&mut self, system: (f32, f32), mic: (f32, f32)) -> (f32, f32) {
2306
+ let mic_gain = self.mic.next_gain(mic.0, mic.1);
2307
+ let left = system.0 + mic.0 * mic_gain;
2308
+ let right = system.1 + mic.1 * mic_gain;
2309
+ let limit = self.limiter.next_gain(left.abs().max(right.abs()));
2310
+ (
2311
+ (left * limit).clamp(-1.0, 1.0),
2312
+ (right * limit).clamp(-1.0, 1.0),
2313
+ )
2314
+ }
2315
+ }
2316
+
2148
2317
  #[derive(Clone, Copy, PartialEq, Eq)]
2149
2318
  /// Which capture stream a pushed PCM chunk came from.
2150
2319
  enum MixSource {
@@ -2224,63 +2393,6 @@ impl MixerTimeline {
2224
2393
  }
2225
2394
  }
2226
2395
 
2227
- #[cfg(test)]
2228
- mod mixer_timeline_tests {
2229
- use super::MixerTimeline;
2230
-
2231
- fn stereo_frames(values: &[f32]) -> Vec<f32> {
2232
- values.iter().flat_map(|value| [*value, *value]).collect()
2233
- }
2234
-
2235
- #[test]
2236
- fn trims_overlapping_frames_instead_of_extending_the_timeline() {
2237
- let mut timeline = MixerTimeline::new();
2238
- timeline.push_at(0, &stereo_frames(&[1.0, 2.0, 3.0, 4.0]), 0, 96_000);
2239
- timeline.push_at(3, &stereo_frames(&[4.0, 5.0, 6.0, 7.0]), 0, 96_000);
2240
-
2241
- assert_eq!(
2242
- timeline.samples,
2243
- stereo_frames(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])
2244
- );
2245
- assert_eq!(timeline.end_frame(), 7);
2246
- }
2247
-
2248
- #[test]
2249
- fn ignores_fully_duplicated_buffers() {
2250
- let mut timeline = MixerTimeline::new();
2251
- let frames = stereo_frames(&[1.0, 2.0, 3.0]);
2252
- timeline.push_at(0, &frames, 0, 96_000);
2253
- timeline.push_at(0, &frames, 0, 96_000);
2254
-
2255
- assert_eq!(timeline.samples, frames);
2256
- assert_eq!(timeline.end_frame(), 3);
2257
- }
2258
-
2259
- #[test]
2260
- fn trims_input_that_starts_behind_already_emitted_output() {
2261
- let mut timeline = MixerTimeline::new();
2262
- timeline.started = true;
2263
- timeline.base_frame = 10;
2264
- timeline.push_at(8, &stereo_frames(&[1.0, 2.0, 3.0, 4.0]), 10, 96_000);
2265
-
2266
- assert_eq!(timeline.samples, stereo_frames(&[3.0, 4.0]));
2267
- assert_eq!(timeline.end_frame(), 12);
2268
- }
2269
-
2270
- #[test]
2271
- fn preserves_forward_timestamp_gaps_as_silence() {
2272
- let mut timeline = MixerTimeline::new();
2273
- timeline.push_at(0, &stereo_frames(&[1.0, 2.0]), 0, 96_000);
2274
- timeline.push_at(5, &stereo_frames(&[6.0]), 0, 96_000);
2275
-
2276
- assert_eq!(
2277
- timeline.samples,
2278
- stereo_frames(&[1.0, 2.0, 0.0, 0.0, 0.0, 6.0])
2279
- );
2280
- assert_eq!(timeline.end_frame(), 6);
2281
- }
2282
- }
2283
-
2284
2396
  enum SourceBound {
2285
2397
  /// Started and recently fed; bounds output to `end_frame`.
2286
2398
  Active(i64),
@@ -2313,6 +2425,9 @@ struct LiveAudioMixer {
2313
2425
  out_pos: i64,
2314
2426
  system: MixerTimeline,
2315
2427
  mic: MixerTimeline,
2428
+ /// Gain stage, carried across chunks: emitted frames are already uploaded,
2429
+ /// so its state is the only memory the mix has.
2430
+ gain: MixGainStage,
2316
2431
  created_at: Instant,
2317
2432
  }
2318
2433
 
@@ -2360,6 +2475,7 @@ impl LiveAudioMixer {
2360
2475
  out_pos: 0,
2361
2476
  system: MixerTimeline::new(),
2362
2477
  mic: MixerTimeline::new(),
2478
+ gain: MixGainStage::new(sample_rate),
2363
2479
  created_at: Instant::now(),
2364
2480
  })
2365
2481
  }
@@ -2465,15 +2581,14 @@ impl LiveAudioMixer {
2465
2581
  }
2466
2582
 
2467
2583
  /// Emit mixed sample buffers covering `out_pos..safe_end` in
2468
- /// `MIX_CHUNK_FRAMES` chunks: average system + mic per frame, clamp, wrap
2469
- /// as LPCM `CMSampleBuffer`s with contiguous PTS.
2584
+ /// `MIX_CHUNK_FRAMES` chunks: auto-gain the mic, sum with system audio,
2585
+ /// limit, wrap as LPCM `CMSampleBuffer`s with contiguous PTS.
2470
2586
  ///
2471
- /// Each source is weighted at 0.5 before summing so that two full-scale
2472
- /// signals (which occurs when a USB audio interface with software monitoring
2473
- /// routes the mic back through system audio) can never exceed ±1.0 and
2474
- /// hard-clip. The standard SCK pipeline applies the same 0.5×L + 0.5×R
2475
- /// pan-downmix for the same reason; loudnorm restores the target loudness
2476
- /// in post-processing.
2587
+ /// Clip safety belongs to `PeakLimiter`, not to a fixed weight per source.
2588
+ /// Two full-scale signals do occur — a USB interface with software
2589
+ /// monitoring routes the mic back through system audio — but attenuating
2590
+ /// every frame for that case is what left live-uploaded clips ~6 dB below
2591
+ /// the rest, with no post-processing stage left to make it back up.
2477
2592
  fn drain_ready(
2478
2593
  &mut self,
2479
2594
  flush: bool,
@@ -2493,10 +2608,11 @@ impl LiveAudioMixer {
2493
2608
  let mut interleaved = vec![0.0_f32; n * 2];
2494
2609
  for f in 0..n {
2495
2610
  let frame = a + f as i64;
2496
- let (sl, sr) = self.system.sample_at(frame);
2497
- let (ml, mr) = self.mic.sample_at(frame);
2498
- interleaved[f * 2] = (sl * 0.5 + ml * 0.5).clamp(-1.0, 1.0);
2499
- interleaved[f * 2 + 1] = (sr * 0.5 + mr * 0.5).clamp(-1.0, 1.0);
2611
+ let (left, right) = self
2612
+ .gain
2613
+ .frame(self.system.sample_at(frame), self.mic.sample_at(frame));
2614
+ interleaved[f * 2] = left;
2615
+ interleaved[f * 2 + 1] = right;
2500
2616
  }
2501
2617
  emitted.push(self.build_sample_buffer(&interleaved, a)?);
2502
2618
  a = b;