@agent-native/core 0.144.2 → 0.145.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +1 -1
- package/corpus/templates/analytics/app/pages/DataSources.tsx +28 -4
- package/corpus/templates/analytics/server/jobs/dashboard-report.ts +7 -3
- package/corpus/templates/analytics/server/plugins/dashboard-report-jobs.ts +13 -3
- package/corpus/templates/calendar/app/components/calendar/GoogleConnectBanner.tsx +64 -23
- package/corpus/templates/calendar/app/hooks/use-google-auth.ts +69 -46
- package/corpus/templates/calendar/app/hooks/use-view-preferences.ts +35 -6
- package/corpus/templates/calendar/app/hooks/use-zoom-auth.ts +24 -3
- package/corpus/templates/clips/app/components/player/transcript-panel.tsx +45 -6
- package/corpus/templates/clips/app/components/recorder/storage-setup-card.tsx +38 -5
- package/corpus/templates/clips/changelog/2026-08-06-desktop-full-screen-recordings-now-normalize-microphone-leve.md +6 -0
- package/corpus/templates/clips/desktop/src/app.tsx +146 -45
- package/corpus/templates/clips/desktop/src-tauri/src/native_screen/custom_capture.rs +185 -69
- package/corpus/templates/clips/server/jobs/brain-export.ts +13 -6
- package/corpus/templates/clips/server/jobs/builder-media-compression.ts +10 -3
- package/corpus/templates/clips/server/jobs/meeting-reminders.ts +10 -3
- package/corpus/templates/clips/server/jobs/poll-calendars.ts +23 -21
- package/corpus/templates/clips/server/jobs/stale-meeting-sweeper.ts +10 -3
- package/corpus/templates/clips/server/jobs/transactional-emails.ts +10 -3
- package/corpus/templates/content/app/components/sidebar/NotionButton.tsx +1 -0
- package/corpus/templates/design/app/components/design/FusionAppBanner.tsx +15 -2
- package/corpus/templates/design/app/components/design/MultiScreenCanvas.tsx +47 -4
- package/corpus/templates/forms/server/plugins/agent-chat.ts +1 -1
- package/corpus/templates/mail/app/components/GoogleConnectBanner.tsx +82 -39
- package/corpus/templates/mail/app/components/layout/AppLayout.tsx +31 -9
- package/corpus/templates/mail/server/plugins/mail-jobs.ts +36 -17
- package/corpus/templates/slides/server/plugins/agent-chat.ts +1 -1
- package/dist/agent/production-agent.d.ts +6 -0
- package/dist/client/AssistantChat.js +73 -43
- package/dist/client/MultiTabAssistantChat.d.ts +3 -1
- package/dist/client/MultiTabAssistantChat.js +79 -92
- package/dist/client/browser-session-bridge.js +97 -54
- package/dist/client/db-admin/useAgentSync.js +34 -43
- package/dist/client/dynamic-suggestions.d.ts +3 -0
- package/dist/client/dynamic-suggestions.js +54 -52
- package/dist/client/hooks/index.d.ts +1 -0
- package/dist/client/hooks/index.js +1 -0
- package/dist/client/i18n.d.ts +7 -0
- package/dist/client/i18n.js +12 -3
- package/dist/client/integrations/useIntegrationStatus.d.ts +1 -1
- package/dist/client/integrations/useIntegrationStatus.js +9 -22
- package/dist/client/notifications/NotificationsBell.js +14 -9
- package/dist/client/progress/RunsTray.js +6 -5
- package/dist/client/settings/useBuilderStatus.js +95 -73
- package/dist/client/transcription/BuilderTranscriptionCta.js +27 -31
- package/dist/client/use-agent-engine-configured.js +21 -3
- package/dist/client/use-pausing-interval.d.ts +6 -0
- package/dist/client/use-pausing-interval.js +6 -0
- package/dist/client/use-poll-loop.d.ts +39 -0
- package/dist/client/use-poll-loop.js +73 -0
- package/dist/client/use-run-stuck-detection.js +41 -13
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/framework-tools.d.ts +126 -0
- package/dist/framework-tools.js +114 -0
- package/dist/integrations/google-docs-poller.js +26 -7
- package/dist/integrations/pending-tasks-retry-job.js +17 -23
- package/dist/integrations/plugin.js +16 -15
- package/dist/integrations/remote-push-delivery-job.js +16 -25
- package/dist/integrations/remote-retry-job.js +19 -23
- package/dist/mcp/screen-memory-stdio.d.ts +7 -7
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/oauth-tokens/google-refresh.js +12 -19
- package/dist/observability/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +9 -9
- package/dist/provider-api/actions/provider-api.d.ts +11 -11
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +3 -3
- package/dist/server/action-discovery.d.ts +27 -0
- package/dist/server/action-discovery.js +89 -0
- package/dist/server/agent-chat/action-filters-a2a.d.ts +11 -0
- package/dist/server/agent-chat/action-filters-a2a.js +17 -1
- package/dist/server/agent-chat/framework-prompts.d.ts +2 -0
- package/dist/server/agent-chat/plugin-options.d.ts +30 -0
- package/dist/server/agent-chat/prompt-resources.d.ts +4 -1
- package/dist/server/agent-chat/prompt-resources.js +8 -2
- package/dist/server/agent-chat-plugin.js +148 -100
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/interval-job.d.ts +25 -0
- package/dist/server/interval-job.js +19 -0
- package/dist/server/prompts/framework-core-compact.d.ts +5 -0
- package/dist/server/prompts/framework-core-compact.js +28 -12
- package/dist/server/prompts/framework-core.d.ts +5 -0
- package/dist/server/prompts/framework-core.js +33 -16
- package/dist/server/prompts/index.d.ts +1 -1
- package/dist/server/prompts/index.js +1 -1
- package/dist/server/prompts/shared-rules.d.ts +12 -4
- package/dist/server/prompts/shared-rules.js +14 -4
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/shared/index.d.ts +1 -0
- package/dist/shared/index.js +1 -0
- package/dist/shared/poll-engine.d.ts +49 -0
- package/dist/shared/poll-engine.js +148 -0
- package/dist/templates/default/.agents/skills/internationalization/SKILL.md +5 -0
- package/dist/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -3
- package/dist/templates/workspace-core/.agents/skills/internationalization/SKILL.md +5 -0
- package/dist/tracking/providers.js +10 -1
- package/docs/content/deployment.mdx +47 -13
- package/docs/content/key-concepts.mdx +13 -6
- package/package.json +2 -1
- package/src/templates/default/.agents/skills/internationalization/SKILL.md +5 -0
- package/src/templates/workspace-core/.agents/skills/extensions/SKILL.md +6 -3
- package/src/templates/workspace-core/.agents/skills/internationalization/SKILL.md +5 -0
|
@@ -61,6 +61,18 @@ export function StorageSetupCard({
|
|
|
61
61
|
const [err, setErr] = useState<string | null>(null);
|
|
62
62
|
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
|
|
63
63
|
const mountedRef = useRef(true);
|
|
64
|
+
const inFlightRef = useRef(false);
|
|
65
|
+
const visibilityHandlerRef = useRef<(() => void) | null>(null);
|
|
66
|
+
|
|
67
|
+
const stopVisibilityHandler = useCallback(() => {
|
|
68
|
+
if (visibilityHandlerRef.current) {
|
|
69
|
+
document.removeEventListener(
|
|
70
|
+
"visibilitychange",
|
|
71
|
+
visibilityHandlerRef.current,
|
|
72
|
+
);
|
|
73
|
+
visibilityHandlerRef.current = null;
|
|
74
|
+
}
|
|
75
|
+
}, []);
|
|
64
76
|
|
|
65
77
|
useEffect(() => {
|
|
66
78
|
mountedRef.current = true;
|
|
@@ -70,14 +82,17 @@ export function StorageSetupCard({
|
|
|
70
82
|
clearInterval(pollRef.current);
|
|
71
83
|
pollRef.current = null;
|
|
72
84
|
}
|
|
85
|
+
stopVisibilityHandler();
|
|
73
86
|
};
|
|
74
|
-
}, []);
|
|
87
|
+
}, [stopVisibilityHandler]);
|
|
75
88
|
|
|
76
89
|
const handleConnect = useCallback(() => {
|
|
77
90
|
if (pollRef.current) {
|
|
78
91
|
clearInterval(pollRef.current);
|
|
79
92
|
pollRef.current = null;
|
|
80
93
|
}
|
|
94
|
+
stopVisibilityHandler();
|
|
95
|
+
inFlightRef.current = false;
|
|
81
96
|
setConnecting(true);
|
|
82
97
|
setErr(null);
|
|
83
98
|
|
|
@@ -93,14 +108,23 @@ export function StorageSetupCard({
|
|
|
93
108
|
clearInterval(pollRef.current);
|
|
94
109
|
pollRef.current = null;
|
|
95
110
|
}
|
|
111
|
+
stopVisibilityHandler();
|
|
96
112
|
};
|
|
97
|
-
|
|
113
|
+
const tick = async () => {
|
|
114
|
+
if (document.hidden || inFlightRef.current) return;
|
|
115
|
+
inFlightRef.current = true;
|
|
116
|
+
const controller = new AbortController();
|
|
117
|
+
const abortTimer = setTimeout(
|
|
118
|
+
() => controller.abort(),
|
|
119
|
+
Math.max(10_000, 2000 * 4),
|
|
120
|
+
);
|
|
98
121
|
try {
|
|
99
122
|
const r = await fetch(
|
|
100
123
|
new URL(
|
|
101
124
|
agentNativePath("/_agent-native/builder/status"),
|
|
102
125
|
window.location.origin,
|
|
103
126
|
).toString(),
|
|
127
|
+
{ signal: controller.signal },
|
|
104
128
|
);
|
|
105
129
|
if (!r.ok) return;
|
|
106
130
|
const s = (await r.json()) as { configured: boolean };
|
|
@@ -119,10 +143,19 @@ export function StorageSetupCard({
|
|
|
119
143
|
setErr(t("storageSetup.builderTimeout"));
|
|
120
144
|
}
|
|
121
145
|
} catch {
|
|
122
|
-
// transient poll error
|
|
146
|
+
// coercion-ok: a transient poll error keeps the loop running; the
|
|
147
|
+
// timeoutMs bound above surfaces builderTimeout if it never succeeds.
|
|
148
|
+
} finally {
|
|
149
|
+
clearTimeout(abortTimer);
|
|
150
|
+
inFlightRef.current = false;
|
|
123
151
|
}
|
|
124
|
-
}
|
|
125
|
-
|
|
152
|
+
};
|
|
153
|
+
pollRef.current = setInterval(() => void tick(), 2000);
|
|
154
|
+
visibilityHandlerRef.current = () => {
|
|
155
|
+
if (!document.hidden) void tick();
|
|
156
|
+
};
|
|
157
|
+
document.addEventListener("visibilitychange", visibilityHandlerRef.current);
|
|
158
|
+
}, [onConfigured, connectSource, connectFlow, stopVisibilityHandler, t]);
|
|
126
159
|
|
|
127
160
|
return (
|
|
128
161
|
<div className="mx-auto flex w-full max-w-md flex-col gap-5 rounded-2xl border border-border bg-card p-6 shadow-lg">
|
|
@@ -427,6 +427,24 @@ function serverUrlForPendingUpload(
|
|
|
427
427
|
// already-connected user to the setup flow.
|
|
428
428
|
type VideoStorageProbe = "configured" | "missing" | "unknown";
|
|
429
429
|
|
|
430
|
+
// Poll cadence for the caller's re-check loop is 5s; bound each probe request
|
|
431
|
+
// well above that so a hung request can't wedge the poll's in-flight guard.
|
|
432
|
+
const VIDEO_STORAGE_PROBE_TIMEOUT_MS = Math.max(10_000, 5000 * 4);
|
|
433
|
+
|
|
434
|
+
async function fetchWithAbortTimeout(
|
|
435
|
+
url: string,
|
|
436
|
+
init: RequestInit,
|
|
437
|
+
timeoutMs: number,
|
|
438
|
+
): Promise<Response> {
|
|
439
|
+
const controller = new AbortController();
|
|
440
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
441
|
+
try {
|
|
442
|
+
return await fetch(url, { ...init, signal: controller.signal });
|
|
443
|
+
} finally {
|
|
444
|
+
clearTimeout(timer);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
|
|
430
448
|
async function hasConfiguredVideoStorage(
|
|
431
449
|
serverUrl: string,
|
|
432
450
|
): Promise<VideoStorageProbe> {
|
|
@@ -437,12 +455,13 @@ async function hasConfiguredVideoStorage(
|
|
|
437
455
|
let sawDefinitiveAnswer = false;
|
|
438
456
|
|
|
439
457
|
try {
|
|
440
|
-
const uploadStatus = await
|
|
458
|
+
const uploadStatus = await fetchWithAbortTimeout(
|
|
441
459
|
`${base}/_agent-native/file-upload/status`,
|
|
442
460
|
{
|
|
443
461
|
credentials: "include",
|
|
444
462
|
cache: "no-store",
|
|
445
463
|
},
|
|
464
|
+
VIDEO_STORAGE_PROBE_TIMEOUT_MS,
|
|
446
465
|
);
|
|
447
466
|
if (uploadStatus.ok) {
|
|
448
467
|
const body = (await uploadStatus.json().catch(() => null)) as {
|
|
@@ -458,10 +477,14 @@ async function hasConfiguredVideoStorage(
|
|
|
458
477
|
}
|
|
459
478
|
|
|
460
479
|
try {
|
|
461
|
-
const builderStatus = await
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
480
|
+
const builderStatus = await fetchWithAbortTimeout(
|
|
481
|
+
`${base}/_agent-native/builder/status`,
|
|
482
|
+
{
|
|
483
|
+
credentials: "include",
|
|
484
|
+
cache: "no-store",
|
|
485
|
+
},
|
|
486
|
+
VIDEO_STORAGE_PROBE_TIMEOUT_MS,
|
|
487
|
+
);
|
|
465
488
|
if (builderStatus.ok) {
|
|
466
489
|
const body = (await builderStatus.json().catch(() => null)) as {
|
|
467
490
|
configured?: boolean;
|
|
@@ -1143,10 +1166,23 @@ export function App() {
|
|
|
1143
1166
|
return;
|
|
1144
1167
|
}
|
|
1145
1168
|
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1169
|
+
let inFlight = false;
|
|
1170
|
+
const tick = () => {
|
|
1171
|
+
if (document.hidden || inFlight) return;
|
|
1172
|
+
inFlight = true;
|
|
1173
|
+
void refreshVideoStorageStatus().finally(() => {
|
|
1174
|
+
inFlight = false;
|
|
1175
|
+
});
|
|
1176
|
+
};
|
|
1177
|
+
const interval = window.setInterval(tick, 5000);
|
|
1178
|
+
const onVisibilityChange = () => {
|
|
1179
|
+
if (!document.hidden) tick();
|
|
1180
|
+
};
|
|
1181
|
+
document.addEventListener("visibilitychange", onVisibilityChange);
|
|
1182
|
+
return () => {
|
|
1183
|
+
window.clearInterval(interval);
|
|
1184
|
+
document.removeEventListener("visibilitychange", onVisibilityChange);
|
|
1185
|
+
};
|
|
1150
1186
|
}, [
|
|
1151
1187
|
authStatus,
|
|
1152
1188
|
localRecordingMode,
|
|
@@ -1630,22 +1666,45 @@ export function App() {
|
|
|
1630
1666
|
return;
|
|
1631
1667
|
}
|
|
1632
1668
|
let cancelled = false;
|
|
1669
|
+
let inFlight = false;
|
|
1633
1670
|
const poll = async () => {
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1671
|
+
if (document.hidden || inFlight) return;
|
|
1672
|
+
inFlight = true;
|
|
1673
|
+
const controller = new AbortController();
|
|
1674
|
+
const abortTimer = setTimeout(
|
|
1675
|
+
() => controller.abort(),
|
|
1676
|
+
Math.max(10_000, 3_000 * 4),
|
|
1638
1677
|
);
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1678
|
+
try {
|
|
1679
|
+
const result = await callClipsAction<{
|
|
1680
|
+
requests?: RewindExtensionRequest[];
|
|
1681
|
+
}>(
|
|
1682
|
+
"list-rewind-extension-requests",
|
|
1683
|
+
{},
|
|
1684
|
+
{ method: "GET", signal: controller.signal },
|
|
1685
|
+
)
|
|
1686
|
+
// coercion-ok: nothing to process this sweep either way; the next
|
|
1687
|
+
// tick re-reads the pending requests.
|
|
1688
|
+
.catch(() => null);
|
|
1689
|
+
if (cancelled) return;
|
|
1690
|
+
for (const request of result?.requests ?? []) {
|
|
1691
|
+
void processRewindExtension(request);
|
|
1692
|
+
}
|
|
1693
|
+
} finally {
|
|
1694
|
+
clearTimeout(abortTimer);
|
|
1695
|
+
inFlight = false;
|
|
1642
1696
|
}
|
|
1643
1697
|
};
|
|
1644
1698
|
void poll();
|
|
1645
1699
|
const timer = window.setInterval(() => void poll(), 3_000);
|
|
1700
|
+
const onVisibilityChange = () => {
|
|
1701
|
+
if (!document.hidden) void poll();
|
|
1702
|
+
};
|
|
1703
|
+
document.addEventListener("visibilitychange", onVisibilityChange);
|
|
1646
1704
|
return () => {
|
|
1647
1705
|
cancelled = true;
|
|
1648
1706
|
window.clearInterval(timer);
|
|
1707
|
+
document.removeEventListener("visibilitychange", onVisibilityChange);
|
|
1649
1708
|
};
|
|
1650
1709
|
}, [
|
|
1651
1710
|
authStatus,
|
|
@@ -1702,41 +1761,63 @@ export function App() {
|
|
|
1702
1761
|
useEffect(() => {
|
|
1703
1762
|
if (featureConfig?.screenMemory?.enabled !== true) return;
|
|
1704
1763
|
let cancelled = false;
|
|
1764
|
+
let inFlight = false;
|
|
1705
1765
|
const sweep = async () => {
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
await
|
|
1724
|
-
|
|
1725
|
-
|
|
1766
|
+
if (document.hidden || inFlight) return;
|
|
1767
|
+
inFlight = true;
|
|
1768
|
+
try {
|
|
1769
|
+
const due = await invoke<DueRewindAgentHandoff[]>(
|
|
1770
|
+
"screen_memory_due_agent_handoffs",
|
|
1771
|
+
)
|
|
1772
|
+
// coercion-ok: handoffs stay due in the local store, so an empty
|
|
1773
|
+
// sweep and a failed one both retry on the next tick.
|
|
1774
|
+
.catch(() => []);
|
|
1775
|
+
if (cancelled) return;
|
|
1776
|
+
for (const item of due) {
|
|
1777
|
+
try {
|
|
1778
|
+
const controller = new AbortController();
|
|
1779
|
+
const abortTimer = setTimeout(
|
|
1780
|
+
() => controller.abort(),
|
|
1781
|
+
Math.max(10_000, 60_000 * 4),
|
|
1782
|
+
);
|
|
1783
|
+
const cleanup = await callClipsAction<{
|
|
1784
|
+
deleted: boolean;
|
|
1785
|
+
reason: string;
|
|
1786
|
+
}>(
|
|
1787
|
+
"delete-agent-recording-if-unpromoted",
|
|
1788
|
+
{ id: item.recordingId },
|
|
1789
|
+
{ signal: controller.signal },
|
|
1790
|
+
).finally(() => clearTimeout(abortTimer));
|
|
1791
|
+
if (cleanup.deleted) {
|
|
1792
|
+
await invoke("screen_memory_mark_agent_handoff_deleted", {
|
|
1793
|
+
requestId: item.requestId,
|
|
1794
|
+
});
|
|
1795
|
+
} else if (cleanup.reason === "promoted") {
|
|
1796
|
+
await invoke("screen_memory_cancel_agent_handoff_cleanup", {
|
|
1797
|
+
requestId: item.requestId,
|
|
1798
|
+
});
|
|
1799
|
+
}
|
|
1800
|
+
} catch (error) {
|
|
1801
|
+
console.warn(
|
|
1802
|
+
"[clips-tray] agent-created Clip cleanup failed:",
|
|
1803
|
+
error,
|
|
1804
|
+
);
|
|
1726
1805
|
}
|
|
1727
|
-
} catch (error) {
|
|
1728
|
-
console.warn(
|
|
1729
|
-
"[clips-tray] agent-created Clip cleanup failed:",
|
|
1730
|
-
error,
|
|
1731
|
-
);
|
|
1732
1806
|
}
|
|
1807
|
+
} finally {
|
|
1808
|
+
inFlight = false;
|
|
1733
1809
|
}
|
|
1734
1810
|
};
|
|
1735
1811
|
void sweep();
|
|
1736
1812
|
const timer = window.setInterval(() => void sweep(), 60_000);
|
|
1813
|
+
const onVisibilityChange = () => {
|
|
1814
|
+
if (!document.hidden) void sweep();
|
|
1815
|
+
};
|
|
1816
|
+
document.addEventListener("visibilitychange", onVisibilityChange);
|
|
1737
1817
|
return () => {
|
|
1738
1818
|
cancelled = true;
|
|
1739
1819
|
window.clearInterval(timer);
|
|
1820
|
+
document.removeEventListener("visibilitychange", onVisibilityChange);
|
|
1740
1821
|
};
|
|
1741
1822
|
}, [callClipsAction, featureConfig?.screenMemory?.enabled]);
|
|
1742
1823
|
|
|
@@ -1967,11 +2048,18 @@ export function App() {
|
|
|
1967
2048
|
if (signInInflightRef.current) return;
|
|
1968
2049
|
signInInflightRef.current = true;
|
|
1969
2050
|
|
|
2051
|
+
let tickInFlight = false;
|
|
2052
|
+
let onVisibilityChange: (() => void) | null = null;
|
|
2053
|
+
|
|
1970
2054
|
function stopPolling() {
|
|
1971
2055
|
if (pollIntervalRef.current !== null) {
|
|
1972
2056
|
clearInterval(pollIntervalRef.current);
|
|
1973
2057
|
pollIntervalRef.current = null;
|
|
1974
2058
|
}
|
|
2059
|
+
if (onVisibilityChange) {
|
|
2060
|
+
document.removeEventListener("visibilitychange", onVisibilityChange);
|
|
2061
|
+
onVisibilityChange = null;
|
|
2062
|
+
}
|
|
1975
2063
|
}
|
|
1976
2064
|
|
|
1977
2065
|
function finishWithError(message: string) {
|
|
@@ -2000,11 +2088,16 @@ export function App() {
|
|
|
2000
2088
|
// Poll the exchange endpoint for the session token.
|
|
2001
2089
|
const start = Date.now();
|
|
2002
2090
|
const TIMEOUT_MS = 180_000; // 3 minutes
|
|
2003
|
-
|
|
2091
|
+
const POLL_ABORT_MS = Math.max(10_000, 1500 * 4);
|
|
2092
|
+
const tick = async () => {
|
|
2093
|
+
if (document.hidden || tickInFlight) return;
|
|
2094
|
+
tickInFlight = true;
|
|
2095
|
+
const controller = new AbortController();
|
|
2096
|
+
const abortTimer = setTimeout(() => controller.abort(), POLL_ABORT_MS);
|
|
2004
2097
|
try {
|
|
2005
2098
|
const xr = await fetch(
|
|
2006
2099
|
`${base}/_agent-native/auth/desktop-exchange?flow_id=${flowId}`,
|
|
2007
|
-
{ credentials: "include" },
|
|
2100
|
+
{ credentials: "include", signal: controller.signal },
|
|
2008
2101
|
);
|
|
2009
2102
|
if (!xr.ok) {
|
|
2010
2103
|
if (Date.now() - start > TIMEOUT_MS) {
|
|
@@ -2030,7 +2123,7 @@ export function App() {
|
|
|
2030
2123
|
// bearer token above is the reliable desktop auth path.
|
|
2031
2124
|
await fetch(
|
|
2032
2125
|
`${base}/_agent-native/auth/session?_session=${xd.token}`,
|
|
2033
|
-
{ credentials: "include" },
|
|
2126
|
+
{ credentials: "include", signal: controller.signal },
|
|
2034
2127
|
);
|
|
2035
2128
|
signInInflightRef.current = false;
|
|
2036
2129
|
setSignInPending(false);
|
|
@@ -2047,8 +2140,16 @@ export function App() {
|
|
|
2047
2140
|
if (Date.now() - start > TIMEOUT_MS) {
|
|
2048
2141
|
finishWithError("Google sign-in timed out. Please try again.");
|
|
2049
2142
|
}
|
|
2143
|
+
} finally {
|
|
2144
|
+
clearTimeout(abortTimer);
|
|
2145
|
+
tickInFlight = false;
|
|
2050
2146
|
}
|
|
2051
|
-
}
|
|
2147
|
+
};
|
|
2148
|
+
pollIntervalRef.current = setInterval(() => void tick(), 1500);
|
|
2149
|
+
onVisibilityChange = () => {
|
|
2150
|
+
if (!document.hidden) void tick();
|
|
2151
|
+
};
|
|
2152
|
+
document.addEventListener("visibilitychange", onVisibilityChange);
|
|
2052
2153
|
} catch (err) {
|
|
2053
2154
|
console.error("[clips-tray] signInExternal failed:", err);
|
|
2054
2155
|
signInInflightRef.current = false;
|
|
@@ -2145,6 +2145,175 @@ const MIX_SOURCE_GRACE: Duration = Duration::from_millis(2000);
|
|
|
2145
2145
|
const AUDIO_FORMAT_LPCM: u32 = 0x6C70_636D; // 'lpcm'
|
|
2146
2146
|
const AUDIO_FORMAT_FLAGS_FLOAT_PACKED: u32 = 1 | 8; // float + packed, interleaved, little-endian
|
|
2147
2147
|
|
|
2148
|
+
/// Speech RMS the mic auto-gain aims for (~-20 dBFS), and how far it may
|
|
2149
|
+
/// travel to get there. It never attenuates: a hot mic keeps its own level
|
|
2150
|
+
/// and the limiter below owns peak safety.
|
|
2151
|
+
const MIC_AGC_TARGET_RMS: f32 = 0.1;
|
|
2152
|
+
const MIC_AGC_MIN_GAIN: f32 = 1.0;
|
|
2153
|
+
const MIC_AGC_MAX_GAIN: f32 = 8.0; // +18 dB
|
|
2154
|
+
/// Below this RMS the mic is between words: hold the current gain rather than
|
|
2155
|
+
/// chasing the target, or every pause swells room tone to speech level.
|
|
2156
|
+
///
|
|
2157
|
+
/// Set at ~-60 dBFS, not the -50 dBFS that room tone alone would justify. A
|
|
2158
|
+
/// quiet built-in MacBook mic — the whole reason this stage exists — can carry
|
|
2159
|
+
/// speech at -45 dBFS RMS, and a floor set just under that would gate out the
|
|
2160
|
+
/// exact signal it is supposed to lift. Worst case here is room tone amplified
|
|
2161
|
+
/// to 0.001 × MIC_AGC_MAX_GAIN, still inaudible.
|
|
2162
|
+
const MIC_AGC_NOISE_FLOOR_RMS: f32 = 0.001;
|
|
2163
|
+
const MIC_AGC_ENVELOPE_SECONDS: f32 = 0.15;
|
|
2164
|
+
/// Asymmetric: back off quickly when the mic gets loud (the limiter should be
|
|
2165
|
+
/// a backstop, not the thing shaping the sound), return slowly so a boost
|
|
2166
|
+
/// doesn't audibly pump across sentences.
|
|
2167
|
+
const MIC_AGC_DUCK_SECONDS: f32 = 0.08;
|
|
2168
|
+
const MIC_AGC_BOOST_SECONDS: f32 = 1.5;
|
|
2169
|
+
/// Until the first speech has been levelled, converge at this rate instead of
|
|
2170
|
+
/// `MIC_AGC_BOOST_SECONDS`. Gain starts at unity, so a quiet mic needs to
|
|
2171
|
+
/// travel most of its range; at the steady-state rate that takes several
|
|
2172
|
+
/// seconds and the opening sentence audibly swells.
|
|
2173
|
+
const MIC_AGC_WARMUP_SECONDS: f32 = 0.2;
|
|
2174
|
+
/// How close to the target counts as levelled and ends the warmup.
|
|
2175
|
+
const MIC_AGC_CONVERGED_TOLERANCE: f32 = 0.05;
|
|
2176
|
+
/// -1.0 dBFS, near the true-peak ceiling the offline loudnorm chain targets.
|
|
2177
|
+
const MIX_LIMIT_CEILING: f32 = 0.891;
|
|
2178
|
+
const MIX_LIMIT_RELEASE_SECONDS: f32 = 0.15;
|
|
2179
|
+
|
|
2180
|
+
/// One-pole smoothing coefficient for a time constant at the mixer's rate.
|
|
2181
|
+
///
|
|
2182
|
+
/// `exp_m1`, not `1.0 - exp(x)`: at 48 kHz the exponent is ~1e-5, so the
|
|
2183
|
+
/// subtraction form cancels away most of the f32 mantissa and the longer time
|
|
2184
|
+
/// constants come out with about three significant digits.
|
|
2185
|
+
fn one_pole_coefficient(tau_seconds: f32, sample_rate: i32) -> f32 {
|
|
2186
|
+
if tau_seconds <= 0.0 || sample_rate <= 0 {
|
|
2187
|
+
return 1.0;
|
|
2188
|
+
}
|
|
2189
|
+
-(-1.0 / (tau_seconds * sample_rate as f32)).exp_m1()
|
|
2190
|
+
}
|
|
2191
|
+
|
|
2192
|
+
/// Causal mic auto-gain.
|
|
2193
|
+
///
|
|
2194
|
+
/// ScreenCaptureKit captures without AGC — unlike the browser path, which gets
|
|
2195
|
+
/// one from `autoGainControl` — so a MacBook mic lands far below the -16 LUFS
|
|
2196
|
+
/// the offline chain in `native_screen.rs` normalizes to. Live-uploaded clips
|
|
2197
|
+
/// never reach that chain: their bytes stream to the server while the writer
|
|
2198
|
+
/// is still producing them, so nothing downstream can re-read the file. Gain
|
|
2199
|
+
/// has to be decided here, from what has already been heard.
|
|
2200
|
+
struct MicAutoGain {
|
|
2201
|
+
mean_square: f32,
|
|
2202
|
+
gain: f32,
|
|
2203
|
+
/// False until the first speech has been levelled; see
|
|
2204
|
+
/// `MIC_AGC_WARMUP_SECONDS`.
|
|
2205
|
+
levelled: bool,
|
|
2206
|
+
envelope_coefficient: f32,
|
|
2207
|
+
warmup_coefficient: f32,
|
|
2208
|
+
duck_coefficient: f32,
|
|
2209
|
+
boost_coefficient: f32,
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
impl MicAutoGain {
|
|
2213
|
+
fn new(sample_rate: i32) -> Self {
|
|
2214
|
+
Self {
|
|
2215
|
+
mean_square: 0.0,
|
|
2216
|
+
gain: 1.0,
|
|
2217
|
+
levelled: false,
|
|
2218
|
+
envelope_coefficient: one_pole_coefficient(MIC_AGC_ENVELOPE_SECONDS, sample_rate),
|
|
2219
|
+
warmup_coefficient: one_pole_coefficient(MIC_AGC_WARMUP_SECONDS, sample_rate),
|
|
2220
|
+
duck_coefficient: one_pole_coefficient(MIC_AGC_DUCK_SECONDS, sample_rate),
|
|
2221
|
+
boost_coefficient: one_pole_coefficient(MIC_AGC_BOOST_SECONDS, sample_rate),
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
fn next_gain(&mut self, left: f32, right: f32) -> f32 {
|
|
2226
|
+
let peak = left.abs().max(right.abs());
|
|
2227
|
+
self.mean_square += (peak * peak - self.mean_square) * self.envelope_coefficient;
|
|
2228
|
+
let rms = self.mean_square.max(0.0).sqrt();
|
|
2229
|
+
if rms >= MIC_AGC_NOISE_FLOOR_RMS {
|
|
2230
|
+
let target = (MIC_AGC_TARGET_RMS / rms).clamp(MIC_AGC_MIN_GAIN, MIC_AGC_MAX_GAIN);
|
|
2231
|
+
let coefficient = if !self.levelled {
|
|
2232
|
+
self.warmup_coefficient
|
|
2233
|
+
} else if target < self.gain {
|
|
2234
|
+
self.duck_coefficient
|
|
2235
|
+
} else {
|
|
2236
|
+
self.boost_coefficient
|
|
2237
|
+
};
|
|
2238
|
+
self.gain += (target - self.gain) * coefficient;
|
|
2239
|
+
if !self.levelled && (self.gain - target).abs() <= target * MIC_AGC_CONVERGED_TOLERANCE
|
|
2240
|
+
{
|
|
2241
|
+
self.levelled = true;
|
|
2242
|
+
// Once per recording. A "clip is too quiet" report is only
|
|
2243
|
+
// diagnosable if the log says what the mic measured and how
|
|
2244
|
+
// much was added; `gain` pinned at MIC_AGC_MAX_GAIN means the
|
|
2245
|
+
// cap, not the target, decided the level.
|
|
2246
|
+
crate::logfile::diagnostic(&format!(
|
|
2247
|
+
"[capture-health] mic auto-gain levelled: rms={rms:.6} gain={:.2} capped={}",
|
|
2248
|
+
self.gain,
|
|
2249
|
+
self.gain >= MIC_AGC_MAX_GAIN * 0.99
|
|
2250
|
+
));
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
self.gain
|
|
2254
|
+
}
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
/// Zero-latency peak limiter over the summed mix.
|
|
2258
|
+
///
|
|
2259
|
+
/// This replaces a flat 0.5x weight on each source, which bought clip safety
|
|
2260
|
+
/// by discarding 6 dB whether or not anything was near full scale. Lookahead
|
|
2261
|
+
/// would buy cleaner transients but costs an output delay, and the live
|
|
2262
|
+
/// uploader has already streamed the frames preceding it.
|
|
2263
|
+
struct PeakLimiter {
|
|
2264
|
+
gain: f32,
|
|
2265
|
+
release_coefficient: f32,
|
|
2266
|
+
}
|
|
2267
|
+
|
|
2268
|
+
impl PeakLimiter {
|
|
2269
|
+
fn new(sample_rate: i32) -> Self {
|
|
2270
|
+
Self {
|
|
2271
|
+
gain: 1.0,
|
|
2272
|
+
release_coefficient: one_pole_coefficient(MIX_LIMIT_RELEASE_SECONDS, sample_rate),
|
|
2273
|
+
}
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
fn next_gain(&mut self, peak: f32) -> f32 {
|
|
2277
|
+
self.gain += (1.0 - self.gain) * self.release_coefficient;
|
|
2278
|
+
if peak > MIX_LIMIT_CEILING {
|
|
2279
|
+
self.gain = self.gain.min(MIX_LIMIT_CEILING / peak);
|
|
2280
|
+
}
|
|
2281
|
+
self.gain
|
|
2282
|
+
}
|
|
2283
|
+
}
|
|
2284
|
+
|
|
2285
|
+
/// The mixer's whole gain stage: auto-gain the mic, sum with system audio at
|
|
2286
|
+
/// unity, limit the result. Kept separate from `LiveAudioMixer` so the mix rule
|
|
2287
|
+
/// is reachable without constructing a `CMFormatDescription` — the reported bug
|
|
2288
|
+
/// was a wrong per-frame weight, so that arithmetic is what needs covering.
|
|
2289
|
+
struct MixGainStage {
|
|
2290
|
+
mic: MicAutoGain,
|
|
2291
|
+
limiter: PeakLimiter,
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
impl MixGainStage {
|
|
2295
|
+
fn new(sample_rate: i32) -> Self {
|
|
2296
|
+
Self {
|
|
2297
|
+
mic: MicAutoGain::new(sample_rate),
|
|
2298
|
+
limiter: PeakLimiter::new(sample_rate),
|
|
2299
|
+
}
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
/// One output frame. Neither source is pre-attenuated: system audio passes
|
|
2303
|
+
/// at unity and the mic is only ever boosted, with the limiter owning peak
|
|
2304
|
+
/// safety for the sum.
|
|
2305
|
+
fn frame(&mut self, system: (f32, f32), mic: (f32, f32)) -> (f32, f32) {
|
|
2306
|
+
let mic_gain = self.mic.next_gain(mic.0, mic.1);
|
|
2307
|
+
let left = system.0 + mic.0 * mic_gain;
|
|
2308
|
+
let right = system.1 + mic.1 * mic_gain;
|
|
2309
|
+
let limit = self.limiter.next_gain(left.abs().max(right.abs()));
|
|
2310
|
+
(
|
|
2311
|
+
(left * limit).clamp(-1.0, 1.0),
|
|
2312
|
+
(right * limit).clamp(-1.0, 1.0),
|
|
2313
|
+
)
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2316
|
+
|
|
2148
2317
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
|
2149
2318
|
/// Which capture stream a pushed PCM chunk came from.
|
|
2150
2319
|
enum MixSource {
|
|
@@ -2224,63 +2393,6 @@ impl MixerTimeline {
|
|
|
2224
2393
|
}
|
|
2225
2394
|
}
|
|
2226
2395
|
|
|
2227
|
-
#[cfg(test)]
|
|
2228
|
-
mod mixer_timeline_tests {
|
|
2229
|
-
use super::MixerTimeline;
|
|
2230
|
-
|
|
2231
|
-
fn stereo_frames(values: &[f32]) -> Vec<f32> {
|
|
2232
|
-
values.iter().flat_map(|value| [*value, *value]).collect()
|
|
2233
|
-
}
|
|
2234
|
-
|
|
2235
|
-
#[test]
|
|
2236
|
-
fn trims_overlapping_frames_instead_of_extending_the_timeline() {
|
|
2237
|
-
let mut timeline = MixerTimeline::new();
|
|
2238
|
-
timeline.push_at(0, &stereo_frames(&[1.0, 2.0, 3.0, 4.0]), 0, 96_000);
|
|
2239
|
-
timeline.push_at(3, &stereo_frames(&[4.0, 5.0, 6.0, 7.0]), 0, 96_000);
|
|
2240
|
-
|
|
2241
|
-
assert_eq!(
|
|
2242
|
-
timeline.samples,
|
|
2243
|
-
stereo_frames(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])
|
|
2244
|
-
);
|
|
2245
|
-
assert_eq!(timeline.end_frame(), 7);
|
|
2246
|
-
}
|
|
2247
|
-
|
|
2248
|
-
#[test]
|
|
2249
|
-
fn ignores_fully_duplicated_buffers() {
|
|
2250
|
-
let mut timeline = MixerTimeline::new();
|
|
2251
|
-
let frames = stereo_frames(&[1.0, 2.0, 3.0]);
|
|
2252
|
-
timeline.push_at(0, &frames, 0, 96_000);
|
|
2253
|
-
timeline.push_at(0, &frames, 0, 96_000);
|
|
2254
|
-
|
|
2255
|
-
assert_eq!(timeline.samples, frames);
|
|
2256
|
-
assert_eq!(timeline.end_frame(), 3);
|
|
2257
|
-
}
|
|
2258
|
-
|
|
2259
|
-
#[test]
|
|
2260
|
-
fn trims_input_that_starts_behind_already_emitted_output() {
|
|
2261
|
-
let mut timeline = MixerTimeline::new();
|
|
2262
|
-
timeline.started = true;
|
|
2263
|
-
timeline.base_frame = 10;
|
|
2264
|
-
timeline.push_at(8, &stereo_frames(&[1.0, 2.0, 3.0, 4.0]), 10, 96_000);
|
|
2265
|
-
|
|
2266
|
-
assert_eq!(timeline.samples, stereo_frames(&[3.0, 4.0]));
|
|
2267
|
-
assert_eq!(timeline.end_frame(), 12);
|
|
2268
|
-
}
|
|
2269
|
-
|
|
2270
|
-
#[test]
|
|
2271
|
-
fn preserves_forward_timestamp_gaps_as_silence() {
|
|
2272
|
-
let mut timeline = MixerTimeline::new();
|
|
2273
|
-
timeline.push_at(0, &stereo_frames(&[1.0, 2.0]), 0, 96_000);
|
|
2274
|
-
timeline.push_at(5, &stereo_frames(&[6.0]), 0, 96_000);
|
|
2275
|
-
|
|
2276
|
-
assert_eq!(
|
|
2277
|
-
timeline.samples,
|
|
2278
|
-
stereo_frames(&[1.0, 2.0, 0.0, 0.0, 0.0, 6.0])
|
|
2279
|
-
);
|
|
2280
|
-
assert_eq!(timeline.end_frame(), 6);
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
|
|
2284
2396
|
enum SourceBound {
|
|
2285
2397
|
/// Started and recently fed; bounds output to `end_frame`.
|
|
2286
2398
|
Active(i64),
|
|
@@ -2313,6 +2425,9 @@ struct LiveAudioMixer {
|
|
|
2313
2425
|
out_pos: i64,
|
|
2314
2426
|
system: MixerTimeline,
|
|
2315
2427
|
mic: MixerTimeline,
|
|
2428
|
+
/// Gain stage, carried across chunks: emitted frames are already uploaded,
|
|
2429
|
+
/// so its state is the only memory the mix has.
|
|
2430
|
+
gain: MixGainStage,
|
|
2316
2431
|
created_at: Instant,
|
|
2317
2432
|
}
|
|
2318
2433
|
|
|
@@ -2360,6 +2475,7 @@ impl LiveAudioMixer {
|
|
|
2360
2475
|
out_pos: 0,
|
|
2361
2476
|
system: MixerTimeline::new(),
|
|
2362
2477
|
mic: MixerTimeline::new(),
|
|
2478
|
+
gain: MixGainStage::new(sample_rate),
|
|
2363
2479
|
created_at: Instant::now(),
|
|
2364
2480
|
})
|
|
2365
2481
|
}
|
|
@@ -2465,15 +2581,14 @@ impl LiveAudioMixer {
|
|
|
2465
2581
|
}
|
|
2466
2582
|
|
|
2467
2583
|
/// Emit mixed sample buffers covering `out_pos..safe_end` in
|
|
2468
|
-
/// `MIX_CHUNK_FRAMES` chunks:
|
|
2469
|
-
/// as LPCM `CMSampleBuffer`s with contiguous PTS.
|
|
2584
|
+
/// `MIX_CHUNK_FRAMES` chunks: auto-gain the mic, sum with system audio,
|
|
2585
|
+
/// limit, wrap as LPCM `CMSampleBuffer`s with contiguous PTS.
|
|
2470
2586
|
///
|
|
2471
|
-
///
|
|
2472
|
-
/// signals
|
|
2473
|
-
/// routes the mic back through system audio
|
|
2474
|
-
///
|
|
2475
|
-
///
|
|
2476
|
-
/// in post-processing.
|
|
2587
|
+
/// Clip safety belongs to `PeakLimiter`, not to a fixed weight per source.
|
|
2588
|
+
/// Two full-scale signals do occur — a USB interface with software
|
|
2589
|
+
/// monitoring routes the mic back through system audio — but attenuating
|
|
2590
|
+
/// every frame for that case is what left live-uploaded clips ~6 dB below
|
|
2591
|
+
/// the rest, with no post-processing stage left to make it back up.
|
|
2477
2592
|
fn drain_ready(
|
|
2478
2593
|
&mut self,
|
|
2479
2594
|
flush: bool,
|
|
@@ -2493,10 +2608,11 @@ impl LiveAudioMixer {
|
|
|
2493
2608
|
let mut interleaved = vec![0.0_f32; n * 2];
|
|
2494
2609
|
for f in 0..n {
|
|
2495
2610
|
let frame = a + f as i64;
|
|
2496
|
-
let (
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
interleaved[f * 2
|
|
2611
|
+
let (left, right) = self
|
|
2612
|
+
.gain
|
|
2613
|
+
.frame(self.system.sample_at(frame), self.mic.sample_at(frame));
|
|
2614
|
+
interleaved[f * 2] = left;
|
|
2615
|
+
interleaved[f * 2 + 1] = right;
|
|
2500
2616
|
}
|
|
2501
2617
|
emitted.push(self.build_sample_buffer(&interleaved, a)?);
|
|
2502
2618
|
a = b;
|