@vellumai/assistant 0.11.5 → 0.11.6-staging.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -1
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/ces-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/gateway-client/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/node_modules/@vellumai/gateway-client/src/gateway-ipc-contracts.ts +70 -0
- package/node_modules/@vellumai/gateway-client/src/inbound-contract.ts +16 -1
- package/node_modules/@vellumai/gateway-client/src/index.ts +6 -2
- package/node_modules/@vellumai/gateway-client/src/outbound-contract.ts +121 -61
- package/node_modules/@vellumai/service-contracts/src/__tests__/ingress.test.ts +118 -0
- package/node_modules/@vellumai/service-contracts/src/ingress.ts +103 -0
- package/openapi.yaml +421 -15
- package/package.json +1 -1
- package/scripts/sync-web-search-catalog.ts +6 -0
- package/src/__tests__/app-pin-store.test.ts +149 -0
- package/src/__tests__/channel-availability-routes.test.ts +23 -1
- package/src/__tests__/channel-readiness-discord.test.ts +231 -0
- package/src/__tests__/channel-readiness-service.test.ts +126 -0
- package/src/__tests__/channel-readiness-slack-remote.test.ts +141 -0
- package/src/__tests__/channel-reply-delivery.test.ts +4 -4
- package/src/__tests__/client-os-metadata-persistence.test.ts +23 -10
- package/src/__tests__/conversation-delete-watch-timeline.test.ts +231 -0
- package/src/__tests__/conversation-error.test.ts +17 -0
- package/src/__tests__/conversation-seed-composer.test.ts +8 -0
- package/src/__tests__/conversation-slash-commands.test.ts +8 -0
- package/src/__tests__/disk-pressure-policy.test.ts +6 -0
- package/src/__tests__/gemini-provider.test.ts +138 -0
- package/src/__tests__/history-repair.test.ts +105 -3
- package/src/__tests__/identity-routes.test.ts +1 -0
- package/src/__tests__/llm-catalog-parity.test.ts +45 -0
- package/src/__tests__/migration-import-from-path.test.ts +349 -0
- package/src/__tests__/notification-telegram-adapter.test.ts +102 -0
- package/src/__tests__/oauth-commands-routes.test.ts +89 -0
- package/src/__tests__/oauth-provider-profiles.test.ts +7 -6
- package/src/__tests__/openai-provider.test.ts +18 -0
- package/src/__tests__/openai-responses-provider.test.ts +18 -0
- package/src/__tests__/platform-callback-registration.test.ts +184 -0
- package/src/__tests__/plugin-api-store-credential.test.ts +71 -3
- package/src/__tests__/pricing.test.ts +2 -2
- package/src/__tests__/public-ingress-urls.test.ts +36 -0
- package/src/__tests__/resolve-trust-class.test.ts +0 -48
- package/src/__tests__/sanitize-config-for-transfer.test.ts +28 -0
- package/src/__tests__/secret-routes-platform-proxy.test.ts +49 -0
- package/src/__tests__/settings-routes.test.ts +85 -3
- package/src/__tests__/web-search-catalog-parity.test.ts +8 -0
- package/src/agent/history-repair/history-repair.ts +45 -14
- package/src/agent/loop.ts +4 -1
- package/src/api/constants/profile-config-validation.ts +60 -0
- package/src/api/events/tool-result.ts +6 -1
- package/src/api/events/watch-retro-completed.ts +52 -0
- package/src/api/index.ts +11 -0
- package/src/apps/app-pin-reconciler.ts +92 -0
- package/src/apps/app-pin-store.ts +125 -0
- package/src/channels/gateway-channel-socket-health.ts +32 -0
- package/src/channels/gateway-discord-admission.ts +32 -0
- package/src/channels/types.ts +20 -0
- package/src/cli/commands/__tests__/conversations-slack.test.ts +1 -1
- package/src/cli/commands/__tests__/inference-profiles.test.ts +16 -4
- package/src/cli/commands/__tests__/inference-providers.test.ts +67 -2
- package/src/cli/commands/channels/__tests__/channels.test.ts +85 -0
- package/src/cli/commands/channels/index.ts +45 -31
- package/src/cli/commands/inference-profiles.ts +56 -3
- package/src/cli/commands/inference-providers.ts +28 -2
- package/src/cli/commands/oauth/index.help.ts +7 -1
- package/src/cli/commands/oauth/request.test.ts +290 -0
- package/src/cli/commands/oauth/request.ts +57 -41
- package/src/cli/lib/bundled-marketplace.json +14 -1
- package/src/cli/lib/open-browser.test.ts +67 -0
- package/src/cli/lib/open-browser.ts +24 -5
- package/src/config/__tests__/profile-materialization.test.ts +26 -0
- package/src/config/bundled-skills/phone-calls/references/TROUBLESHOOTING.md +6 -0
- package/src/config/bundled-skills/schedule/SKILL.md +1 -1
- package/src/config/feature-flag-registry.json +17 -1
- package/src/config/profile-materialization.ts +29 -0
- package/src/config/sanitize-for-transfer.ts +16 -0
- package/src/config/schemas/llm.ts +7 -0
- package/src/config/schemas/services.ts +6 -0
- package/src/context/outbound-sanitize.ts +6 -0
- package/src/daemon/__tests__/lifecycle-watch-timeline-sweep.test.ts +98 -0
- package/src/daemon/conversation-error.ts +24 -2
- package/src/daemon/conversation-slash.ts +6 -15
- package/src/daemon/daemon-control.ts +1 -0
- package/src/daemon/disk-pressure-policy.ts +7 -1
- package/src/daemon/handlers/__tests__/config-ingress-tunnel-records.test.ts +208 -0
- package/src/daemon/handlers/config-ingress.ts +115 -5
- package/src/daemon/lifecycle.ts +26 -0
- package/src/daemon/message-types/web-activity.ts +3 -2
- package/src/daemon/trust-context.ts +0 -37
- package/src/inbound/__tests__/tunnel-probe.test.ts +448 -0
- package/src/inbound/platform-callback-registration.ts +28 -2
- package/src/inbound/public-ingress-urls.ts +12 -0
- package/src/inbound/tunnel-probe.ts +261 -0
- package/src/live-voice/__tests__/live-voice-connection.test.ts +25 -0
- package/src/live-voice/__tests__/live-voice-flux-turn-end.test.ts +8 -2
- package/src/live-voice/__tests__/live-voice-session-manager.test.ts +212 -10
- package/src/live-voice/__tests__/live-voice-session-telemetry.test.ts +5 -2
- package/src/live-voice/live-voice-connection.ts +46 -8
- package/src/live-voice/live-voice-manager.ts +25 -0
- package/src/live-voice/live-voice-session-manager.ts +318 -2
- package/src/live-voice/live-voice-session.ts +52 -2
- package/src/messaging/providers/__tests__/transport-dispatch.test.ts +126 -68
- package/src/messaging/providers/channel-transport.ts +64 -47
- package/src/messaging/providers/discord/send.test.ts +46 -1
- package/src/messaging/providers/discord/send.ts +51 -0
- package/src/messaging/providers/discord/transport.ts +26 -3
- package/src/messaging/providers/index.ts +22 -47
- package/src/messaging/providers/slack/send.test.ts +83 -26
- package/src/messaging/providers/slack/send.ts +120 -51
- package/src/messaging/providers/slack/stream-tasks.test.ts +26 -0
- package/src/messaging/providers/slack/stream-tasks.ts +39 -0
- package/src/messaging/providers/slack/transport.ts +24 -22
- package/src/messaging/providers/telegram-bot/send.test.ts +109 -12
- package/src/messaging/providers/telegram-bot/send.ts +43 -0
- package/src/messaging/providers/telegram-bot/transport.ts +25 -8
- package/src/notifications/__tests__/assistant-reply-producer.test.ts +30 -7
- package/src/notifications/adapters/telegram.ts +48 -1
- package/src/notifications/assistant-reply-producer.ts +7 -7
- package/src/notifications/conversation-seed-composer.ts +7 -2
- package/src/oauth/byo-connection.test.ts +63 -0
- package/src/oauth/byo-connection.ts +16 -15
- package/src/oauth/connection.test.ts +111 -0
- package/src/oauth/connection.ts +142 -1
- package/src/oauth/platform-connection.test.ts +34 -0
- package/src/oauth/platform-connection.ts +28 -5
- package/src/oauth/seed-providers.ts +15 -1
- package/src/permissions/types.ts +3 -1
- package/src/persistence/conversation-crud.ts +46 -0
- package/src/persistence/conversation-types.ts +11 -9
- package/src/persistence/db-async-query.ts +2 -1
- package/src/persistence/db-maintenance.ts +15 -0
- package/src/persistence/embeddings/qdrant-manager.ts +1 -0
- package/src/persistence/migrations/367-create-watch-timeline-entries.ts +46 -0
- package/src/persistence/migrations/368-watch-timeline-screenshot-blob.ts +33 -0
- package/src/persistence/migrations/369-create-app-pins.ts +37 -0
- package/src/persistence/migrations/__tests__/367-create-watch-timeline-entries.test.ts +98 -0
- package/src/persistence/migrations/__tests__/368-watch-timeline-screenshot-blob.test.ts +98 -0
- package/src/persistence/schema/index.ts +1 -0
- package/src/persistence/schema/infrastructure.ts +17 -0
- package/src/persistence/schema/watch.ts +29 -0
- package/src/persistence/steps.ts +6 -0
- package/src/plugins/mtime-cache.ts +11 -0
- package/src/providers/__tests__/retry-network-error.test.ts +84 -0
- package/src/providers/connection-resolution.ts +23 -1
- package/src/providers/content-blocks.ts +9 -0
- package/src/providers/fetch-provider-catalog.ts +19 -0
- package/src/providers/gemini/client.ts +13 -5
- package/src/providers/inference/__tests__/endpoint-probe.test.ts +92 -0
- package/src/providers/inference/__tests__/profile-config-validation.test.ts +39 -0
- package/src/providers/inference/__tests__/profile-probe-classify.test.ts +66 -0
- package/src/providers/inference/adapter-factory.ts +0 -9
- package/src/providers/inference/credential-rotation.ts +61 -0
- package/src/providers/inference/endpoint-probe.ts +115 -0
- package/src/providers/inference/profile-probe.ts +256 -0
- package/src/providers/model-catalog.ts +170 -125
- package/src/providers/openai/__tests__/api-error-normalization.test.ts +17 -1
- package/src/providers/openai/__tests__/chat-completions-provider-reasoning.test.ts +42 -60
- package/src/providers/openai/__tests__/connection-error-wrap.test.ts +44 -0
- package/src/providers/openai/__tests__/orphan-tool-result-guard.test.ts +34 -2
- package/src/providers/openai/api-error-normalization.ts +16 -2
- package/src/providers/openai/chat-completions-provider.ts +75 -29
- package/src/providers/openai/responses-provider.ts +5 -2
- package/src/providers/openrouter/client.ts +0 -1
- package/src/providers/provider-send-message.ts +11 -0
- package/src/providers/retry.ts +6 -0
- package/src/providers/search-provider-catalog.ts +20 -0
- package/src/providers/vercel-ai-gateway/client.ts +0 -1
- package/src/runtime/AGENTS.md +1 -0
- package/src/runtime/__tests__/desktop-presence.test.ts +27 -4
- package/src/runtime/__tests__/host-observe.test.ts +302 -0
- package/src/runtime/channel-readiness-service.ts +214 -14
- package/src/runtime/channel-readiness-types.ts +49 -2
- package/src/runtime/channel-reply-delivery.ts +2 -2
- package/src/runtime/desktop-presence.ts +24 -21
- package/src/runtime/host-observe.ts +246 -0
- package/src/runtime/http-server.ts +181 -1
- package/src/runtime/migrations/__tests__/staged-import-path.test.ts +104 -0
- package/src/runtime/migrations/staged-import-path.ts +116 -0
- package/src/runtime/routes/__tests__/app-pin-routes.test.ts +383 -0
- package/src/runtime/routes/__tests__/conversation-query-routes.test.ts +80 -0
- package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts +118 -0
- package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts +20 -0
- package/src/runtime/routes/__tests__/ingress-status-routes.test.ts +508 -0
- package/src/runtime/routes/__tests__/plugins-routes.test.ts +35 -56
- package/src/runtime/routes/__tests__/watch-routes-guardian-cache.test.ts +139 -0
- package/src/runtime/routes/__tests__/watch-routes.test.ts +598 -0
- package/src/runtime/routes/app-management-routes.ts +140 -29
- package/src/runtime/routes/channel-availability-routes.ts +1 -0
- package/src/runtime/routes/channel-readiness-routes.ts +14 -2
- package/src/runtime/routes/conversation-query-routes.ts +10 -0
- package/src/runtime/routes/guardian-approval-interception.ts +24 -33
- package/src/runtime/routes/host-cu-routes.ts +18 -0
- package/src/runtime/routes/identity-routes.ts +2 -0
- package/src/runtime/routes/inbound-message-handler.ts +10 -7
- package/src/runtime/routes/inbound-stages/background-dispatch.test.ts +166 -308
- package/src/runtime/routes/inbound-stages/background-dispatch.ts +158 -335
- package/src/runtime/routes/index.ts +2 -0
- package/src/runtime/routes/inference-profiles-routes.ts +232 -31
- package/src/runtime/routes/inference-provider-connection-routes.ts +24 -4
- package/src/runtime/routes/ingress-status-routes.ts +180 -0
- package/src/runtime/routes/live-voice-routes.test.ts +40 -1
- package/src/runtime/routes/live-voice-routes.ts +34 -0
- package/src/runtime/routes/migration-routes.ts +218 -10
- package/src/runtime/routes/oauth-commands-routes.ts +23 -16
- package/src/runtime/routes/plugins-routes.ts +12 -28
- package/src/runtime/routes/question-routes.ts +6 -0
- package/src/runtime/routes/secret-routes.ts +7 -27
- package/src/runtime/routes/settings-routes.ts +9 -6
- package/src/runtime/routes/watch-routes.ts +807 -0
- package/src/runtime/slack-reply-session.test.ts +230 -121
- package/src/runtime/slack-reply-session.ts +113 -81
- package/src/runtime/{slack-task-progress.test.ts → task-progress.test.ts} +1 -28
- package/src/runtime/{slack-task-progress.ts → task-progress.ts} +30 -51
- package/src/security/__tests__/untrusted-content.test.ts +42 -0
- package/src/security/untrusted-content.ts +28 -9
- package/src/telemetry/__tests__/live-voice-funnel.test.ts +108 -0
- package/src/telemetry/live-voice-funnel.ts +75 -8
- package/src/tools/credentials/store.ts +18 -6
- package/src/tools/network/__tests__/firecrawl-compat.test.ts +77 -0
- package/src/tools/network/__tests__/web-fetch-fastcrw.test.ts +169 -0
- package/src/tools/network/__tests__/web-search.test.ts +97 -2
- package/src/tools/network/firecrawl-compat.ts +90 -0
- package/src/tools/network/web-fetch.ts +142 -62
- package/src/tools/network/web-search.ts +141 -55
- package/src/tools/types.ts +2 -1
- package/src/util/oauth-request-body.test.ts +74 -0
- package/src/util/oauth-request-body.ts +60 -0
- package/src/util/worker-process.ts +1 -0
- package/src/watch/__tests__/watch-retro.test.ts +665 -0
- package/src/watch/__tests__/watch-session-manager.test.ts +566 -0
- package/src/watch/__tests__/watch-timeline.test.ts +670 -0
- package/src/watch/watch-retro.ts +480 -0
- package/src/watch/watch-session-manager.ts +575 -0
- package/src/watch/watch-timeline.ts +848 -0
|
@@ -4,8 +4,46 @@ import type { ChannelId } from "../channels/types.js";
|
|
|
4
4
|
|
|
5
5
|
export type { ChannelId };
|
|
6
6
|
|
|
7
|
-
/**
|
|
8
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Setup progress for a channel: not_configured → incomplete → ready.
|
|
9
|
+
*
|
|
10
|
+
* Progress only. A channel that is fully configured is `ready` here even when
|
|
11
|
+
* it is not currently working, because "have I finished setting this up" and
|
|
12
|
+
* "is it working right now" are different questions with different answers and
|
|
13
|
+
* different remedies. Operational state is {@link ChannelHealth}.
|
|
14
|
+
*/
|
|
15
|
+
export const SETUP_STATUSES = [
|
|
16
|
+
"not_configured",
|
|
17
|
+
"incomplete",
|
|
18
|
+
"ready",
|
|
19
|
+
] as const;
|
|
20
|
+
export type SetupStatus = (typeof SETUP_STATUSES)[number];
|
|
21
|
+
|
|
22
|
+
/** What a check establishes. */
|
|
23
|
+
export const CHECK_KINDS = [
|
|
24
|
+
/** Whether the channel is configured: credentials, ingress, policy. */
|
|
25
|
+
"configuration",
|
|
26
|
+
/** Whether the channel is currently working: is anything arriving. */
|
|
27
|
+
"operational",
|
|
28
|
+
] as const;
|
|
29
|
+
export type CheckKind = (typeof CHECK_KINDS)[number];
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Whether a configured channel is currently working.
|
|
33
|
+
*
|
|
34
|
+
* Absent when a channel measures no operational checks at all, which is not
|
|
35
|
+
* the same as failing to establish them: nothing was asked, so nothing is
|
|
36
|
+
* claimed either way, and readiness is decided on configuration alone.
|
|
37
|
+
*/
|
|
38
|
+
export const CHANNEL_HEALTHS = [
|
|
39
|
+
/** Every operational check confirmed the channel is working. */
|
|
40
|
+
"ok",
|
|
41
|
+
/** An operational check found a fault. */
|
|
42
|
+
"failing",
|
|
43
|
+
/** Operational checks ran but established nothing either way. */
|
|
44
|
+
"unknown",
|
|
45
|
+
] as const;
|
|
46
|
+
export type ChannelHealth = (typeof CHANNEL_HEALTHS)[number];
|
|
9
47
|
|
|
10
48
|
/** Result of a single readiness check (local or remote). */
|
|
11
49
|
export interface ReadinessCheckResult {
|
|
@@ -30,13 +68,22 @@ export interface ReadinessCheckResult {
|
|
|
30
68
|
* require this to be absent or false.
|
|
31
69
|
*/
|
|
32
70
|
indeterminate?: boolean;
|
|
71
|
+
/**
|
|
72
|
+
* Which question this check answers. Defaults to `configuration`, so a
|
|
73
|
+
* check that does not say is treated as setup evidence rather than as a
|
|
74
|
+
* claim about whether the channel is working.
|
|
75
|
+
*/
|
|
76
|
+
kind?: CheckKind;
|
|
33
77
|
}
|
|
34
78
|
|
|
35
79
|
/** Point-in-time snapshot of a channel's readiness state. */
|
|
36
80
|
export interface ChannelReadinessSnapshot {
|
|
37
81
|
channel: ChannelId;
|
|
82
|
+
/** Configured and confirmed working: `setupStatus === "ready"` and health is not in doubt. */
|
|
38
83
|
ready: boolean;
|
|
39
84
|
setupStatus: SetupStatus;
|
|
85
|
+
/** Absent when the channel measures no operational checks. */
|
|
86
|
+
health?: ChannelHealth;
|
|
40
87
|
checkedAt: number;
|
|
41
88
|
stale: boolean;
|
|
42
89
|
reasons: Array<{ code: string; text: string }>;
|
|
@@ -217,7 +217,7 @@ export async function deliverRenderedReplyViaCallback(
|
|
|
217
217
|
chatId,
|
|
218
218
|
messageId: editTarget,
|
|
219
219
|
text: segmentText,
|
|
220
|
-
|
|
220
|
+
renderRichly: true,
|
|
221
221
|
});
|
|
222
222
|
// An edit replaces the text of one message. Attachments are always new
|
|
223
223
|
// messages, so they still have to be posted alongside it.
|
|
@@ -245,7 +245,7 @@ export async function deliverRenderedReplyViaCallback(
|
|
|
245
245
|
result = await deliverChannelReply(callbackUrl, {
|
|
246
246
|
chatId,
|
|
247
247
|
text: segmentText,
|
|
248
|
-
|
|
248
|
+
renderRichly: true,
|
|
249
249
|
attachments: segmentAttachments,
|
|
250
250
|
assistantId,
|
|
251
251
|
audience,
|
|
@@ -2,12 +2,12 @@
|
|
|
2
2
|
* Desktop presence policy.
|
|
3
3
|
*
|
|
4
4
|
* Answers exactly one question for notification producers: is the user
|
|
5
|
-
* demonstrably at their
|
|
5
|
+
* demonstrably at their desktop right now? Producers use the answer to skip a
|
|
6
6
|
* push that the user would see on screen anyway.
|
|
7
7
|
*
|
|
8
8
|
* Fails open by design. Presence is in-memory, best-effort, and reported by
|
|
9
9
|
* a client that can drop off at any time, so anything short of a fresh
|
|
10
|
-
* `active` report from a
|
|
10
|
+
* `active` report from a desktop client answers `false` and the push goes
|
|
11
11
|
* out. A missed notification is strictly worse than a redundant one.
|
|
12
12
|
*/
|
|
13
13
|
|
|
@@ -15,16 +15,17 @@ import { getLogger } from "../util/logger.js";
|
|
|
15
15
|
import { assistantEventHub } from "./assistant-event-hub.js";
|
|
16
16
|
|
|
17
17
|
const log = getLogger("desktop-presence");
|
|
18
|
+
const DESKTOP_INTERFACES = ["macos", "windows"] as const;
|
|
18
19
|
|
|
19
20
|
/**
|
|
20
|
-
* Bounds how long suppression can outlive the
|
|
21
|
+
* Bounds how long suppression can outlive the desktop sleeping or dropping
|
|
21
22
|
* off, while still tolerating two dropped reports at the 30s report interval.
|
|
22
23
|
*/
|
|
23
24
|
export const PRESENCE_STALE_AFTER_MS = 90_000;
|
|
24
25
|
|
|
25
26
|
export interface DesktopAttendanceOptions {
|
|
26
27
|
/**
|
|
27
|
-
* Only count
|
|
28
|
+
* Only count desktop clients whose verified actor principal matches this id.
|
|
28
29
|
* A client that connected without a principal (legacy or service token)
|
|
29
30
|
* never matches a supplied id.
|
|
30
31
|
*/
|
|
@@ -34,13 +35,13 @@ export interface DesktopAttendanceOptions {
|
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
/**
|
|
37
|
-
* Whether some
|
|
38
|
-
* Stale, absent, non-
|
|
38
|
+
* Whether some desktop client has reported `active` recently enough to trust.
|
|
39
|
+
* Stale, absent, non-desktop, and error reads all answer `false`.
|
|
39
40
|
*
|
|
40
41
|
* Callers whose notification targets one recipient (guardian-scoped or
|
|
41
42
|
* otherwise per-recipient pushes) must pass that recipient's
|
|
42
|
-
* `actorPrincipalId`, or another user's attended
|
|
43
|
-
* Omitting it treats any attended
|
|
43
|
+
* `actorPrincipalId`, or another user's attended desktop suppresses the push.
|
|
44
|
+
* Omitting it treats any attended desktop client as attendance, which only
|
|
44
45
|
* suits notifications with no single recipient.
|
|
45
46
|
*/
|
|
46
47
|
export function isDesktopAttended(
|
|
@@ -48,19 +49,21 @@ export function isDesktopAttended(
|
|
|
48
49
|
): boolean {
|
|
49
50
|
const { actorPrincipalId, now = new Date() } = options;
|
|
50
51
|
try {
|
|
51
|
-
return
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
52
|
+
return DESKTOP_INTERFACES.some((interfaceId) =>
|
|
53
|
+
assistantEventHub.listClientsByInterface(interfaceId).some((client) => {
|
|
54
|
+
if (
|
|
55
|
+
actorPrincipalId !== undefined &&
|
|
56
|
+
client.actorPrincipalId !== actorPrincipalId
|
|
57
|
+
) {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
return (
|
|
61
|
+
client.presence?.state === "active" &&
|
|
62
|
+
now.getTime() - client.presence.reportedAt.getTime() <=
|
|
63
|
+
PRESENCE_STALE_AFTER_MS
|
|
64
|
+
);
|
|
65
|
+
}),
|
|
66
|
+
);
|
|
64
67
|
} catch (err) {
|
|
65
68
|
// Returning false sends the push, which is the safe direction.
|
|
66
69
|
log.warn({ err }, "desktop presence read failed; treating as unattended");
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Screen observation raised by the daemon outside any conversation or tool
|
|
3
|
+
* call.
|
|
4
|
+
*
|
|
5
|
+
* `computer_use_observe` normally reaches the desktop client through
|
|
6
|
+
* `HostCuProxy`, which only exists inside an agent turn. This helper drives the
|
|
7
|
+
* `host_cu` wire protocol directly for callers that have no turn to hang the
|
|
8
|
+
* request off: it mints a requestId, registers a pending interaction,
|
|
9
|
+
* broadcasts a `host_cu_request` envelope to a single client, and awaits that
|
|
10
|
+
* client's POST to `/v1/host-cu-result`.
|
|
11
|
+
*
|
|
12
|
+
* Every request is bound to the actor principal that initiated it, the same
|
|
13
|
+
* binding `HostBashProxy.request()` applies. The target client is either
|
|
14
|
+
* auto-resolved among that actor's own `host_cu` clients
|
|
15
|
+
* ({@link pickSameUserAutoResolve}) or, when named explicitly, checked against
|
|
16
|
+
* the actor before dispatch ({@link enforceSameActorOrErrorResult}), so a
|
|
17
|
+
* caller can only capture the accessibility tree and screenshot of a machine
|
|
18
|
+
* its own authenticated user is signed in on.
|
|
19
|
+
*
|
|
20
|
+
* Every failure path resolves to `{ ok: false, reason }` rather than throwing:
|
|
21
|
+
* a caller observing in the background must degrade, never crash.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { randomUUID } from "node:crypto";
|
|
25
|
+
|
|
26
|
+
import { getLogger } from "../util/logger.js";
|
|
27
|
+
import { assistantEventHub, broadcastMessage } from "./assistant-event-hub.js";
|
|
28
|
+
import {
|
|
29
|
+
ambiguousSameUserError,
|
|
30
|
+
enforceSameActorOrErrorResult,
|
|
31
|
+
pickSameUserAutoResolve,
|
|
32
|
+
} from "./auth/same-actor.js";
|
|
33
|
+
import * as pendingInteractions from "./pending-interactions.js";
|
|
34
|
+
|
|
35
|
+
const log = getLogger("host-observe");
|
|
36
|
+
|
|
37
|
+
const DEFAULT_OBSERVE_TIMEOUT_MS = 30_000;
|
|
38
|
+
const MAX_OBSERVE_TIMEOUT_MS = 120_000;
|
|
39
|
+
|
|
40
|
+
const NO_CLIENT_REASON =
|
|
41
|
+
"No connected client supports screen observation for this user. Make sure the desktop app is running and signed in.";
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Observation fields returned by the host CU executor. Mirrors the optional
|
|
45
|
+
* fields of `CU_RESULT_SCHEMA` (see
|
|
46
|
+
* `packages/electron-desktop/src/host-proxy/cu-executor.ts`) so the executor's
|
|
47
|
+
* result deserializes as-is. `executionResult` and `secondaryWindows` are
|
|
48
|
+
* omitted: an observe-only request executes no action, so they carry nothing
|
|
49
|
+
* for the caller.
|
|
50
|
+
*/
|
|
51
|
+
export interface HostObservationFields {
|
|
52
|
+
axTree?: string;
|
|
53
|
+
axDiff?: string;
|
|
54
|
+
screenshot?: string;
|
|
55
|
+
screenshotWidthPx?: number;
|
|
56
|
+
screenshotHeightPx?: number;
|
|
57
|
+
screenWidthPt?: number;
|
|
58
|
+
screenHeightPt?: number;
|
|
59
|
+
executionError?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** A successful observation, or a structured failure. Never throws. */
|
|
63
|
+
export type HostObservation =
|
|
64
|
+
| ({ ok: true } & HostObservationFields)
|
|
65
|
+
| { ok: false; reason: string; timedOut?: boolean };
|
|
66
|
+
|
|
67
|
+
export interface ObserveHostScreenOptions {
|
|
68
|
+
/**
|
|
69
|
+
* Principal id of the actor on whose behalf the observation is taken.
|
|
70
|
+
* Required, and the only identity the target client is matched against.
|
|
71
|
+
* `undefined` (no authenticated actor) fails closed: it selects no client
|
|
72
|
+
* and rejects any explicitly named one.
|
|
73
|
+
*/
|
|
74
|
+
sourceActorPrincipalId: string | undefined;
|
|
75
|
+
/** How long to wait for the client. Defaults to 30s, capped at 120s. */
|
|
76
|
+
timeoutMs?: number;
|
|
77
|
+
/** Keep the base64 screenshot in the result. Defaults to true. */
|
|
78
|
+
includeScreenshot?: boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Target a specific client. Must belong to `sourceActorPrincipalId`.
|
|
81
|
+
* Defaults to the actor's own `host_cu` client when exactly one is
|
|
82
|
+
* connected.
|
|
83
|
+
*/
|
|
84
|
+
clientId?: string;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Failure raised by this module's own lifecycle paths (timeout, send failure),
|
|
89
|
+
* as opposed to an observation delivered by the client. Distinguished from
|
|
90
|
+
* {@link HostObservationFields} by its `failureReason` key.
|
|
91
|
+
*/
|
|
92
|
+
interface LocalFailure {
|
|
93
|
+
failureReason: string;
|
|
94
|
+
timedOut: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Resolve the `host_cu` client to observe, bound to the initiating actor.
|
|
99
|
+
* Returns the client id, or the failure to hand back to the caller.
|
|
100
|
+
*/
|
|
101
|
+
function resolveObserveTarget(
|
|
102
|
+
sourceActorPrincipalId: string | undefined,
|
|
103
|
+
clientId: string | undefined,
|
|
104
|
+
): { clientId: string } | { reason: string } {
|
|
105
|
+
let resolvedClientId: string;
|
|
106
|
+
if (clientId) {
|
|
107
|
+
const target = assistantEventHub.getClientById(clientId);
|
|
108
|
+
if (!target?.capabilities.includes("host_cu")) {
|
|
109
|
+
return {
|
|
110
|
+
reason: `Client "${clientId}" is not connected or does not support host_cu.`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
resolvedClientId = clientId;
|
|
114
|
+
} else {
|
|
115
|
+
// Auto-resolve to the unique same-user client. Refusing the ambiguous case
|
|
116
|
+
// keeps one observation from fanning out across every machine the user has
|
|
117
|
+
// connected.
|
|
118
|
+
const resolved = pickSameUserAutoResolve({
|
|
119
|
+
hub: assistantEventHub,
|
|
120
|
+
capability: "host_cu",
|
|
121
|
+
sourceActorPrincipalId,
|
|
122
|
+
});
|
|
123
|
+
if (resolved.kind === "ambiguous") {
|
|
124
|
+
return { reason: ambiguousSameUserError("host_cu").content };
|
|
125
|
+
}
|
|
126
|
+
if (resolved.kind === "none") {
|
|
127
|
+
return { reason: NO_CLIENT_REASON };
|
|
128
|
+
}
|
|
129
|
+
resolvedClientId = resolved.clientId;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Fail closed before registration and before broadcast, so no caller reaches
|
|
133
|
+
// a client whose authenticated user is not its own.
|
|
134
|
+
const rejection = enforceSameActorOrErrorResult({
|
|
135
|
+
hub: assistantEventHub,
|
|
136
|
+
sourceActorPrincipalId,
|
|
137
|
+
targetClientId: resolvedClientId,
|
|
138
|
+
op: "host_cu",
|
|
139
|
+
});
|
|
140
|
+
if (rejection) {
|
|
141
|
+
return { reason: rejection.content };
|
|
142
|
+
}
|
|
143
|
+
return { clientId: resolvedClientId };
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Ask the initiating actor's `host_cu`-capable client for the current screen
|
|
148
|
+
* state.
|
|
149
|
+
*/
|
|
150
|
+
export async function observeHostScreen(
|
|
151
|
+
options: ObserveHostScreenOptions,
|
|
152
|
+
): Promise<HostObservation> {
|
|
153
|
+
const { sourceActorPrincipalId, includeScreenshot = true } = options;
|
|
154
|
+
const timeoutMs = Math.min(
|
|
155
|
+
options.timeoutMs ?? DEFAULT_OBSERVE_TIMEOUT_MS,
|
|
156
|
+
MAX_OBSERVE_TIMEOUT_MS,
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
const target = resolveObserveTarget(sourceActorPrincipalId, options.clientId);
|
|
160
|
+
if ("reason" in target) {
|
|
161
|
+
return { ok: false, reason: target.reason };
|
|
162
|
+
}
|
|
163
|
+
const targetClientId = target.clientId;
|
|
164
|
+
|
|
165
|
+
const requestId = randomUUID();
|
|
166
|
+
|
|
167
|
+
// `/v1/host-cu-result` resolves this promise with the client's raw
|
|
168
|
+
// observation fields; the lifecycle paths below resolve it with a failure.
|
|
169
|
+
const payload = await new Promise<HostObservationFields | LocalFailure>(
|
|
170
|
+
(resolvePromise) => {
|
|
171
|
+
const timer = setTimeout(() => {
|
|
172
|
+
// Resolve the tracker first so a late client POST is tolerated as a
|
|
173
|
+
// no-op instead of double-resolving this promise.
|
|
174
|
+
if (!pendingInteractions.resolve(requestId, "cancelled")) {
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
broadcastMessage(
|
|
178
|
+
{ type: "host_cu_cancel", requestId, conversationId: "" },
|
|
179
|
+
undefined,
|
|
180
|
+
{ targetClientId },
|
|
181
|
+
);
|
|
182
|
+
resolvePromise({
|
|
183
|
+
failureReason: `Timed out after ${timeoutMs}ms waiting for the desktop client. It may be busy, outdated, or disconnected.`,
|
|
184
|
+
timedOut: true,
|
|
185
|
+
});
|
|
186
|
+
}, timeoutMs);
|
|
187
|
+
|
|
188
|
+
// The target's actor principal is present by construction: the same-actor
|
|
189
|
+
// check above rejects a client that registered without one.
|
|
190
|
+
const targetActorPrincipalId =
|
|
191
|
+
assistantEventHub.getActorPrincipalIdForClient(targetClientId);
|
|
192
|
+
pendingInteractions.register(requestId, {
|
|
193
|
+
kind: "host_cu",
|
|
194
|
+
rpcResolve: resolvePromise as (value: unknown) => void,
|
|
195
|
+
timer,
|
|
196
|
+
targetClientId,
|
|
197
|
+
...(targetActorPrincipalId ? { targetActorPrincipalId } : {}),
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
try {
|
|
201
|
+
broadcastMessage(
|
|
202
|
+
{
|
|
203
|
+
type: "host_cu_request",
|
|
204
|
+
requestId,
|
|
205
|
+
// Conversation-agnostic. The native helper keys its per-session
|
|
206
|
+
// state off this id, and an empty string is already what the
|
|
207
|
+
// desktop executor sends when no conversation is attached.
|
|
208
|
+
conversationId: "",
|
|
209
|
+
toolName: "computer_use_observe",
|
|
210
|
+
input: {},
|
|
211
|
+
stepNumber: 1,
|
|
212
|
+
},
|
|
213
|
+
undefined,
|
|
214
|
+
{ targetClientId },
|
|
215
|
+
);
|
|
216
|
+
} catch (err) {
|
|
217
|
+
pendingInteractions.resolve(requestId, "cancelled");
|
|
218
|
+
log.warn({ requestId, err }, "Screen observation broadcast failed");
|
|
219
|
+
resolvePromise({
|
|
220
|
+
failureReason: `Failed to reach the desktop client: ${String(err)}`,
|
|
221
|
+
timedOut: false,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
},
|
|
225
|
+
);
|
|
226
|
+
|
|
227
|
+
if ("failureReason" in payload) {
|
|
228
|
+
return {
|
|
229
|
+
ok: false,
|
|
230
|
+
reason: payload.failureReason,
|
|
231
|
+
...(payload.timedOut ? { timedOut: true } : {}),
|
|
232
|
+
};
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const { executionError, ...observation } = payload;
|
|
236
|
+
if (executionError) {
|
|
237
|
+
return { ok: false, reason: executionError };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (!includeScreenshot) {
|
|
241
|
+
delete observation.screenshot;
|
|
242
|
+
delete observation.screenshotWidthPx;
|
|
243
|
+
delete observation.screenshotHeightPx;
|
|
244
|
+
}
|
|
245
|
+
return { ok: true, ...observation };
|
|
246
|
+
}
|
|
@@ -7,6 +7,10 @@
|
|
|
7
7
|
|
|
8
8
|
import type { ServerWebSocket } from "bun";
|
|
9
9
|
|
|
10
|
+
import {
|
|
11
|
+
startAppPinReconcileSweep,
|
|
12
|
+
stopAppPinReconcileSweep,
|
|
13
|
+
} from "../apps/app-pin-reconciler.js";
|
|
10
14
|
import {
|
|
11
15
|
activeMediaStreamSessions,
|
|
12
16
|
MediaStreamCallSession,
|
|
@@ -93,6 +97,12 @@ import {
|
|
|
93
97
|
startInferenceProfileSessionReaper,
|
|
94
98
|
stopInferenceProfileSessionReaper,
|
|
95
99
|
} from "./routes/inference-profile-session-reaper.js";
|
|
100
|
+
import {
|
|
101
|
+
activeWatchStreamSessions,
|
|
102
|
+
closeWatchIngress,
|
|
103
|
+
drainWatchRetros,
|
|
104
|
+
WatchStreamSession,
|
|
105
|
+
} from "./routes/watch-routes.js";
|
|
96
106
|
|
|
97
107
|
// Re-export for consumers
|
|
98
108
|
export { isPrivateAddress } from "./middleware/auth.js";
|
|
@@ -166,6 +176,25 @@ interface LiveVoiceWebSocketData {
|
|
|
166
176
|
wsType: "live-voice";
|
|
167
177
|
}
|
|
168
178
|
|
|
179
|
+
/**
|
|
180
|
+
* WebSocket data attached to `/v1/watch/stream` connections. The `wsType`
|
|
181
|
+
* discriminator routes frames to the watch session orchestrator instead of
|
|
182
|
+
* the other WebSocket handlers.
|
|
183
|
+
*/
|
|
184
|
+
interface WatchStreamWebSocketData {
|
|
185
|
+
wsType: "watch-stream";
|
|
186
|
+
mimeType: string;
|
|
187
|
+
sampleRate?: number;
|
|
188
|
+
/** Conversation the session's timeline is keyed on, when the client names one. */
|
|
189
|
+
conversationId?: string;
|
|
190
|
+
/** Desktop client to observe, when the actor has more than one connected. */
|
|
191
|
+
clientId?: string;
|
|
192
|
+
/** The session ID for tracking in the active sessions registry. */
|
|
193
|
+
sessionId: string;
|
|
194
|
+
/** Bound at open time so the close handler tears down the exact session. */
|
|
195
|
+
session?: WatchStreamSession;
|
|
196
|
+
}
|
|
197
|
+
|
|
169
198
|
export class RuntimeHttpServer {
|
|
170
199
|
private server: ReturnType<typeof Bun.serve> | null = null;
|
|
171
200
|
private port: number;
|
|
@@ -204,7 +233,8 @@ export class RuntimeHttpServer {
|
|
|
204
233
|
type AllWebSocketData =
|
|
205
234
|
| MediaStreamWebSocketData
|
|
206
235
|
| SttStreamWebSocketData
|
|
207
|
-
| LiveVoiceWebSocketData
|
|
236
|
+
| LiveVoiceWebSocketData
|
|
237
|
+
| WatchStreamWebSocketData;
|
|
208
238
|
this.server = Bun.serve<AllWebSocketData>({
|
|
209
239
|
port: this.port,
|
|
210
240
|
hostname: this.hostname,
|
|
@@ -310,11 +340,41 @@ export class RuntimeHttpServer {
|
|
|
310
340
|
send: (frame) => {
|
|
311
341
|
ws.send(JSON.stringify(frame));
|
|
312
342
|
},
|
|
343
|
+
// Lets the daemon hang up on a client that stopped answering.
|
|
344
|
+
// A normal close (not a retryable one) so the client ends the
|
|
345
|
+
// call rather than reconnecting into a session that is gone.
|
|
346
|
+
close: () => {
|
|
347
|
+
ws.close(1000, "Live voice session released");
|
|
348
|
+
},
|
|
313
349
|
}),
|
|
314
350
|
);
|
|
315
351
|
log.info("Live voice WebSocket opened");
|
|
316
352
|
return;
|
|
317
353
|
}
|
|
354
|
+
if (data.wsType === "watch-stream") {
|
|
355
|
+
const watchData = data;
|
|
356
|
+
log.info(
|
|
357
|
+
{
|
|
358
|
+
sessionId: watchData.sessionId,
|
|
359
|
+
mimeType: watchData.mimeType,
|
|
360
|
+
},
|
|
361
|
+
"Watch stream WebSocket opened",
|
|
362
|
+
);
|
|
363
|
+
const session = new WatchStreamSession(ws, {
|
|
364
|
+
mimeType: watchData.mimeType,
|
|
365
|
+
...(watchData.sampleRate !== undefined
|
|
366
|
+
? { sampleRate: watchData.sampleRate }
|
|
367
|
+
: {}),
|
|
368
|
+
...(watchData.conversationId
|
|
369
|
+
? { conversationId: watchData.conversationId }
|
|
370
|
+
: {}),
|
|
371
|
+
...(watchData.clientId ? { clientId: watchData.clientId } : {}),
|
|
372
|
+
});
|
|
373
|
+
watchData.session = session;
|
|
374
|
+
activeWatchStreamSessions.set(watchData.sessionId, session);
|
|
375
|
+
void session.start();
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
318
378
|
log.warn("WebSocket opened with unknown data type — closing");
|
|
319
379
|
ws.close(1008, "Unknown WebSocket type");
|
|
320
380
|
},
|
|
@@ -353,6 +413,18 @@ export class RuntimeHttpServer {
|
|
|
353
413
|
void connection?.handleMessage(message);
|
|
354
414
|
return;
|
|
355
415
|
}
|
|
416
|
+
if (data.wsType === "watch-stream") {
|
|
417
|
+
const session = data.session;
|
|
418
|
+
if (!session) {
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
if (typeof message === "string") {
|
|
422
|
+
session.handleMessage(message);
|
|
423
|
+
} else {
|
|
424
|
+
session.handleBinaryAudio(message);
|
|
425
|
+
}
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
356
428
|
log.warn("WebSocket message on unknown data type — closing");
|
|
357
429
|
ws.close(1008, "Unknown WebSocket type");
|
|
358
430
|
},
|
|
@@ -423,6 +495,29 @@ export class RuntimeHttpServer {
|
|
|
423
495
|
connection?.release();
|
|
424
496
|
return;
|
|
425
497
|
}
|
|
498
|
+
if (data.wsType === "watch-stream") {
|
|
499
|
+
const watchData = data;
|
|
500
|
+
log.info(
|
|
501
|
+
{
|
|
502
|
+
sessionId: watchData.sessionId,
|
|
503
|
+
code,
|
|
504
|
+
reason: reason?.toString(),
|
|
505
|
+
},
|
|
506
|
+
"Watch stream WebSocket closed",
|
|
507
|
+
);
|
|
508
|
+
const session = watchData.session;
|
|
509
|
+
if (session) {
|
|
510
|
+
session.handleClose(code, reason?.toString());
|
|
511
|
+
// Only drop our own session from the registry, since a
|
|
512
|
+
// reconnect may have already replaced it under a new id.
|
|
513
|
+
if (
|
|
514
|
+
activeWatchStreamSessions.get(watchData.sessionId) === session
|
|
515
|
+
) {
|
|
516
|
+
activeWatchStreamSessions.delete(watchData.sessionId);
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
426
521
|
log.warn(
|
|
427
522
|
{ code, reason: reason?.toString() },
|
|
428
523
|
"WebSocket with unknown data type closed",
|
|
@@ -523,10 +618,16 @@ export class RuntimeHttpServer {
|
|
|
523
618
|
// migrations are unready.
|
|
524
619
|
startPluginScheduleReconcileSweep();
|
|
525
620
|
log.info("Plugin schedule reconcile sweep started");
|
|
621
|
+
|
|
622
|
+
// Same backstop for sidebar pins: a disabled plugin's app stops existing,
|
|
623
|
+
// and its id returns intact when the plugin is re-enabled.
|
|
624
|
+
startAppPinReconcileSweep();
|
|
625
|
+
log.info("App pin reconcile sweep started");
|
|
526
626
|
}
|
|
527
627
|
|
|
528
628
|
async stop(): Promise<void> {
|
|
529
629
|
stopGuardianExpirySweep();
|
|
630
|
+
stopAppPinReconcileSweep();
|
|
530
631
|
stopInferenceProfileSessionReaper();
|
|
531
632
|
stopTelegramWebhookHealthSweep();
|
|
532
633
|
stopPluginScheduleReconcileSweep();
|
|
@@ -543,6 +644,20 @@ export class RuntimeHttpServer {
|
|
|
543
644
|
activeSttStreamSessions.delete(sessionId);
|
|
544
645
|
}
|
|
545
646
|
|
|
647
|
+
// Watch shuts down in order: refuse new sessions, tear down the open ones,
|
|
648
|
+
// then wait on the retrospectives they left running. Refusing first is what
|
|
649
|
+
// makes the wait meaningful, because the Bun server below keeps accepting
|
|
650
|
+
// connections until the very end and a session opened during the wait would
|
|
651
|
+
// register a retrospective after it had already taken its snapshot.
|
|
652
|
+
closeWatchIngress();
|
|
653
|
+
for (const [sessionId, session] of activeWatchStreamSessions) {
|
|
654
|
+
session.destroy();
|
|
655
|
+
activeWatchStreamSessions.delete(sessionId);
|
|
656
|
+
}
|
|
657
|
+
// Destroying a session starts no retrospective, so this waits only on one
|
|
658
|
+
// that a socket closing just before shutdown had already under way.
|
|
659
|
+
await drainWatchRetros();
|
|
660
|
+
|
|
546
661
|
const liveVoiceManager = getLiveVoiceSessionManager();
|
|
547
662
|
const liveVoiceSessionId = liveVoiceManager.activeSessionId;
|
|
548
663
|
if (liveVoiceSessionId) {
|
|
@@ -636,6 +751,16 @@ export class RuntimeHttpServer {
|
|
|
636
751
|
return this.handleLiveVoiceUpgrade(req, server);
|
|
637
752
|
}
|
|
638
753
|
|
|
754
|
+
// WebSocket upgrade for watch narration capture, under the same
|
|
755
|
+
// private-network restrictions and gateway-service token verification as
|
|
756
|
+
// STT streaming.
|
|
757
|
+
if (
|
|
758
|
+
path === "/v1/watch/stream" &&
|
|
759
|
+
req.headers.get("upgrade")?.toLowerCase() === "websocket"
|
|
760
|
+
) {
|
|
761
|
+
return this.handleWatchStreamUpgrade(req, server);
|
|
762
|
+
}
|
|
763
|
+
|
|
639
764
|
// Twilio webhook endpoints — before auth check because Twilio
|
|
640
765
|
// webhook POSTs don't include bearer tokens.
|
|
641
766
|
const twilioResponse = await this.handleTwilioWebhook(req, path);
|
|
@@ -923,6 +1048,61 @@ export class RuntimeHttpServer {
|
|
|
923
1048
|
return undefined!;
|
|
924
1049
|
}
|
|
925
1050
|
|
|
1051
|
+
/**
|
|
1052
|
+
* Handle WebSocket upgrade for `/v1/watch/stream`.
|
|
1053
|
+
*
|
|
1054
|
+
* Gated exactly as `/v1/stt/stream` is: private network peers and origins
|
|
1055
|
+
* only, then a gateway service token. The gateway owns downstream client
|
|
1056
|
+
* auth and dials this upstream on the client's behalf.
|
|
1057
|
+
*/
|
|
1058
|
+
private handleWatchStreamUpgrade(
|
|
1059
|
+
req: Request,
|
|
1060
|
+
server: ReturnType<typeof Bun.serve>,
|
|
1061
|
+
): Response {
|
|
1062
|
+
if (!isPrivateNetworkPeer(server, req) || !isPrivateNetworkOrigin(req)) {
|
|
1063
|
+
return httpError(
|
|
1064
|
+
"FORBIDDEN",
|
|
1065
|
+
"Direct watch stream access disabled: only private network peers allowed",
|
|
1066
|
+
403,
|
|
1067
|
+
);
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
const tokenError = this.verifyGatewayServiceToken(req);
|
|
1071
|
+
if (tokenError) {
|
|
1072
|
+
return tokenError;
|
|
1073
|
+
}
|
|
1074
|
+
|
|
1075
|
+
const wsUrl = new URL(req.url);
|
|
1076
|
+
const mimeType = wsUrl.searchParams.get("mimeType");
|
|
1077
|
+
if (!mimeType) {
|
|
1078
|
+
return new Response("Missing required query parameter: mimeType", {
|
|
1079
|
+
status: 400,
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
const sampleRateRaw = wsUrl.searchParams.get("sampleRate");
|
|
1084
|
+
const sampleRate = sampleRateRaw ? parseInt(sampleRateRaw, 10) : undefined;
|
|
1085
|
+
const conversationId =
|
|
1086
|
+
wsUrl.searchParams.get("conversationId")?.trim() || undefined;
|
|
1087
|
+
const clientId = wsUrl.searchParams.get("clientId")?.trim() || undefined;
|
|
1088
|
+
|
|
1089
|
+
const upgraded = server.upgrade(req, {
|
|
1090
|
+
data: {
|
|
1091
|
+
wsType: "watch-stream",
|
|
1092
|
+
mimeType,
|
|
1093
|
+
sampleRate,
|
|
1094
|
+
conversationId,
|
|
1095
|
+
clientId,
|
|
1096
|
+
sessionId: crypto.randomUUID(),
|
|
1097
|
+
} satisfies WatchStreamWebSocketData,
|
|
1098
|
+
});
|
|
1099
|
+
if (!upgraded) {
|
|
1100
|
+
return new Response("WebSocket upgrade failed", { status: 500 });
|
|
1101
|
+
}
|
|
1102
|
+
// Bun's WebSocket upgrade consumes the request, so no Response is sent.
|
|
1103
|
+
return undefined!;
|
|
1104
|
+
}
|
|
1105
|
+
|
|
926
1106
|
private async handleTwilioWebhook(
|
|
927
1107
|
req: Request,
|
|
928
1108
|
path: string,
|