@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -29
- package/dist/audio/audioMeter.d.ts +71 -0
- package/dist/audio/audioMeter.d.ts.map +1 -1
- package/dist/audio/audioMeter.js +83 -0
- package/dist/audio/audioMeter.js.map +1 -1
- package/dist/audio/mediaElementAudioRouter.d.ts +41 -0
- package/dist/audio/mediaElementAudioRouter.d.ts.map +1 -0
- package/dist/audio/mediaElementAudioRouter.js +82 -0
- package/dist/audio/mediaElementAudioRouter.js.map +1 -0
- package/dist/audio/micCapture.d.ts +12 -3
- package/dist/audio/micCapture.d.ts.map +1 -1
- package/dist/audio/micCapture.js +15 -1
- package/dist/audio/micCapture.js.map +1 -1
- package/dist/audio/pcmPlayback.d.ts +72 -4
- package/dist/audio/pcmPlayback.d.ts.map +1 -1
- package/dist/audio/pcmPlayback.js +101 -1
- package/dist/audio/pcmPlayback.js.map +1 -1
- package/dist/drivers/assemblyAIRealtimeClient.d.ts +2 -0
- package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/assemblyAIRealtimeClient.js +14 -0
- package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -1
- package/dist/drivers/elevenLabsRealtimeClient.d.ts +2 -0
- package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/elevenLabsRealtimeClient.js +14 -0
- package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -1
- package/dist/drivers/geminiAvatarOutput.d.ts +132 -0
- package/dist/drivers/geminiAvatarOutput.d.ts.map +1 -0
- package/dist/drivers/geminiAvatarOutput.js +310 -0
- package/dist/drivers/geminiAvatarOutput.js.map +1 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts +38 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.js +107 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.js.map +1 -0
- package/dist/drivers/geminiRealtimeClient.d.ts +314 -48
- package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/geminiRealtimeClient.js +759 -176
- package/dist/drivers/geminiRealtimeClient.js.map +1 -1
- package/dist/drivers/openAILiveClient.d.ts +5 -8
- package/dist/drivers/openAILiveClient.d.ts.map +1 -1
- package/dist/drivers/openAILiveClient.js +19 -32
- package/dist/drivers/openAILiveClient.js.map +1 -1
- package/dist/drivers/openAIRealtimeClient.d.ts +13 -26
- package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/openAIRealtimeClient.js +28 -55
- package/dist/drivers/openAIRealtimeClient.js.map +1 -1
- package/dist/generic/baseRealtimeClient.d.ts +173 -29
- package/dist/generic/baseRealtimeClient.d.ts.map +1 -1
- package/dist/generic/baseRealtimeClient.js +233 -14
- package/dist/generic/baseRealtimeClient.js.map +1 -1
- package/dist/generic/openAIProtocolClient.d.ts +2 -0
- package/dist/generic/openAIProtocolClient.d.ts.map +1 -1
- package/dist/generic/openAIProtocolClient.js +14 -0
- package/dist/generic/openAIProtocolClient.js.map +1 -1
- package/dist/index.d.ts +2 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/media/attachVideoSource.d.ts +23 -0
- package/dist/media/attachVideoSource.d.ts.map +1 -0
- package/dist/media/attachVideoSource.js +49 -0
- package/dist/media/attachVideoSource.js.map +1 -0
- package/dist/media/channelVideoSource.d.ts +110 -14
- package/dist/media/channelVideoSource.d.ts.map +1 -1
- package/dist/media/channelVideoSource.js +183 -18
- package/dist/media/channelVideoSource.js.map +1 -1
- package/dist/media/decoders/frameScheduler.d.ts +119 -0
- package/dist/media/decoders/frameScheduler.d.ts.map +1 -0
- package/dist/media/decoders/frameScheduler.js +238 -0
- package/dist/media/decoders/frameScheduler.js.map +1 -0
- package/dist/media/decoders/imageFrameDecoder.d.ts +58 -0
- package/dist/media/decoders/imageFrameDecoder.d.ts.map +1 -0
- package/dist/media/decoders/imageFrameDecoder.js +110 -0
- package/dist/media/decoders/imageFrameDecoder.js.map +1 -0
- package/dist/media/decoders/mseFmp4Decoder.d.ts +130 -0
- package/dist/media/decoders/mseFmp4Decoder.d.ts.map +1 -0
- package/dist/media/decoders/mseFmp4Decoder.js +402 -0
- package/dist/media/decoders/mseFmp4Decoder.js.map +1 -0
- package/dist/media/decoders/streamCanvas.d.ts +46 -0
- package/dist/media/decoders/streamCanvas.d.ts.map +1 -0
- package/dist/media/decoders/streamCanvas.js +163 -0
- package/dist/media/decoders/streamCanvas.js.map +1 -0
- package/dist/media/decoders/videoMimeTypes.d.ts +29 -0
- package/dist/media/decoders/videoMimeTypes.d.ts.map +1 -0
- package/dist/media/decoders/videoMimeTypes.js +50 -0
- package/dist/media/decoders/videoMimeTypes.js.map +1 -0
- package/dist/media/decoders/webCodecsChunkDecoder.d.ts +99 -0
- package/dist/media/decoders/webCodecsChunkDecoder.d.ts.map +1 -0
- package/dist/media/decoders/webCodecsChunkDecoder.js +265 -0
- package/dist/media/decoders/webCodecsChunkDecoder.js.map +1 -0
- package/dist/media/displayCapture.d.ts +128 -0
- package/dist/media/displayCapture.d.ts.map +1 -0
- package/dist/media/displayCapture.js +245 -0
- package/dist/media/displayCapture.js.map +1 -0
- package/dist/media/frameCapture.d.ts +25 -8
- package/dist/media/frameCapture.d.ts.map +1 -1
- package/dist/media/frameCapture.js +42 -72
- package/dist/media/frameCapture.js.map +1 -1
- package/dist/media/frameSampler.d.ts +94 -0
- package/dist/media/frameSampler.d.ts.map +1 -0
- package/dist/media/frameSampler.js +134 -0
- package/dist/media/frameSampler.js.map +1 -0
- package/dist/media/index.d.ts +32 -0
- package/dist/media/index.d.ts.map +1 -0
- package/dist/media/index.js +32 -0
- package/dist/media/index.js.map +1 -0
- package/dist/media/localMediaController.d.ts +116 -0
- package/dist/media/localMediaController.d.ts.map +1 -0
- package/dist/media/localMediaController.js +281 -0
- package/dist/media/localMediaController.js.map +1 -0
- package/dist/media/mediaLayout.d.ts +93 -0
- package/dist/media/mediaLayout.d.ts.map +1 -0
- package/dist/media/mediaLayout.js +187 -0
- package/dist/media/mediaLayout.js.map +1 -0
- package/dist/media/mediaPreview.d.ts +94 -0
- package/dist/media/mediaPreview.d.ts.map +1 -0
- package/dist/media/mediaPreview.js +154 -0
- package/dist/media/mediaPreview.js.map +1 -0
- package/dist/media/mediaStage.d.ts +99 -0
- package/dist/media/mediaStage.d.ts.map +1 -0
- package/dist/media/mediaStage.js +171 -0
- package/dist/media/mediaStage.js.map +1 -0
- package/dist/media/model.d.ts +154 -0
- package/dist/media/model.d.ts.map +1 -0
- package/dist/media/model.js +9 -0
- package/dist/media/model.js.map +1 -0
- package/dist/media/playbackClock.d.ts +20 -0
- package/dist/media/playbackClock.d.ts.map +1 -0
- package/dist/media/playbackClock.js +13 -0
- package/dist/media/playbackClock.js.map +1 -0
- package/dist/media/videoFrameDecoder.d.ts +86 -0
- package/dist/media/videoFrameDecoder.d.ts.map +1 -0
- package/dist/media/videoFrameDecoder.js +2 -0
- package/dist/media/videoFrameDecoder.js.map +1 -0
- package/dist/media/videoFrameDecoderRegistry.d.ts +40 -0
- package/dist/media/videoFrameDecoderRegistry.d.ts.map +1 -0
- package/dist/media/videoFrameDecoderRegistry.js +69 -0
- package/dist/media/videoFrameDecoderRegistry.js.map +1 -0
- package/dist/media/videoPacing.d.ts +32 -0
- package/dist/media/videoPacing.d.ts.map +1 -0
- package/dist/media/videoPacing.js +37 -0
- package/dist/media/videoPacing.js.map +1 -0
- package/dist/media/videoPlayout.d.ts +168 -0
- package/dist/media/videoPlayout.d.ts.map +1 -0
- package/dist/media/videoPlayout.js +217 -0
- package/dist/media/videoPlayout.js.map +1 -0
- package/dist/media/videoSourceArbiter.d.ts +250 -0
- package/dist/media/videoSourceArbiter.d.ts.map +1 -0
- package/dist/media/videoSourceArbiter.js +423 -0
- package/dist/media/videoSourceArbiter.js.map +1 -0
- package/dist/testing/conformanceAssertions.d.ts +25 -0
- package/dist/testing/conformanceAssertions.d.ts.map +1 -0
- package/dist/testing/conformanceAssertions.js +64 -0
- package/dist/testing/conformanceAssertions.js.map +1 -0
- package/dist/testing/conformanceFixtures.d.ts +76 -0
- package/dist/testing/conformanceFixtures.d.ts.map +1 -0
- package/dist/testing/conformanceFixtures.js +169 -0
- package/dist/testing/conformanceFixtures.js.map +1 -0
- package/dist/testing/conformanceGates.d.ts +32 -0
- package/dist/testing/conformanceGates.d.ts.map +1 -0
- package/dist/testing/conformanceGates.js +35 -0
- package/dist/testing/conformanceGates.js.map +1 -0
- package/dist/testing/conformanceMicrophone.d.ts +10 -0
- package/dist/testing/conformanceMicrophone.d.ts.map +1 -0
- package/dist/testing/conformanceMicrophone.js +71 -0
- package/dist/testing/conformanceMicrophone.js.map +1 -0
- package/dist/testing/conformanceTimeline.d.ts +128 -0
- package/dist/testing/conformanceTimeline.d.ts.map +1 -0
- package/dist/testing/conformanceTimeline.js +32 -0
- package/dist/testing/conformanceTimeline.js.map +1 -0
- package/dist/testing/conformanceVideoStream.d.ts +29 -0
- package/dist/testing/conformanceVideoStream.d.ts.map +1 -0
- package/dist/testing/conformanceVideoStream.js +39 -0
- package/dist/testing/conformanceVideoStream.js.map +1 -0
- package/dist/testing/index.d.ts +29 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +28 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/realtimeVideoConformance.d.ts +23 -0
- package/dist/testing/realtimeVideoConformance.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformance.js +139 -0
- package/dist/testing/realtimeVideoConformance.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceChecks.js +164 -0
- package/dist/testing/realtimeVideoConformanceChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.js +63 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceSession.d.ts +108 -0
- package/dist/testing/realtimeVideoConformanceSession.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceSession.js +270 -0
- package/dist/testing/realtimeVideoConformanceSession.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.js +248 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceTypes.d.ts +155 -0
- package/dist/testing/realtimeVideoConformanceTypes.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceTypes.js +2 -0
- package/dist/testing/realtimeVideoConformanceTypes.js.map +1 -0
- package/dist/testing/recordingPlayout.d.ts +84 -0
- package/dist/testing/recordingPlayout.d.ts.map +1 -0
- package/dist/testing/recordingPlayout.js +120 -0
- package/dist/testing/recordingPlayout.js.map +1 -0
- package/package.json +29 -4
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview ATTACH VIDEO SOURCE: shows any {@link MediaVideoSource} in a `<video>` element and hands back
|
|
3
|
+
* a function that takes it out again. UI code calls this instead of a vendor's `track.attach(element)`, so a
|
|
4
|
+
* tile renders a camera, a shared screen, a meeting participant or an avatar the same way.
|
|
5
|
+
*
|
|
6
|
+
* - A **stream** source (a camera, a shared screen, a WebRTC track) plays muted and inline. A video surface
|
|
7
|
+
* never plays audio: the realtime client or the meeting room plays each voice exactly once.
|
|
8
|
+
* - An **element** source (a player that must own the element, such as MSE playout of an avatar's MP4) is
|
|
9
|
+
* handed the element and decides everything, including its audio: an avatar's voice is in its video.
|
|
10
|
+
*
|
|
11
|
+
* Detaching never stops the source's tracks; whoever opened them stops them.
|
|
12
|
+
*
|
|
13
|
+
* @module @memberjunction/ai-realtime-client/media
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Shows a video source in a `<video>` element.
|
|
17
|
+
*
|
|
18
|
+
* @returns A function that takes the source out of the element. Calling it twice does nothing, and it leaves
|
|
19
|
+
* the element alone if another source has been attached to it since.
|
|
20
|
+
*/
|
|
21
|
+
export function AttachVideoSource(source, element) {
|
|
22
|
+
if (source.Kind === 'element') {
|
|
23
|
+
return once(source.Attach(element));
|
|
24
|
+
}
|
|
25
|
+
element.muted = true;
|
|
26
|
+
element.playsInline = true;
|
|
27
|
+
element.autoplay = true;
|
|
28
|
+
element.srcObject = source.Stream;
|
|
29
|
+
void element.play().catch(() => {
|
|
30
|
+
// Muted inline video may play; if the browser still waits (a hidden element), autoplay starts it later.
|
|
31
|
+
});
|
|
32
|
+
return once(() => {
|
|
33
|
+
if (element.srcObject === source.Stream) {
|
|
34
|
+
element.pause();
|
|
35
|
+
element.srcObject = null;
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
/** Wraps a function so that only its first call does anything. */
|
|
40
|
+
function once(action) {
|
|
41
|
+
let done = false;
|
|
42
|
+
return () => {
|
|
43
|
+
if (!done) {
|
|
44
|
+
done = true;
|
|
45
|
+
action();
|
|
46
|
+
}
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=attachVideoSource.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"attachVideoSource.js","sourceRoot":"","sources":["../../src/media/attachVideoSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAIH;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAwB,EAAE,OAAyB;IACjF,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC;IACrB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAC3B,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;IACxB,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;IAClC,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;QAC3B,wGAAwG;IAC5G,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,GAAG,EAAE;QACb,IAAI,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YACtC,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;QAC7B,CAAC;IACL,CAAC,CAAC,CAAC;AACP,CAAC;AAED,kEAAkE;AAClE,SAAS,IAAI,CAAC,MAAkB;IAC5B,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,OAAO,GAAG,EAAE;QACR,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,IAAI,GAAG,IAAI,CAAC;YACZ,MAAM,EAAE,CAAC;QACb,CAAC;IACL,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview SHARED CHANNEL INBOUND VIDEO BRIDGE (Phase F5).
|
|
3
3
|
*
|
|
4
|
-
* Single shared implementation used by
|
|
5
|
-
*
|
|
4
|
+
* Single shared implementation used by the Whiteboard, Remote Browser and any other channel that
|
|
5
|
+
* shows the model pixels. It no longer writes to the model directly: it REGISTERS its channel as a
|
|
6
|
+
* source with the session's {@link VideoSourceArbiter} and pushes frames to it, so the arbiter stays
|
|
7
|
+
* the single writer of inbound video and decides which source the model sees when several are live.
|
|
6
8
|
*
|
|
7
9
|
* Enforces:
|
|
8
10
|
* 1. Model capability & track gate: if the session has not established an inbound video track,
|
|
9
11
|
* falls back to tool-only behaviour — NO frames sent and NO errors raised.
|
|
10
|
-
* 2. Cadence
|
|
11
|
-
*
|
|
12
|
+
* 2. Cadence: the poller (when used) runs at the NEGOTIATED inbound rate, read from the client;
|
|
13
|
+
* there is no rate constant here. The arbiter and the driver enforce the same rate as a spacing.
|
|
14
|
+
* 3. Clean teardown on Stop(), which also unregisters the source.
|
|
15
|
+
*
|
|
16
|
+
* An owner that sends frames only when its picture changes learns from {@link ChannelInboundVideoBridge.OnSourceActivated}
|
|
17
|
+
* when the arbiter starts sending its source to the model again, so it can send the picture the model missed.
|
|
12
18
|
*
|
|
13
19
|
* @module @memberjunction/ai-realtime-client
|
|
14
20
|
* @author MemberJunction.com
|
|
15
21
|
*/
|
|
16
|
-
import type
|
|
22
|
+
import { VideoSourceArbiter, type IVideoFrameSink, type VideoSourceKind } from './videoSourceArbiter.js';
|
|
17
23
|
/**
|
|
18
24
|
* Provider implemented by a channel or surface to produce visual frames.
|
|
19
25
|
*/
|
|
@@ -28,13 +34,36 @@ export interface IChannelFrameProvider {
|
|
|
28
34
|
* Options for the inbound channel video bridge.
|
|
29
35
|
*/
|
|
30
36
|
export interface ChannelInboundVideoBridgeOptions {
|
|
31
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Target poll rate in fps for {@link ChannelInboundVideoBridge.Start}.
|
|
39
|
+
*
|
|
40
|
+
* @deprecated The poll follows the rate the inbound video track NEGOTIATED (the model's own
|
|
41
|
+
* ceiling), read from the client. When set, the poll runs at the lower of this and the negotiated
|
|
42
|
+
* rate; it is no longer clamped to a constant 1 fps.
|
|
43
|
+
*/
|
|
32
44
|
Rate?: number;
|
|
33
45
|
/** Output MIME type. Defaults to 'image/jpeg'. */
|
|
34
46
|
MimeType?: string;
|
|
47
|
+
/**
|
|
48
|
+
* The id this source registers under. Defaults to a unique generated id. A channel passes a stable
|
|
49
|
+
* one (`<channel>#<instance>`) so the UI can address the source.
|
|
50
|
+
*/
|
|
51
|
+
SourceID?: string;
|
|
52
|
+
/** Human-readable name used in notes to the model and in the UI's "agent can see" list. Defaults to the source id. */
|
|
53
|
+
Label?: string;
|
|
54
|
+
/** What kind of source this is. Defaults to `'surface'` (a channel surface, not a user capture). */
|
|
55
|
+
Kind?: VideoSourceKind;
|
|
56
|
+
/** The channel this source belongs to, so the UI can map it to the user's persisted per-channel choice. */
|
|
57
|
+
ChannelKey?: string;
|
|
58
|
+
/**
|
|
59
|
+
* The arbiter to register with. Defaults to the one for the session's client
|
|
60
|
+
* ({@link VideoSourceArbiter.ForSink}), which is what makes it a single writer. Override for tests.
|
|
61
|
+
*/
|
|
62
|
+
Arbiter?: VideoSourceArbiter;
|
|
35
63
|
}
|
|
36
64
|
/**
|
|
37
|
-
* Shared bridge connecting a visual channel surface to the session's inbound video track
|
|
65
|
+
* Shared bridge connecting a visual channel surface to the session's inbound video track, by way of
|
|
66
|
+
* the session's {@link VideoSourceArbiter}.
|
|
38
67
|
*/
|
|
39
68
|
export declare class ChannelInboundVideoBridge {
|
|
40
69
|
private readonly clientOrGetter;
|
|
@@ -43,8 +72,63 @@ export declare class ChannelInboundVideoBridge {
|
|
|
43
72
|
private timer;
|
|
44
73
|
private active;
|
|
45
74
|
private frameErrorLogged;
|
|
46
|
-
|
|
75
|
+
private readonly sourceId;
|
|
76
|
+
/** The arbiter this bridge is currently registered with; `null` when it is not registered. */
|
|
77
|
+
private registeredWith;
|
|
78
|
+
/** Whether the owner switched this source off (exposure policy or the user). Survives re-registration. */
|
|
79
|
+
private sourceEnabled;
|
|
80
|
+
/** The owner's handlers for {@link OnSourceActivated}. They survive re-registration. */
|
|
81
|
+
private readonly activatedHandlers;
|
|
82
|
+
/** Stops following the arbiter's changes; `null` while not following (not registered, or no handler). */
|
|
83
|
+
private stopFollowingArbiter;
|
|
84
|
+
/** Whether the arbiter was sending this source to the model as of its last change. */
|
|
85
|
+
private forwarded;
|
|
86
|
+
/**
|
|
87
|
+
* @param clientOrGetter The session's client, or a getter for it (a reconnect replaces the client). Any
|
|
88
|
+
* {@link IVideoFrameSink} works; a `BaseRealtimeClient` is one.
|
|
89
|
+
* @param provider Produces the frames.
|
|
90
|
+
* @param options Source identity, MIME type and the deprecated rate cap.
|
|
91
|
+
*/
|
|
92
|
+
constructor(clientOrGetter: IVideoFrameSink | (() => IVideoFrameSink | null | undefined) | null | undefined, provider: IChannelFrameProvider, options?: ChannelInboundVideoBridgeOptions);
|
|
47
93
|
private get client();
|
|
94
|
+
/** The id this bridge's source is registered under. */
|
|
95
|
+
get SourceID(): string;
|
|
96
|
+
/** The arbiter for the current client, or `null` when there is no client yet. */
|
|
97
|
+
private resolveArbiter;
|
|
98
|
+
/**
|
|
99
|
+
* Registers this bridge's source with the session's arbiter (idempotent). A new session means a new
|
|
100
|
+
* client and so a new arbiter: the source is re-registered there and removed from the old one, so a
|
|
101
|
+
* reconnect never leaves a ghost source behind.
|
|
102
|
+
*
|
|
103
|
+
* @returns The arbiter, or `null` when there is no client to register with.
|
|
104
|
+
*/
|
|
105
|
+
private ensureRegistered;
|
|
106
|
+
/**
|
|
107
|
+
* Calls `handler` each time the arbiter starts sending this source to the model after it was not: another source
|
|
108
|
+
* that held the model's only stream stopped or was set aside, or this source was switched back on. The source's
|
|
109
|
+
* first registration does not count. A frame offered while the source was not sent never reached the model, so an
|
|
110
|
+
* owner that sends frames only when its picture changes sends its current picture here.
|
|
111
|
+
*
|
|
112
|
+
* The handler is kept across a re-registration (a reconnect's new client) and called until removed.
|
|
113
|
+
*
|
|
114
|
+
* @param handler Called synchronously from the arbiter's change event; it should not throw.
|
|
115
|
+
* @returns A function that removes the handler.
|
|
116
|
+
*/
|
|
117
|
+
OnSourceActivated(handler: () => void): () => void;
|
|
118
|
+
/** Starts following the arbiter this source is registered with, when an owner asked to hear of activations. */
|
|
119
|
+
private followArbiter;
|
|
120
|
+
/** Stops following the arbiter's changes. */
|
|
121
|
+
private unfollowArbiter;
|
|
122
|
+
/** Calls each activation handler, containing a handler that throws. */
|
|
123
|
+
private notifyActivated;
|
|
124
|
+
/**
|
|
125
|
+
* Registers this bridge's source with the session's arbiter now, without starting any poll — so the
|
|
126
|
+
* source is LISTED (and can be switched on or off) before it has pushed a frame. A source that is
|
|
127
|
+
* switched off never pushes one, so without this it could never be switched back on.
|
|
128
|
+
*
|
|
129
|
+
* @returns `true` if registered; `false` when there is no client or no inbound video track (nothing to register with).
|
|
130
|
+
*/
|
|
131
|
+
Register(): boolean;
|
|
48
132
|
/**
|
|
49
133
|
* Starts the periodic frame pump if the session has an established inbound video track.
|
|
50
134
|
*
|
|
@@ -52,21 +136,32 @@ export declare class ChannelInboundVideoBridge {
|
|
|
52
136
|
* inbound video (or track was not established), falling back to tool-only behaviour.
|
|
53
137
|
*/
|
|
54
138
|
Start(): boolean;
|
|
139
|
+
/** The poll interval: the negotiated rate, capped by the (deprecated) `Rate` option when one was given. */
|
|
140
|
+
private pollIntervalMs;
|
|
55
141
|
/**
|
|
56
|
-
*
|
|
57
|
-
* and the
|
|
142
|
+
* Offers a frame to the arbiter, which forwards it only if this source is among those the model
|
|
143
|
+
* should see (and enabled, and inside the negotiated pacing interval).
|
|
58
144
|
*
|
|
59
|
-
* @returns `true` if dispatched to the
|
|
145
|
+
* @returns `true` if dispatched to the model; `false` otherwise.
|
|
60
146
|
*/
|
|
61
147
|
private sendFrameDirect;
|
|
62
148
|
/**
|
|
63
149
|
* Pushes an event-driven frame (e.g. screencast push from RemoteBrowser or stroke completion).
|
|
64
|
-
*
|
|
150
|
+
* Registers the source on first use, so a purely event-driven channel never needs {@link Start}.
|
|
65
151
|
*
|
|
66
152
|
* @param base64Jpeg The base64-encoded image frame.
|
|
67
|
-
* @returns `true` if the frame was sent; `false` if dropped (throttled,
|
|
153
|
+
* @returns `true` if the frame was sent; `false` if dropped (throttled, not the source the model is
|
|
154
|
+
* viewing, turned off, track unestablished, or driver lacks video support).
|
|
68
155
|
*/
|
|
69
156
|
PushFrame(base64Jpeg: string): boolean;
|
|
157
|
+
/**
|
|
158
|
+
* Turns this source on or off at the arbiter (a no-op before the first registration other than
|
|
159
|
+
* remembering the choice). A disabled source is not a candidate and forwards nothing.
|
|
160
|
+
*
|
|
161
|
+
* @param enabled Whether the model may see this source.
|
|
162
|
+
* @param notify Whether the arbiter tells the model. Pass `false` when the caller sends its own note.
|
|
163
|
+
*/
|
|
164
|
+
SetSourceEnabled(enabled: boolean, notify?: boolean): void;
|
|
70
165
|
/**
|
|
71
166
|
* Periodic poll tick that fetches the latest frame from the provider and pushes it.
|
|
72
167
|
* Uses `sendFrameDirect` so periodic polling paced by `setInterval` does not fight
|
|
@@ -74,7 +169,8 @@ export declare class ChannelInboundVideoBridge {
|
|
|
74
169
|
*/
|
|
75
170
|
private pumpFrame;
|
|
76
171
|
/**
|
|
77
|
-
* Stops the bridge and
|
|
172
|
+
* Stops the bridge: clears the poll timer and removes this source from the arbiter. Activation handlers stay, and
|
|
173
|
+
* are called again if the source registers again.
|
|
78
174
|
*/
|
|
79
175
|
Stop(): void;
|
|
80
176
|
/** Whether the video bridge is currently actively pumping frames. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"channelVideoSource.d.ts","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"channelVideoSource.d.ts","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAGH,OAAO,EAAE,kBAAkB,EAAE,KAAK,eAAe,EAAE,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAEtG;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,cAAc,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC;CAC5D;AAED;;GAEG;AACH,MAAM,WAAW,gCAAgC;IAC7C;;;;;;OAMG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sHAAsH;IACtH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB,2GAA2G;IAC3G,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,OAAO,CAAC,EAAE,kBAAkB,CAAC;CAChC;AAKD;;;GAGG;AACH,qBAAa,yBAAyB;IAuB9B,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAxB5B,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,8FAA8F;IAC9F,OAAO,CAAC,cAAc,CAAmC;IACzD,0GAA0G;IAC1G,OAAO,CAAC,aAAa,CAAQ;IAC7B,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAyB;IAC3D,yGAAyG;IACzG,OAAO,CAAC,oBAAoB,CAA6B;IACzD,sFAAsF;IACtF,OAAO,CAAC,SAAS,CAAS;IAE1B;;;;;OAKG;gBAEkB,cAAc,EAAE,eAAe,GAAG,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,SAAS,EAC/F,QAAQ,EAAE,qBAAqB,EAC/B,OAAO,GAAE,gCAAqC;IAKnE,OAAO,KAAK,MAAM,GAEjB;IAED,uDAAuD;IACvD,IAAW,QAAQ,IAAI,MAAM,CAE5B;IAED,iFAAiF;IACjF,OAAO,CAAC,cAAc;IAQtB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IAwBxB;;;;;;;;;;OAUG;IACI,iBAAiB,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAWzD,+GAA+G;IAC/G,OAAO,CAAC,aAAa;IAgBrB,6CAA6C;IAC7C,OAAO,CAAC,eAAe;IAMvB,uEAAuE;IACvE,OAAO,CAAC,eAAe;IAUvB;;;;;;OAMG;IACI,QAAQ,IAAI,OAAO;IAQ1B;;;;;OAKG;IACI,KAAK,IAAI,OAAO;IAiBvB,2GAA2G;IAC3G,OAAO,CAAC,cAAc;IAUtB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAYvB;;;;;;;OAOG;IACI,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO;IAI7C;;;;;;OAMG;IACI,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,UAAO,GAAG,IAAI;IAK9D;;;;OAIG;YACW,SAAS;IAkBvB;;;OAGG;IACI,IAAI,IAAI,IAAI;IAYnB,qEAAqE;IACrE,IAAW,QAAQ,IAAI,OAAO,CAE7B;CACJ"}
|
|
@@ -1,22 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @fileoverview SHARED CHANNEL INBOUND VIDEO BRIDGE (Phase F5).
|
|
3
3
|
*
|
|
4
|
-
* Single shared implementation used by
|
|
5
|
-
*
|
|
4
|
+
* Single shared implementation used by the Whiteboard, Remote Browser and any other channel that
|
|
5
|
+
* shows the model pixels. It no longer writes to the model directly: it REGISTERS its channel as a
|
|
6
|
+
* source with the session's {@link VideoSourceArbiter} and pushes frames to it, so the arbiter stays
|
|
7
|
+
* the single writer of inbound video and decides which source the model sees when several are live.
|
|
6
8
|
*
|
|
7
9
|
* Enforces:
|
|
8
10
|
* 1. Model capability & track gate: if the session has not established an inbound video track,
|
|
9
11
|
* falls back to tool-only behaviour — NO frames sent and NO errors raised.
|
|
10
|
-
* 2. Cadence
|
|
11
|
-
*
|
|
12
|
+
* 2. Cadence: the poller (when used) runs at the NEGOTIATED inbound rate, read from the client;
|
|
13
|
+
* there is no rate constant here. The arbiter and the driver enforce the same rate as a spacing.
|
|
14
|
+
* 3. Clean teardown on Stop(), which also unregisters the source.
|
|
15
|
+
*
|
|
16
|
+
* An owner that sends frames only when its picture changes learns from {@link ChannelInboundVideoBridge.OnSourceActivated}
|
|
17
|
+
* when the arbiter starts sending its source to the model again, so it can send the picture the model missed.
|
|
12
18
|
*
|
|
13
19
|
* @module @memberjunction/ai-realtime-client
|
|
14
20
|
* @author MemberJunction.com
|
|
15
21
|
*/
|
|
22
|
+
import { NominalVideoFrameIntervalMs } from './videoPacing.js';
|
|
23
|
+
import { VideoSourceArbiter } from './videoSourceArbiter.js';
|
|
24
|
+
/** Counter behind the generated source ids. */
|
|
25
|
+
let generatedSourceCounter = 0;
|
|
16
26
|
/**
|
|
17
|
-
* Shared bridge connecting a visual channel surface to the session's inbound video track
|
|
27
|
+
* Shared bridge connecting a visual channel surface to the session's inbound video track, by way of
|
|
28
|
+
* the session's {@link VideoSourceArbiter}.
|
|
18
29
|
*/
|
|
19
30
|
export class ChannelInboundVideoBridge {
|
|
31
|
+
/**
|
|
32
|
+
* @param clientOrGetter The session's client, or a getter for it (a reconnect replaces the client). Any
|
|
33
|
+
* {@link IVideoFrameSink} works; a `BaseRealtimeClient` is one.
|
|
34
|
+
* @param provider Produces the frames.
|
|
35
|
+
* @param options Source identity, MIME type and the deprecated rate cap.
|
|
36
|
+
*/
|
|
20
37
|
constructor(clientOrGetter, provider, options = {}) {
|
|
21
38
|
this.clientOrGetter = clientOrGetter;
|
|
22
39
|
this.provider = provider;
|
|
@@ -24,10 +41,131 @@ export class ChannelInboundVideoBridge {
|
|
|
24
41
|
this.timer = null;
|
|
25
42
|
this.active = false;
|
|
26
43
|
this.frameErrorLogged = false;
|
|
44
|
+
/** The arbiter this bridge is currently registered with; `null` when it is not registered. */
|
|
45
|
+
this.registeredWith = null;
|
|
46
|
+
/** Whether the owner switched this source off (exposure policy or the user). Survives re-registration. */
|
|
47
|
+
this.sourceEnabled = true;
|
|
48
|
+
/** The owner's handlers for {@link OnSourceActivated}. They survive re-registration. */
|
|
49
|
+
this.activatedHandlers = new Set();
|
|
50
|
+
/** Stops following the arbiter's changes; `null` while not following (not registered, or no handler). */
|
|
51
|
+
this.stopFollowingArbiter = null;
|
|
52
|
+
/** Whether the arbiter was sending this source to the model as of its last change. */
|
|
53
|
+
this.forwarded = false;
|
|
54
|
+
this.sourceId = options.SourceID ?? `channel-video-${++generatedSourceCounter}`;
|
|
27
55
|
}
|
|
28
56
|
get client() {
|
|
29
57
|
return typeof this.clientOrGetter === 'function' ? this.clientOrGetter() : this.clientOrGetter;
|
|
30
58
|
}
|
|
59
|
+
/** The id this bridge's source is registered under. */
|
|
60
|
+
get SourceID() {
|
|
61
|
+
return this.sourceId;
|
|
62
|
+
}
|
|
63
|
+
/** The arbiter for the current client, or `null` when there is no client yet. */
|
|
64
|
+
resolveArbiter() {
|
|
65
|
+
if (this.options.Arbiter) {
|
|
66
|
+
return this.options.Arbiter;
|
|
67
|
+
}
|
|
68
|
+
const client = this.client;
|
|
69
|
+
return client ? VideoSourceArbiter.ForSink(client) : null;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Registers this bridge's source with the session's arbiter (idempotent). A new session means a new
|
|
73
|
+
* client and so a new arbiter: the source is re-registered there and removed from the old one, so a
|
|
74
|
+
* reconnect never leaves a ghost source behind.
|
|
75
|
+
*
|
|
76
|
+
* @returns The arbiter, or `null` when there is no client to register with.
|
|
77
|
+
*/
|
|
78
|
+
ensureRegistered() {
|
|
79
|
+
const arbiter = this.resolveArbiter();
|
|
80
|
+
if (!arbiter) {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
if (this.registeredWith !== arbiter) {
|
|
84
|
+
this.unfollowArbiter();
|
|
85
|
+
this.registeredWith?.UnregisterSource(this.sourceId);
|
|
86
|
+
arbiter.RegisterSource({
|
|
87
|
+
SourceID: this.sourceId,
|
|
88
|
+
Label: this.options.Label ?? this.sourceId,
|
|
89
|
+
Kind: this.options.Kind ?? 'surface',
|
|
90
|
+
ChannelKey: this.options.ChannelKey,
|
|
91
|
+
});
|
|
92
|
+
this.registeredWith = arbiter;
|
|
93
|
+
if (!this.sourceEnabled) {
|
|
94
|
+
arbiter.SetSourceEnabled(this.sourceId, false, false);
|
|
95
|
+
}
|
|
96
|
+
// Followed from here, once registered as it will stay: the registration itself is not an activation.
|
|
97
|
+
this.followArbiter();
|
|
98
|
+
}
|
|
99
|
+
return arbiter;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Calls `handler` each time the arbiter starts sending this source to the model after it was not: another source
|
|
103
|
+
* that held the model's only stream stopped or was set aside, or this source was switched back on. The source's
|
|
104
|
+
* first registration does not count. A frame offered while the source was not sent never reached the model, so an
|
|
105
|
+
* owner that sends frames only when its picture changes sends its current picture here.
|
|
106
|
+
*
|
|
107
|
+
* The handler is kept across a re-registration (a reconnect's new client) and called until removed.
|
|
108
|
+
*
|
|
109
|
+
* @param handler Called synchronously from the arbiter's change event; it should not throw.
|
|
110
|
+
* @returns A function that removes the handler.
|
|
111
|
+
*/
|
|
112
|
+
OnSourceActivated(handler) {
|
|
113
|
+
this.activatedHandlers.add(handler);
|
|
114
|
+
this.followArbiter();
|
|
115
|
+
return () => {
|
|
116
|
+
this.activatedHandlers.delete(handler);
|
|
117
|
+
if (this.activatedHandlers.size === 0) {
|
|
118
|
+
this.unfollowArbiter();
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
/** Starts following the arbiter this source is registered with, when an owner asked to hear of activations. */
|
|
123
|
+
followArbiter() {
|
|
124
|
+
const arbiter = this.registeredWith;
|
|
125
|
+
if (!arbiter || this.stopFollowingArbiter || this.activatedHandlers.size === 0) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
this.forwarded = arbiter.GetActiveSourceIDs().includes(this.sourceId);
|
|
129
|
+
this.stopFollowingArbiter = arbiter.OnChange(() => {
|
|
130
|
+
const forwarded = arbiter.GetActiveSourceIDs().includes(this.sourceId);
|
|
131
|
+
const activated = forwarded && !this.forwarded;
|
|
132
|
+
this.forwarded = forwarded;
|
|
133
|
+
if (activated) {
|
|
134
|
+
this.notifyActivated();
|
|
135
|
+
}
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
/** Stops following the arbiter's changes. */
|
|
139
|
+
unfollowArbiter() {
|
|
140
|
+
this.stopFollowingArbiter?.();
|
|
141
|
+
this.stopFollowingArbiter = null;
|
|
142
|
+
this.forwarded = false;
|
|
143
|
+
}
|
|
144
|
+
/** Calls each activation handler, containing a handler that throws. */
|
|
145
|
+
notifyActivated() {
|
|
146
|
+
for (const handler of [...this.activatedHandlers]) {
|
|
147
|
+
try {
|
|
148
|
+
handler();
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
console.error('[ChannelInboundVideoBridge] Error in a source-activated handler:', err);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Registers this bridge's source with the session's arbiter now, without starting any poll — so the
|
|
157
|
+
* source is LISTED (and can be switched on or off) before it has pushed a frame. A source that is
|
|
158
|
+
* switched off never pushes one, so without this it could never be switched back on.
|
|
159
|
+
*
|
|
160
|
+
* @returns `true` if registered; `false` when there is no client or no inbound video track (nothing to register with).
|
|
161
|
+
*/
|
|
162
|
+
Register() {
|
|
163
|
+
const client = this.client;
|
|
164
|
+
if (!client || !client.IsTrackEstablished('video', 'inbound')) {
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
return this.ensureRegistered() !== null;
|
|
168
|
+
}
|
|
31
169
|
/**
|
|
32
170
|
* Starts the periodic frame pump if the session has an established inbound video track.
|
|
33
171
|
*
|
|
@@ -38,43 +176,66 @@ export class ChannelInboundVideoBridge {
|
|
|
38
176
|
if (this.active) {
|
|
39
177
|
return true;
|
|
40
178
|
}
|
|
41
|
-
|
|
179
|
+
const client = this.client;
|
|
180
|
+
if (!client || !client.IsTrackEstablished('video', 'inbound')) {
|
|
42
181
|
// Non-video model or unrequested video track: graceful fallback with no error
|
|
43
182
|
return false;
|
|
44
183
|
}
|
|
184
|
+
this.ensureRegistered();
|
|
45
185
|
this.active = true;
|
|
46
|
-
const rate = Math.min(Math.max(this.options.Rate ?? 1, 0.1), 1);
|
|
47
|
-
const intervalMs = Math.floor(1000 / rate);
|
|
48
186
|
this.timer = setInterval(() => {
|
|
49
187
|
void this.pumpFrame();
|
|
50
|
-
},
|
|
188
|
+
}, this.pollIntervalMs(client));
|
|
51
189
|
return true;
|
|
52
190
|
}
|
|
191
|
+
/** The poll interval: the negotiated rate, capped by the (deprecated) `Rate` option when one was given. */
|
|
192
|
+
pollIntervalMs(client) {
|
|
193
|
+
const negotiated = client.InboundVideoRate;
|
|
194
|
+
const requested = this.options.Rate;
|
|
195
|
+
const rate = typeof requested === 'number' && requested > 0
|
|
196
|
+
? Math.max(0.1, typeof negotiated === 'number' ? Math.min(requested, negotiated) : requested)
|
|
197
|
+
: negotiated;
|
|
198
|
+
return NominalVideoFrameIntervalMs(rate);
|
|
199
|
+
}
|
|
53
200
|
/**
|
|
54
|
-
*
|
|
55
|
-
* and the
|
|
201
|
+
* Offers a frame to the arbiter, which forwards it only if this source is among those the model
|
|
202
|
+
* should see (and enabled, and inside the negotiated pacing interval).
|
|
56
203
|
*
|
|
57
|
-
* @returns `true` if dispatched to the
|
|
204
|
+
* @returns `true` if dispatched to the model; `false` otherwise.
|
|
58
205
|
*/
|
|
59
206
|
sendFrameDirect(base64Jpeg) {
|
|
60
|
-
|
|
207
|
+
const client = this.client;
|
|
208
|
+
if (!client || !client.IsTrackEstablished('video', 'inbound')) {
|
|
61
209
|
return false;
|
|
62
210
|
}
|
|
63
|
-
if (typeof
|
|
211
|
+
if (typeof client.SendVideoFrame !== 'function') {
|
|
64
212
|
return false;
|
|
65
213
|
}
|
|
66
|
-
|
|
214
|
+
const arbiter = this.ensureRegistered();
|
|
215
|
+
return arbiter ? arbiter.PushFrame(this.sourceId, base64Jpeg, this.options.MimeType ?? 'image/jpeg') : false;
|
|
67
216
|
}
|
|
68
217
|
/**
|
|
69
218
|
* Pushes an event-driven frame (e.g. screencast push from RemoteBrowser or stroke completion).
|
|
70
|
-
*
|
|
219
|
+
* Registers the source on first use, so a purely event-driven channel never needs {@link Start}.
|
|
71
220
|
*
|
|
72
221
|
* @param base64Jpeg The base64-encoded image frame.
|
|
73
|
-
* @returns `true` if the frame was sent; `false` if dropped (throttled,
|
|
222
|
+
* @returns `true` if the frame was sent; `false` if dropped (throttled, not the source the model is
|
|
223
|
+
* viewing, turned off, track unestablished, or driver lacks video support).
|
|
74
224
|
*/
|
|
75
225
|
PushFrame(base64Jpeg) {
|
|
76
226
|
return this.sendFrameDirect(base64Jpeg);
|
|
77
227
|
}
|
|
228
|
+
/**
|
|
229
|
+
* Turns this source on or off at the arbiter (a no-op before the first registration other than
|
|
230
|
+
* remembering the choice). A disabled source is not a candidate and forwards nothing.
|
|
231
|
+
*
|
|
232
|
+
* @param enabled Whether the model may see this source.
|
|
233
|
+
* @param notify Whether the arbiter tells the model. Pass `false` when the caller sends its own note.
|
|
234
|
+
*/
|
|
235
|
+
SetSourceEnabled(enabled, notify = true) {
|
|
236
|
+
this.sourceEnabled = enabled;
|
|
237
|
+
this.registeredWith?.SetSourceEnabled(this.sourceId, enabled, notify);
|
|
238
|
+
}
|
|
78
239
|
/**
|
|
79
240
|
* Periodic poll tick that fetches the latest frame from the provider and pushes it.
|
|
80
241
|
* Uses `sendFrameDirect` so periodic polling paced by `setInterval` does not fight
|
|
@@ -99,7 +260,8 @@ export class ChannelInboundVideoBridge {
|
|
|
99
260
|
}
|
|
100
261
|
}
|
|
101
262
|
/**
|
|
102
|
-
* Stops the bridge and
|
|
263
|
+
* Stops the bridge: clears the poll timer and removes this source from the arbiter. Activation handlers stay, and
|
|
264
|
+
* are called again if the source registers again.
|
|
103
265
|
*/
|
|
104
266
|
Stop() {
|
|
105
267
|
this.active = false;
|
|
@@ -108,6 +270,9 @@ export class ChannelInboundVideoBridge {
|
|
|
108
270
|
clearInterval(this.timer);
|
|
109
271
|
this.timer = null;
|
|
110
272
|
}
|
|
273
|
+
this.unfollowArbiter();
|
|
274
|
+
this.registeredWith?.UnregisterSource(this.sourceId);
|
|
275
|
+
this.registeredWith = null;
|
|
111
276
|
}
|
|
112
277
|
/** Whether the video bridge is currently actively pumping frames. */
|
|
113
278
|
get IsActive() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"channelVideoSource.js","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"channelVideoSource.js","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAA8C,MAAM,sBAAsB,CAAC;AA6CtG,+CAA+C;AAC/C,IAAI,sBAAsB,GAAG,CAAC,CAAC;AAE/B;;;GAGG;AACH,MAAM,OAAO,yBAAyB;IAgBlC;;;;;OAKG;IACH,YACqB,cAA+F,EAC/F,QAA+B,EAC/B,UAA4C,EAAE;QAF9C,mBAAc,GAAd,cAAc,CAAiF;QAC/F,aAAQ,GAAR,QAAQ,CAAuB;QAC/B,YAAO,GAAP,OAAO,CAAuC;QAxB3D,UAAK,GAA0C,IAAI,CAAC;QACpD,WAAM,GAAG,KAAK,CAAC;QACf,qBAAgB,GAAG,KAAK,CAAC;QAEjC,8FAA8F;QACtF,mBAAc,GAA8B,IAAI,CAAC;QACzD,0GAA0G;QAClG,kBAAa,GAAG,IAAI,CAAC;QAC7B,wFAAwF;QACvE,sBAAiB,GAAG,IAAI,GAAG,EAAc,CAAC;QAC3D,yGAAyG;QACjG,yBAAoB,GAAwB,IAAI,CAAC;QACzD,sFAAsF;QAC9E,cAAS,GAAG,KAAK,CAAC;QAatB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,EAAE,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED,IAAY,MAAM;QACd,OAAO,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;IACnG,CAAC;IAED,uDAAuD;IACvD,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED,iFAAiF;IACzE,cAAc;QAClB,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,OAAO,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,KAAK,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,CAAC,cAAc,CAAC;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ;gBAC1C,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS;gBACpC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;aACtC,CAAC,CAAC;YACH,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,qGAAqG;YACrG,IAAI,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACI,iBAAiB,CAAC,OAAmB;QACxC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,OAAO,GAAG,EAAE;YACR,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3B,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED,+GAA+G;IACvG,aAAa;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QACpC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7E,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtE,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvE,MAAM,SAAS,GAAG,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAC/C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;YAC3B,IAAI,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3B,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,6CAA6C;IACrC,eAAe;QACnB,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC9B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,uEAAuE;IAC/D,eAAe;QACnB,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC;gBACD,OAAO,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,kEAAkE,EAAE,GAAG,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACI,QAAQ;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,EAAE,KAAK,IAAI,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACI,KAAK;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,8EAA8E;YAC9E,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1B,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QAC1B,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,2GAA2G;IACnG,cAAc,CAAC,MAAuB;QAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QACpC,MAAM,IAAI,GACN,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7F,CAAC,CAAC,UAAU,CAAC;QACrB,OAAO,2BAA2B,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,UAAkB;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAC9C,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjH,CAAC;IAED;;;;;;;OAOG;IACI,SAAS,CAAC,UAAkB;QAC/B,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB,CAAC,OAAgB,EAAE,MAAM,GAAG,IAAI;QACnD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,CAAC,KAAK,CAAC,0GAA0G,EAAE,GAAG,CAAC,CAAC;YACnI,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,IAAI;QACP,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC/B,CAAC;IAED,qEAAqE;IACrE,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ"}
|