@pexip/media 17.1.2 → 17.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/dist/index.d.ts +100 -2
- package/dist/index.mjs +215 -19
- package/package.json +6 -6
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,31 @@
|
|
|
1
1
|
# @pexip/media
|
|
2
2
|
|
|
3
|
+
## 17.2.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 0a58ae61: Stablize constraints after release
|
|
8
|
+
- 26878b60: Add `contentHint` to constraints
|
|
9
|
+
|
|
10
|
+
- Add a new constraint: `contentHint`
|
|
11
|
+
|
|
12
|
+
- 7e02391e9c: Changest to support FECC and PTZ capabilities
|
|
13
|
+
- 244cb04c: Add `createGetDisplayMedia`
|
|
14
|
+
|
|
15
|
+
### Patch Changes
|
|
16
|
+
|
|
17
|
+
- 4ea55a5205: Prettier and linting for new prettier version.
|
|
18
|
+
- Updated dependencies [244cb04c]
|
|
19
|
+
- Updated dependencies [26878b60]
|
|
20
|
+
- Updated dependencies [c418c778]
|
|
21
|
+
- Updated dependencies [244cb04c]
|
|
22
|
+
- Updated dependencies [7e02391e9c]
|
|
23
|
+
- Updated dependencies [4ea55a5205]
|
|
24
|
+
- @pexip/media-control@17.2.0
|
|
25
|
+
- @pexip/media-processor@17.2.0
|
|
26
|
+
- @pexip/utils@16.8.0
|
|
27
|
+
- @pexip/signal@16.6.0
|
|
28
|
+
|
|
3
29
|
## 17.1.2
|
|
4
30
|
|
|
5
31
|
### Patch Changes
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import * as _pexip_media_control from '@pexip/media-control';
|
|
2
|
-
import { MediaDeviceRequest, MediaDeviceInfoLike, InputConstraintSet } from '@pexip/media-control';
|
|
2
|
+
import { MediaDeviceRequest, MediaDeviceInfoLike, InputConstraintSet, DisplayMediaOptions } from '@pexip/media-control';
|
|
3
3
|
import { RenderEffects, SegmentationModel, RenderParams, Segmenter, ThrottleOptions, AudioGraphOptions, AudioNodeInit, VideoProcessor, SegmentationTransform } from '@pexip/media-processor';
|
|
4
4
|
import { Signal, SignalVariant } from '@pexip/signal';
|
|
5
5
|
import { AsyncQueueOptions } from '@pexip/utils';
|
|
@@ -18,6 +18,10 @@ interface ExtendedMediaTrackSettings extends MediaTrackSettings {
|
|
|
18
18
|
foregroundThreshold?: number;
|
|
19
19
|
videoSegmentation?: RenderEffects;
|
|
20
20
|
videoSegmentationModel?: SegmentationModel;
|
|
21
|
+
pan?: boolean;
|
|
22
|
+
tilt?: boolean;
|
|
23
|
+
zoom?: boolean;
|
|
24
|
+
contentHint?: AudioContentHint | VideoContentHint;
|
|
21
25
|
}
|
|
22
26
|
type ExtendedMediaTrackSettingsKey = keyof ExtendedMediaTrackSettings;
|
|
23
27
|
interface MediaSettings {
|
|
@@ -429,6 +433,9 @@ type VideoRenderParams = Omit<RenderParams, 'backgroundImage' | 'effects'> & {
|
|
|
429
433
|
* Render Effects
|
|
430
434
|
*/
|
|
431
435
|
videoSegmentation: RenderEffects;
|
|
436
|
+
pan?: boolean;
|
|
437
|
+
tilt?: boolean;
|
|
438
|
+
zoom?: boolean;
|
|
432
439
|
};
|
|
433
440
|
interface StreamTrackSignals {
|
|
434
441
|
/**
|
|
@@ -472,6 +479,86 @@ declare enum DeniedDevices {
|
|
|
472
479
|
interface Segmenters {
|
|
473
480
|
mediapipeSelfie: Segmenter;
|
|
474
481
|
}
|
|
482
|
+
/**
|
|
483
|
+
* Audio content hints are only applicable when the MediaStreamTrack contains an
|
|
484
|
+
* audio track
|
|
485
|
+
*
|
|
486
|
+
* {@link https://www.w3.org/TR/mst-content-hint/#audio-content-hints}
|
|
487
|
+
*/
|
|
488
|
+
declare const AUDIO_CONTENT_HINTS: {
|
|
489
|
+
/**
|
|
490
|
+
* No hint has been provided, the implementation should make its
|
|
491
|
+
* best-informed guess on how to handle contained audio data. This may be
|
|
492
|
+
* inferred from how the track was opened or by doing content analysis
|
|
493
|
+
*/
|
|
494
|
+
readonly NoHint: "";
|
|
495
|
+
/**
|
|
496
|
+
* The track should be treated as if it contains speech data. Consuming this
|
|
497
|
+
* signal it may be appropriate to apply noise suppression or boost
|
|
498
|
+
* intelligibility of the incoming signal.
|
|
499
|
+
*/
|
|
500
|
+
readonly Speech: "speech";
|
|
501
|
+
/**
|
|
502
|
+
* The track should be treated as if it contains data for the purpose of
|
|
503
|
+
* speech recognition by a machine. Consuming this signal it may be
|
|
504
|
+
* appropriate to boost intelligibility of the incoming signal for
|
|
505
|
+
* transcription and turn off audio-processing components that are used for
|
|
506
|
+
* human consumption.
|
|
507
|
+
*/
|
|
508
|
+
readonly SpeechRecognition: "speech-recognition";
|
|
509
|
+
/**
|
|
510
|
+
* The track should be treated as if it contains music data. Generally this
|
|
511
|
+
* might imply tuning or turning off audio-processing components that are
|
|
512
|
+
* used to process speech data to prevent the audio from being distorted.
|
|
513
|
+
*/
|
|
514
|
+
readonly Music: "music";
|
|
515
|
+
};
|
|
516
|
+
type AudioContentHint = (typeof AUDIO_CONTENT_HINTS)[keyof typeof AUDIO_CONTENT_HINTS];
|
|
517
|
+
/**
|
|
518
|
+
* Video content hints are only applicable when the MediaStreamTrack contains a
|
|
519
|
+
* video track.
|
|
520
|
+
*
|
|
521
|
+
* {@link https://www.w3.org/TR/mst-content-hint/#video-content-hints}
|
|
522
|
+
*/
|
|
523
|
+
declare const VIDEO_CONTENT_HINTS: {
|
|
524
|
+
/**
|
|
525
|
+
* No hint has been provided, the implementation should make its
|
|
526
|
+
* best-informed guess on how contained video content should be treated.
|
|
527
|
+
* This can for example be inferred from how the track was opened or by
|
|
528
|
+
* doing content analysis.
|
|
529
|
+
*/
|
|
530
|
+
readonly NoHint: "";
|
|
531
|
+
/**
|
|
532
|
+
* The track should be treated as if it contains video where motion is
|
|
533
|
+
* important. This is normally webcam video, movies or video games.
|
|
534
|
+
* Quantization artefacts and downscaling are acceptable in order to
|
|
535
|
+
* preserve motion as well as possible while still retaining target
|
|
536
|
+
* bitrates. During low bitrates when compromises have to be made, more
|
|
537
|
+
* effort is spent on preserving frame rate than edge quality and details.
|
|
538
|
+
*/
|
|
539
|
+
readonly Motion: "motion";
|
|
540
|
+
/**
|
|
541
|
+
* The track should be treated as if video details are extra important.
|
|
542
|
+
* This is generally applicable to presentations or web pages with text
|
|
543
|
+
* content, painting or line art. This setting would normally optimize for
|
|
544
|
+
* detail in the resulting individual frames rather than smooth playback.
|
|
545
|
+
* Artefacts from quantization or downscaling that make small text or line
|
|
546
|
+
* art unintelligible should be avoided.
|
|
547
|
+
*/
|
|
548
|
+
readonly Detail: "detail";
|
|
549
|
+
/**
|
|
550
|
+
* The track should be treated as if video details are extra important, and
|
|
551
|
+
* that significant sharp edges and areas of consistent color can occur
|
|
552
|
+
* frequently. This is generally applicable to presentations or web pages
|
|
553
|
+
* with text content. This setting would normally optimize for detail in the
|
|
554
|
+
* resulting individual frames rather than smooth playback, and may take
|
|
555
|
+
* advantage of encoder tools that optimize for text rendering. Artefacts
|
|
556
|
+
* from quantization or downscaling that make small text or line art
|
|
557
|
+
* unintelligible should be avoided.
|
|
558
|
+
*/
|
|
559
|
+
readonly Text: "text";
|
|
560
|
+
};
|
|
561
|
+
type VideoContentHint = (typeof VIDEO_CONTENT_HINTS)[keyof typeof VIDEO_CONTENT_HINTS];
|
|
475
562
|
|
|
476
563
|
/**
|
|
477
564
|
* Create an object to interact with the media scream, which is usually used for
|
|
@@ -731,9 +818,20 @@ declare const REQUIRED_SIGNAL_KEYS: readonly ["onMediaChanged", "onVAD", "onSile
|
|
|
731
818
|
*/
|
|
732
819
|
declare const createMediaSignals: <K extends "onDevicesChanged" | "onStatusChanged" | keyof StreamTrackSignals>(more: K[], scope?: string) => Pick<Required<MediaSignals>, "onMediaChanged" | "onVAD" | "onSilentDetected" | K>;
|
|
733
820
|
|
|
821
|
+
/**
|
|
822
|
+
* Apply the content hint to the track
|
|
823
|
+
*
|
|
824
|
+
* @param hint - Content hint
|
|
825
|
+
* @param track - The track to be applied
|
|
826
|
+
*/
|
|
827
|
+
declare const applyContentHint: <T extends "" | "speech" | "speech-recognition" | "music" | "motion" | "detail" | "text">(hint?: T | undefined) => (track: MediaStreamTrack) => void;
|
|
828
|
+
|
|
734
829
|
/**
|
|
735
830
|
* Create a Audio Mixing Processor and will own the stream passed-in
|
|
736
831
|
*/
|
|
737
832
|
declare const createAudioMixingProcess: (getCurrrentMedia: () => MediaStream | undefined, scope?: string) => Process<Promise<Media>>;
|
|
738
833
|
|
|
739
|
-
|
|
834
|
+
declare const createGetDisplayMedia: (getDefaultConstraints: () => DisplayMediaOptions, getDisplayMedia?: MediaDevices['getDisplayMedia']) => (constraints?: DisplayMediaOptions) => Promise<MediaStream | undefined>;
|
|
835
|
+
type GetDisplayMedia = ReturnType<typeof createGetDisplayMedia>;
|
|
836
|
+
|
|
837
|
+
export { AUDIO_CONTENT_HINTS, AudioContentHint, AudioDetectionSignals, AudioSignalDetectionOptions, CreatePreviewStreamController, DeniedDevices, DenoiseParams, ExtendedMediaTrackSettings, ExtendedMediaTrackSettingsKey, GetDisplayMedia, Media, MediaAttributes, MediaChangesSignals, MediaController, MediaOptions, MediaProcessor, MediaProps, MediaSettings, MediaSignals, MediaSignalsOptional, MediaSignalsRequired, Pipeline, PreviewControllerProps, PreviewEventHandler, PreviewInput, PreviewStreamController, PreviewStreamParams, Process, ProcessMedia, REQUIRED_SIGNAL_KEYS, Segmenters, StreamTrackSignals, Unsubscribe, UserMediaStatus, UserMediaValidator, VIDEO_CONTENT_HINTS, VideoContentHint, VideoRenderParams, VideoStreamTrackProcessorAPIs, VoiceActivityDetectionOptions, applyContentHint, areBothGranted, createAudioMixingProcess, createAudioStreamProcess, createGetDisplayMedia, createMedia, createMediaSignal, createMediaSignals, createPreviewStreamController, createVideoStreamProcess, deriveInitialPermissionStatus, getPermissionStatus, hasNoAudioDevices, hasNoDevice, hasNoVideoDevices, isAudioDeviceInUse, isDeviceInUse, isFallback, isFallbackAudio, isFallbackVideo, isGranted, isGrantedAudio, isGrantedOnlyAudio, isGrantedOnlyAudioNoVideoDevices, isGrantedOnlyVideo, isGrantedOnlyVideoNoAudioDevices, isGrantedVideo, isInitial, isInitialPermissions, isInitialPermissionsGranted, isInitialPermissionsNotGranted, isOnlyAudioError, isOnlyVideoError, isOverConstrained, isPromptAudio, isPromptVideo, isRejected, isRejectedOnlyAudio, isRejectedOnlyVideo, isUnknownError, isVideoDeviceInUse, setLogger, toDeniedDevices };
|
package/dist/index.mjs
CHANGED
|
@@ -69,6 +69,72 @@ var DeniedDevices = /* @__PURE__ */ ((DeniedDevices2) => {
|
|
|
69
69
|
DeniedDevices2["Both"] = "microphone-and-camera";
|
|
70
70
|
return DeniedDevices2;
|
|
71
71
|
})(DeniedDevices || {});
|
|
72
|
+
var AUDIO_CONTENT_HINTS = {
|
|
73
|
+
/**
|
|
74
|
+
* No hint has been provided, the implementation should make its
|
|
75
|
+
* best-informed guess on how to handle contained audio data. This may be
|
|
76
|
+
* inferred from how the track was opened or by doing content analysis
|
|
77
|
+
*/
|
|
78
|
+
NoHint: "",
|
|
79
|
+
/**
|
|
80
|
+
* The track should be treated as if it contains speech data. Consuming this
|
|
81
|
+
* signal it may be appropriate to apply noise suppression or boost
|
|
82
|
+
* intelligibility of the incoming signal.
|
|
83
|
+
*/
|
|
84
|
+
Speech: "speech",
|
|
85
|
+
/**
|
|
86
|
+
* The track should be treated as if it contains data for the purpose of
|
|
87
|
+
* speech recognition by a machine. Consuming this signal it may be
|
|
88
|
+
* appropriate to boost intelligibility of the incoming signal for
|
|
89
|
+
* transcription and turn off audio-processing components that are used for
|
|
90
|
+
* human consumption.
|
|
91
|
+
*/
|
|
92
|
+
SpeechRecognition: "speech-recognition",
|
|
93
|
+
/**
|
|
94
|
+
* The track should be treated as if it contains music data. Generally this
|
|
95
|
+
* might imply tuning or turning off audio-processing components that are
|
|
96
|
+
* used to process speech data to prevent the audio from being distorted.
|
|
97
|
+
*/
|
|
98
|
+
Music: "music"
|
|
99
|
+
};
|
|
100
|
+
var VIDEO_CONTENT_HINTS = {
|
|
101
|
+
/**
|
|
102
|
+
* No hint has been provided, the implementation should make its
|
|
103
|
+
* best-informed guess on how contained video content should be treated.
|
|
104
|
+
* This can for example be inferred from how the track was opened or by
|
|
105
|
+
* doing content analysis.
|
|
106
|
+
*/
|
|
107
|
+
NoHint: "",
|
|
108
|
+
/**
|
|
109
|
+
* The track should be treated as if it contains video where motion is
|
|
110
|
+
* important. This is normally webcam video, movies or video games.
|
|
111
|
+
* Quantization artefacts and downscaling are acceptable in order to
|
|
112
|
+
* preserve motion as well as possible while still retaining target
|
|
113
|
+
* bitrates. During low bitrates when compromises have to be made, more
|
|
114
|
+
* effort is spent on preserving frame rate than edge quality and details.
|
|
115
|
+
*/
|
|
116
|
+
Motion: "motion",
|
|
117
|
+
/**
|
|
118
|
+
* The track should be treated as if video details are extra important.
|
|
119
|
+
* This is generally applicable to presentations or web pages with text
|
|
120
|
+
* content, painting or line art. This setting would normally optimize for
|
|
121
|
+
* detail in the resulting individual frames rather than smooth playback.
|
|
122
|
+
* Artefacts from quantization or downscaling that make small text or line
|
|
123
|
+
* art unintelligible should be avoided.
|
|
124
|
+
*/
|
|
125
|
+
Detail: "detail",
|
|
126
|
+
/**
|
|
127
|
+
* The track should be treated as if video details are extra important, and
|
|
128
|
+
* that significant sharp edges and areas of consistent color can occur
|
|
129
|
+
* frequently. This is generally applicable to presentations or web pages
|
|
130
|
+
* with text content. This setting would normally optimize for detail in the
|
|
131
|
+
* resulting individual frames rather than smooth playback, and may take
|
|
132
|
+
* advantage of encoder tools that optimize for text rendering. Artefacts
|
|
133
|
+
* from quantization or downscaling that make small text or line art
|
|
134
|
+
* unintelligible should be avoided.
|
|
135
|
+
*/
|
|
136
|
+
Text: "text"
|
|
137
|
+
};
|
|
72
138
|
|
|
73
139
|
// src/status.ts
|
|
74
140
|
var isFallbackVideo = (status) => [
|
|
@@ -373,6 +439,18 @@ var isMedia = (value) => {
|
|
|
373
439
|
}
|
|
374
440
|
return false;
|
|
375
441
|
};
|
|
442
|
+
var isAudioContentHint = (value) => {
|
|
443
|
+
if (typeof value === "string" && Object.values(AUDIO_CONTENT_HINTS).includes(value)) {
|
|
444
|
+
return true;
|
|
445
|
+
}
|
|
446
|
+
return false;
|
|
447
|
+
};
|
|
448
|
+
var isVideoContentHint = (value) => {
|
|
449
|
+
if (typeof value === "string" && Object.values(VIDEO_CONTENT_HINTS).includes(value)) {
|
|
450
|
+
return true;
|
|
451
|
+
}
|
|
452
|
+
return false;
|
|
453
|
+
};
|
|
376
454
|
|
|
377
455
|
// src/utils.ts
|
|
378
456
|
var makeDeriveDeviceStatus = (constraints) => (audio, video, both) => {
|
|
@@ -599,7 +677,8 @@ var applyExtendedConstraints = (media, applyExtended) => (
|
|
|
599
677
|
var AUDIO_SETTINGS_KEYS = [
|
|
600
678
|
"denoise",
|
|
601
679
|
"vad",
|
|
602
|
-
"asd"
|
|
680
|
+
"asd",
|
|
681
|
+
"contentHint"
|
|
603
682
|
];
|
|
604
683
|
var VIDEO_SETTINGS_KEYS = [
|
|
605
684
|
"frameRate",
|
|
@@ -611,7 +690,8 @@ var VIDEO_SETTINGS_KEYS = [
|
|
|
611
690
|
"flipHorizontal",
|
|
612
691
|
"bgImageUrl",
|
|
613
692
|
"width",
|
|
614
|
-
"height"
|
|
693
|
+
"height",
|
|
694
|
+
"contentHint"
|
|
615
695
|
];
|
|
616
696
|
var MIXING_SETTINGS_KEYS = [
|
|
617
697
|
"mixWithAdditionalMedia"
|
|
@@ -667,6 +747,11 @@ var getBlurKernelSize = (percentage, height, max = 20) => {
|
|
|
667
747
|
}
|
|
668
748
|
return Math.min(Math.ceil(percentage * 1e-3 * height), max);
|
|
669
749
|
};
|
|
750
|
+
var applyContentHint = (hint) => (track) => {
|
|
751
|
+
if (hint !== void 0 && hint !== track.contentHint) {
|
|
752
|
+
track.contentHint = hint;
|
|
753
|
+
}
|
|
754
|
+
};
|
|
670
755
|
|
|
671
756
|
// src/userMedia.ts
|
|
672
757
|
var toErrorDeviceStatus = (error, deriveDeviceStatus) => {
|
|
@@ -1067,13 +1152,25 @@ var FEATURE_KEYS = [
|
|
|
1067
1152
|
"videoSegmentation",
|
|
1068
1153
|
"videoSegmentationModel",
|
|
1069
1154
|
"width",
|
|
1070
|
-
"height"
|
|
1155
|
+
"height",
|
|
1156
|
+
"pan",
|
|
1157
|
+
"tilt",
|
|
1158
|
+
"zoom",
|
|
1159
|
+
"contentHint"
|
|
1071
1160
|
];
|
|
1072
1161
|
var getVideoConstraints = extractConstraintsWithKeys2(FEATURE_KEYS);
|
|
1073
1162
|
var updateFeatureProps = (constraints, props) => {
|
|
1074
1163
|
const extracted = getVideoConstraints(constraints);
|
|
1075
1164
|
return FEATURE_KEYS.reduce((accm, key) => {
|
|
1076
1165
|
switch (key) {
|
|
1166
|
+
case "contentHint": {
|
|
1167
|
+
const [[feature] = []] = extracted[key];
|
|
1168
|
+
if (isVideoContentHint(feature) && props[key] !== feature) {
|
|
1169
|
+
props[key] = feature;
|
|
1170
|
+
return { ...accm, [key]: feature };
|
|
1171
|
+
}
|
|
1172
|
+
return accm;
|
|
1173
|
+
}
|
|
1077
1174
|
case "videoSegmentation": {
|
|
1078
1175
|
const [[feature] = []] = extracted[key];
|
|
1079
1176
|
if (isRenderEffects(feature) && props[key] !== feature) {
|
|
@@ -1117,7 +1214,10 @@ var updateFeatureProps = (constraints, props) => {
|
|
|
1117
1214
|
}
|
|
1118
1215
|
return accm;
|
|
1119
1216
|
}
|
|
1120
|
-
case "flipHorizontal":
|
|
1217
|
+
case "flipHorizontal":
|
|
1218
|
+
case "pan":
|
|
1219
|
+
case "tilt":
|
|
1220
|
+
case "zoom": {
|
|
1121
1221
|
const [feature] = extracted[key];
|
|
1122
1222
|
if (feature !== void 0 && props[key] !== feature) {
|
|
1123
1223
|
props[key] = feature;
|
|
@@ -1370,6 +1470,7 @@ var createVideoStreamProcess = ({
|
|
|
1370
1470
|
release,
|
|
1371
1471
|
getSettings: () => {
|
|
1372
1472
|
const { audio, video } = prevGetSettings();
|
|
1473
|
+
const contentHint = stream.getAudioTracks().at(0)?.contentHint ?? "";
|
|
1373
1474
|
const videoSettings = {
|
|
1374
1475
|
videoSegmentation: transformer.effects,
|
|
1375
1476
|
foregroundThreshold: transformer.foregroundThreshold,
|
|
@@ -1378,7 +1479,11 @@ var createVideoStreamProcess = ({
|
|
|
1378
1479
|
edgeBlurAmount: transformer.edgeBlurAmount,
|
|
1379
1480
|
flipHorizontal: transformer.flipHorizontal,
|
|
1380
1481
|
bgImageUrl: transformer.backgroundImage && props.bgImageUrl,
|
|
1381
|
-
videoSegmentationModel: transformer.segmenter.model
|
|
1482
|
+
videoSegmentationModel: transformer.segmenter.model,
|
|
1483
|
+
pan: props.pan,
|
|
1484
|
+
tilt: props.tilt,
|
|
1485
|
+
zoom: props.zoom,
|
|
1486
|
+
contentHint
|
|
1382
1487
|
};
|
|
1383
1488
|
const settings = {
|
|
1384
1489
|
audio,
|
|
@@ -1430,17 +1535,38 @@ var fetchDenoiseWasm = async (denoiseWasm, wasmURL) => {
|
|
|
1430
1535
|
}
|
|
1431
1536
|
return await (await fetch(wasmURL)).arrayBuffer();
|
|
1432
1537
|
};
|
|
1433
|
-
var FEATURE_KEYS2 = [
|
|
1538
|
+
var FEATURE_KEYS2 = [
|
|
1539
|
+
"denoise",
|
|
1540
|
+
"vad",
|
|
1541
|
+
"asd",
|
|
1542
|
+
"contentHint"
|
|
1543
|
+
];
|
|
1434
1544
|
var getAudioConstraints = extractConstraintsWithKeys3(FEATURE_KEYS2);
|
|
1435
1545
|
var updateFeatureProps2 = (constraints, props) => {
|
|
1436
1546
|
const extracted = getAudioConstraints(constraints);
|
|
1437
1547
|
return FEATURE_KEYS2.reduce((accm, key) => {
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1548
|
+
switch (key) {
|
|
1549
|
+
case "contentHint": {
|
|
1550
|
+
const [[feature] = []] = extracted[key];
|
|
1551
|
+
if (isAudioContentHint(feature) && props[key] !== feature) {
|
|
1552
|
+
props[key] = feature;
|
|
1553
|
+
return { ...accm, [key]: feature };
|
|
1554
|
+
}
|
|
1555
|
+
return accm;
|
|
1556
|
+
}
|
|
1557
|
+
case "denoise":
|
|
1558
|
+
case "asd":
|
|
1559
|
+
case "vad": {
|
|
1560
|
+
const [feature] = extracted[key];
|
|
1561
|
+
if (feature !== void 0 && props[key] !== feature) {
|
|
1562
|
+
props[key] = feature;
|
|
1563
|
+
return { ...accm, [key]: feature };
|
|
1564
|
+
}
|
|
1565
|
+
return accm;
|
|
1566
|
+
}
|
|
1567
|
+
default:
|
|
1568
|
+
return accm;
|
|
1442
1569
|
}
|
|
1443
|
-
return accm;
|
|
1444
1570
|
}, {});
|
|
1445
1571
|
};
|
|
1446
1572
|
var createAudioStreamProcess = ({
|
|
@@ -1710,10 +1836,12 @@ var createAudioStreamProcess = ({
|
|
|
1710
1836
|
const denoise = !!props.denoiseNode && source.hasConnectedTo(props.denoiseNode);
|
|
1711
1837
|
const asd = !!props.asd;
|
|
1712
1838
|
const vad = !!props.vad;
|
|
1839
|
+
const contentHint = stream.getAudioTracks().at(0)?.contentHint ?? "";
|
|
1713
1840
|
const audioSettings = {
|
|
1714
1841
|
denoise,
|
|
1715
1842
|
asd,
|
|
1716
|
-
vad
|
|
1843
|
+
vad,
|
|
1844
|
+
contentHint
|
|
1717
1845
|
};
|
|
1718
1846
|
const settings = {
|
|
1719
1847
|
audio: audio.map((settings2) => ({
|
|
@@ -1998,7 +2126,10 @@ var createMedia = ({
|
|
|
1998
2126
|
mediaProcessors,
|
|
1999
2127
|
getDefaultConstraints = () => ({})
|
|
2000
2128
|
}) => {
|
|
2001
|
-
const initMedia = (status = "initial" /* Initial */, devices = []) => buildMedia(
|
|
2129
|
+
const initMedia = (status = "initial" /* Initial */, devices = [], constraints) => buildMedia(
|
|
2130
|
+
() => ({ status, devices, constraints }),
|
|
2131
|
+
signals.onStatusChanged?.emit
|
|
2132
|
+
);
|
|
2002
2133
|
const _props = {
|
|
2003
2134
|
devices: [],
|
|
2004
2135
|
media: initMedia(),
|
|
@@ -2021,7 +2152,7 @@ var createMedia = ({
|
|
|
2021
2152
|
props.discardMedia = true;
|
|
2022
2153
|
}
|
|
2023
2154
|
const status = await deriveInitialPermissionStatus(props.media.status);
|
|
2024
|
-
props.media = initMedia(status, props.devices);
|
|
2155
|
+
props.media = initMedia(status, props.devices, props.media.constraints);
|
|
2025
2156
|
};
|
|
2026
2157
|
const mediaPipeline = createMediaPipeline([
|
|
2027
2158
|
createGetUserMediaProcess(
|
|
@@ -2065,6 +2196,19 @@ var createMedia = ({
|
|
|
2065
2196
|
await media.release();
|
|
2066
2197
|
await cleanup();
|
|
2067
2198
|
};
|
|
2199
|
+
const applyConstraints3 = async (constraints) => {
|
|
2200
|
+
await media.applyConstraints(constraints);
|
|
2201
|
+
const videoFeatures = updateFeatureProps(
|
|
2202
|
+
constraints.video,
|
|
2203
|
+
{}
|
|
2204
|
+
);
|
|
2205
|
+
const audioFeatures = updateFeatureProps2(
|
|
2206
|
+
constraints.audio,
|
|
2207
|
+
{}
|
|
2208
|
+
);
|
|
2209
|
+
media.stream?.getAudioTracks().forEach(applyContentHint(audioFeatures.contentHint));
|
|
2210
|
+
media.stream?.getVideoTracks().forEach(applyContentHint(videoFeatures.contentHint));
|
|
2211
|
+
};
|
|
2068
2212
|
logger2.debug(
|
|
2069
2213
|
{ finalAudioMute: audio, finalVideoMute: video },
|
|
2070
2214
|
"End of media pipeline"
|
|
@@ -2073,7 +2217,8 @@ var createMedia = ({
|
|
|
2073
2217
|
shallowCopy(media, {
|
|
2074
2218
|
release,
|
|
2075
2219
|
muteAudio,
|
|
2076
|
-
muteVideo
|
|
2220
|
+
muteVideo,
|
|
2221
|
+
applyConstraints: applyConstraints3
|
|
2077
2222
|
})
|
|
2078
2223
|
);
|
|
2079
2224
|
})
|
|
@@ -2096,12 +2241,16 @@ var createMedia = ({
|
|
|
2096
2241
|
audio: [prevAudioSettings],
|
|
2097
2242
|
video: [prevVideoSettings]
|
|
2098
2243
|
} = props.media.getSettings();
|
|
2099
|
-
const hasRequestedResolution = !props.devices.some(
|
|
2100
|
-
(device) => device.kind === "videoinput" && device.label
|
|
2101
|
-
) || isRequestedResolution2(constraints.video, props.media?.videoInput);
|
|
2102
2244
|
const videoFeatures = updateFeatureProps(constraints.video, {});
|
|
2103
2245
|
const audioFeatures = updateFeatureProps2(constraints.audio, {});
|
|
2104
2246
|
const mixingFeatures = updateFeatureProps3(constraints.audio, {});
|
|
2247
|
+
const hasRequestedResolution = ["blur", "overlay"].includes(
|
|
2248
|
+
videoFeatures.videoSegmentation ?? ""
|
|
2249
|
+
) || // Skip when there is a render effect to modify the video
|
|
2250
|
+
!props.devices.some(
|
|
2251
|
+
(device) => device.kind === "videoinput" && device.label
|
|
2252
|
+
) || // Skip when no authorized video found
|
|
2253
|
+
isRequestedResolution2(constraints.video, props.media?.videoInput);
|
|
2105
2254
|
const audioFeaturesChanged = hasSettingsChanged(AUDIO_SETTINGS_KEYS)(
|
|
2106
2255
|
prevAudioSettings,
|
|
2107
2256
|
audioFeatures
|
|
@@ -2114,6 +2263,10 @@ var createMedia = ({
|
|
|
2114
2263
|
prevAudioSettings,
|
|
2115
2264
|
mixingFeatures
|
|
2116
2265
|
);
|
|
2266
|
+
const ptzFeaturesChanges = hasSettingsChanged(["pan", "tilt", "zoom"])(
|
|
2267
|
+
prevVideoSettings,
|
|
2268
|
+
videoFeatures
|
|
2269
|
+
);
|
|
2117
2270
|
logger2.debug(
|
|
2118
2271
|
{
|
|
2119
2272
|
constraints,
|
|
@@ -2128,7 +2281,7 @@ var createMedia = ({
|
|
|
2128
2281
|
},
|
|
2129
2282
|
"Is currently streaming requested stream"
|
|
2130
2283
|
);
|
|
2131
|
-
if (shouldRequestAudio || shouldRequestVideo || !hasRequestedResolution || !props.media.stream) {
|
|
2284
|
+
if (shouldRequestAudio || shouldRequestVideo || ptzFeaturesChanges || !hasRequestedResolution || !props.media.stream) {
|
|
2132
2285
|
if (props.media) {
|
|
2133
2286
|
await props.media.release();
|
|
2134
2287
|
}
|
|
@@ -2136,6 +2289,8 @@ var createMedia = ({
|
|
|
2136
2289
|
props.discardMedia = false;
|
|
2137
2290
|
}
|
|
2138
2291
|
const media = await mediaPipeline.execute(constraints);
|
|
2292
|
+
media.stream?.getAudioTracks().forEach(applyContentHint(audioFeatures.contentHint));
|
|
2293
|
+
media.stream?.getVideoTracks().forEach(applyContentHint(videoFeatures.contentHint));
|
|
2139
2294
|
if (props.discardMedia) {
|
|
2140
2295
|
logger2.debug("Discard media");
|
|
2141
2296
|
return await media.release();
|
|
@@ -2656,13 +2811,54 @@ var createMediaSignals = (more, scope = "") => {
|
|
|
2656
2811
|
{}
|
|
2657
2812
|
);
|
|
2658
2813
|
};
|
|
2814
|
+
|
|
2815
|
+
// src/displayMedia.ts
|
|
2816
|
+
import {
|
|
2817
|
+
limitSharingMonitorInterface,
|
|
2818
|
+
stopMediaStream as stopMediaStream5
|
|
2819
|
+
} from "@pexip/media-control";
|
|
2820
|
+
import { cancellablePromise } from "@pexip/utils";
|
|
2821
|
+
var createGetDisplayMedia = (getDefaultConstraints, getDisplayMedia = (constraints) => navigator.mediaDevices.getDisplayMedia(constraints)) => {
|
|
2822
|
+
const props = {
|
|
2823
|
+
requesting: false
|
|
2824
|
+
};
|
|
2825
|
+
return async (constraints) => {
|
|
2826
|
+
const [getCancelableDisplayMedia, cancel] = cancellablePromise(
|
|
2827
|
+
getDisplayMedia,
|
|
2828
|
+
(stream) => {
|
|
2829
|
+
stopMediaStream5(stream);
|
|
2830
|
+
}
|
|
2831
|
+
);
|
|
2832
|
+
if (props.requesting) {
|
|
2833
|
+
cancel();
|
|
2834
|
+
}
|
|
2835
|
+
try {
|
|
2836
|
+
props.requesting = true;
|
|
2837
|
+
const mergedConstraints = {
|
|
2838
|
+
...getDefaultConstraints(),
|
|
2839
|
+
...constraints
|
|
2840
|
+
};
|
|
2841
|
+
const stream = await getCancelableDisplayMedia(mergedConstraints);
|
|
2842
|
+
if (!stream) {
|
|
2843
|
+
return;
|
|
2844
|
+
}
|
|
2845
|
+
return limitSharingMonitorInterface(mergedConstraints, stream);
|
|
2846
|
+
} finally {
|
|
2847
|
+
props.requesting = false;
|
|
2848
|
+
}
|
|
2849
|
+
};
|
|
2850
|
+
};
|
|
2659
2851
|
export {
|
|
2852
|
+
AUDIO_CONTENT_HINTS,
|
|
2660
2853
|
DeniedDevices,
|
|
2661
2854
|
REQUIRED_SIGNAL_KEYS,
|
|
2662
2855
|
UserMediaStatus,
|
|
2856
|
+
VIDEO_CONTENT_HINTS,
|
|
2857
|
+
applyContentHint,
|
|
2663
2858
|
areBothGranted,
|
|
2664
2859
|
createAudioMixingProcess,
|
|
2665
2860
|
createAudioStreamProcess,
|
|
2861
|
+
createGetDisplayMedia,
|
|
2666
2862
|
createMedia,
|
|
2667
2863
|
createMediaSignal,
|
|
2668
2864
|
createMediaSignals,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@pexip/media",
|
|
3
|
-
"version": "17.
|
|
3
|
+
"version": "17.2.0",
|
|
4
4
|
"description": "Home for media related stuff",
|
|
5
5
|
"homepage": "https://gitlab.com/pexip/zoo",
|
|
6
6
|
"bugs": "https://gitlab.com/pexip/zoo/issues",
|
|
@@ -32,15 +32,15 @@
|
|
|
32
32
|
"typecheck": "yarn tsc --noEmit -p ."
|
|
33
33
|
},
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"@pexip/media-control": "17.
|
|
36
|
-
"@pexip/media-processor": "17.
|
|
35
|
+
"@pexip/media-control": "17.2.0",
|
|
36
|
+
"@pexip/media-processor": "17.2.0",
|
|
37
37
|
"@pexip/signal": "16.6.0",
|
|
38
|
-
"@pexip/utils": "16.
|
|
38
|
+
"@pexip/utils": "16.8.0"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"@jest/globals": "^29.5.0",
|
|
42
|
-
"@pexip/bundler": "16.4.
|
|
43
|
-
"prettier": "^
|
|
42
|
+
"@pexip/bundler": "16.4.1",
|
|
43
|
+
"prettier": "^3.0.3",
|
|
44
44
|
"typescript": "~5.0.0"
|
|
45
45
|
},
|
|
46
46
|
"publishConfig": {
|