@pexip/media-processor 20.3.4 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/api-docs/README.mdx +438 -0
- package/api-docs/enumerations/AbortReason.mdx +5 -0
- package/api-docs/functions/avg.mdx +15 -0
- package/api-docs/functions/calculateDistance.mdx +18 -0
- package/api-docs/functions/closedCurve.mdx +27 -0
- package/api-docs/functions/copyByteBufferToFloatBuffer.mdx +16 -0
- package/api-docs/functions/createAnalyzerGraphNode.mdx +19 -0
- package/api-docs/functions/createAnalyzerSubscribableGraphNode.mdx +15 -0
- package/api-docs/functions/createAsyncCallbackLoop.mdx +35 -0
- package/api-docs/functions/createAudioContext.mdx +22 -0
- package/api-docs/functions/createAudioDestinationGraphNode.mdx +9 -0
- package/api-docs/functions/createAudioGraph.mdx +28 -0
- package/api-docs/functions/createAudioGraphProxy.mdx +14 -0
- package/api-docs/functions/createAudioSignalDetector.mdx +51 -0
- package/api-docs/functions/createAudioStats.mdx +19 -0
- package/api-docs/functions/createBenchmark.mdx +26 -0
- package/api-docs/functions/createCanvas.mdx +16 -0
- package/api-docs/functions/createCanvasTransform.mdx +14 -0
- package/api-docs/functions/createChannelMergerGraphNode.mdx +19 -0
- package/api-docs/functions/createChannelSplitterGraphNode.mdx +19 -0
- package/api-docs/functions/createDelayGraphNode.mdx +19 -0
- package/api-docs/functions/createDenoiseWorkletGraphNode.mdx +16 -0
- package/api-docs/functions/createFrameCallbackRequest.mdx +29 -0
- package/api-docs/functions/createGainGraphNode.mdx +15 -0
- package/api-docs/functions/createMediaElementSourceGraphNode.mdx +15 -0
- package/api-docs/functions/createSegmenter.mdx +14 -0
- package/api-docs/functions/createStreamDestinationGraphNode.mdx +15 -0
- package/api-docs/functions/createStreamSourceGraphNode.mdx +16 -0
- package/api-docs/functions/createVADetector.mdx +54 -0
- package/api-docs/functions/createVideoProcessor.mdx +14 -0
- package/api-docs/functions/createVideoTrackProcessor.mdx +9 -0
- package/api-docs/functions/createVideoTrackProcessorWithFallback.mdx +15 -0
- package/api-docs/functions/createVoiceDetectorFromProbability.mdx +15 -0
- package/api-docs/functions/createVoiceDetectorFromTimeData.mdx +19 -0
- package/api-docs/functions/createWindowedStats.mdx +23 -0
- package/api-docs/functions/curve.mdx +15 -0
- package/api-docs/functions/fromByteToFloat.mdx +27 -0
- package/api-docs/functions/fromFloatToByte.mdx +27 -0
- package/api-docs/functions/getAudioStats.mdx +19 -0
- package/api-docs/functions/getBezierCurveControlPoints.mdx +26 -0
- package/api-docs/functions/iou.mdx +19 -0
- package/api-docs/functions/isAnalyzerNodeInit.mdx +13 -0
- package/api-docs/functions/isAudioNode.mdx +13 -0
- package/api-docs/functions/isAudioNodeInit.mdx +13 -0
- package/api-docs/functions/isAudioParam.mdx +13 -0
- package/api-docs/functions/isClipping.mdx +18 -0
- package/api-docs/functions/isEqualSize.mdx +22 -0
- package/api-docs/functions/isLowVolume.mdx +19 -0
- package/api-docs/functions/isMono.mdx +22 -0
- package/api-docs/functions/isRenderEffects.mdx +13 -0
- package/api-docs/functions/isSilent.mdx +22 -0
- package/api-docs/functions/isVoiceActivity.mdx +29 -0
- package/api-docs/functions/labelComponents.mdx +20 -0
- package/api-docs/functions/line.mdx +15 -0
- package/api-docs/functions/loadScript.mdx +14 -0
- package/api-docs/functions/loadWasms.mdx +13 -0
- package/api-docs/functions/nearestPowerOfTwo.mdx +21 -0
- package/api-docs/functions/playVideo.mdx +16 -0
- package/api-docs/functions/pow.mdx +29 -0
- package/api-docs/functions/processAverageVolume.mdx +15 -0
- package/api-docs/functions/resumeAudioOnInterruption.mdx +23 -0
- package/api-docs/functions/resumeAudioOnUnmute.mdx +27 -0
- package/api-docs/functions/rms.mdx +15 -0
- package/api-docs/functions/round.mdx +23 -0
- package/api-docs/functions/scoreComponent.mdx +22 -0
- package/api-docs/functions/setVideoElementSrc.mdx +14 -0
- package/api-docs/functions/subscribeTimeoutAnalyzerNode.mdx +22 -0
- package/api-docs/functions/subscribeWorkletNode.mdx +28 -0
- package/api-docs/functions/sum.mdx +15 -0
- package/api-docs/functions/toDecibel.mdx +18 -0
- package/api-docs/interfaces/Analyzer.mdx +368 -0
- package/api-docs/interfaces/AnalyzerSubscribableOptions.mdx +11 -0
- package/api-docs/interfaces/AsyncAssets.mdx +7 -0
- package/api-docs/interfaces/AudioGraph.mdx +92 -0
- package/api-docs/interfaces/AudioGraphOptions.mdx +6 -0
- package/api-docs/interfaces/AudioNodeInit.mdx +127 -0
- package/api-docs/interfaces/AudioNodeProps.mdx +15 -0
- package/api-docs/interfaces/AudioStats.mdx +16 -0
- package/api-docs/interfaces/BackgroundImage.mdx +6 -0
- package/api-docs/interfaces/Benchmark.mdx +47 -0
- package/api-docs/interfaces/Clock.mdx +11 -0
- package/api-docs/interfaces/Denoise.mdx +50 -0
- package/api-docs/interfaces/Detector.mdx +81 -0
- package/api-docs/interfaces/Frame.mdx +11 -0
- package/api-docs/interfaces/Gain.mdx +227 -0
- package/api-docs/interfaces/ImageRecord.mdx +6 -0
- package/api-docs/interfaces/ImageSegmenterOptions.mdx +8 -0
- package/api-docs/interfaces/MPMask.mdx +145 -0
- package/api-docs/interfaces/Point.mdx +10 -0
- package/api-docs/interfaces/Process.mdx +46 -0
- package/api-docs/interfaces/ProcessEventClose.mdx +7 -0
- package/api-docs/interfaces/ProcessEventDebug.mdx +14 -0
- package/api-docs/interfaces/ProcessEventDestroy.mdx +7 -0
- package/api-docs/interfaces/ProcessEventOpen.mdx +14 -0
- package/api-docs/interfaces/ProcessEventProcess.mdx +14 -0
- package/api-docs/interfaces/ProcessEventUpdate.mdx +14 -0
- package/api-docs/interfaces/ProcessWorkerDebugMessage.mdx +6 -0
- package/api-docs/interfaces/ProcessWorkerEvent.mdx +12 -0
- package/api-docs/interfaces/ProcessWorkerEventDebug.mdx +12 -0
- package/api-docs/interfaces/ProcessWorkerEventOpened.mdx +5 -0
- package/api-docs/interfaces/ProcessWorkerEventProcessed.mdx +12 -0
- package/api-docs/interfaces/ProcessWorkerEventUpdated.mdx +5 -0
- package/api-docs/interfaces/ProcessorEvent.mdx +7 -0
- package/api-docs/interfaces/ProcessorOptions.mdx +16 -0
- package/api-docs/interfaces/ProcessorProcessOptions.mdx +6 -0
- package/api-docs/interfaces/Rect.mdx +8 -0
- package/api-docs/interfaces/RenderEventHandlers.mdx +7 -0
- package/api-docs/interfaces/Renderer.mdx +64 -0
- package/api-docs/interfaces/RendererOptions.mdx +32 -0
- package/api-docs/interfaces/Runner.mdx +41 -0
- package/api-docs/interfaces/Segmentation.mdx +5 -0
- package/api-docs/interfaces/SegmentationModelAsset.mdx +8 -0
- package/api-docs/interfaces/SegmentationParams.mdx +35 -0
- package/api-docs/interfaces/SegmentationSmoothingConfig.mdx +5 -0
- package/api-docs/interfaces/Segmenter.mdx +121 -0
- package/api-docs/interfaces/SegmenterOptions.mdx +13 -0
- package/api-docs/interfaces/SelectionOptions.mdx +15 -0
- package/api-docs/interfaces/Size.mdx +10 -0
- package/api-docs/interfaces/Stats.mdx +11 -0
- package/api-docs/interfaces/StatsOptions.mdx +10 -0
- package/api-docs/interfaces/SubscribableOptions.mdx +9 -0
- package/api-docs/interfaces/ThrottleOptions.mdx +8 -0
- package/api-docs/interfaces/Transform.mdx +62 -0
- package/api-docs/interfaces/VideoFrameLike.mdx +8 -0
- package/api-docs/interfaces/VideoProcessor.mdx +38 -0
- package/api-docs/interfaces/Weights.mdx +12 -0
- package/api-docs/interfaces/WorkletMessagePortOptions.mdx +16 -0
- package/api-docs/interfaces/WorkletModule.mdx +6 -0
- package/api-docs/type-aliases/AnalyzerNodeInit.mdx +3 -0
- package/api-docs/type-aliases/AsyncCallback.mdx +7 -0
- package/api-docs/type-aliases/AudioBufferBytes.mdx +5 -0
- package/api-docs/type-aliases/AudioBufferFloats.mdx +6 -0
- package/api-docs/type-aliases/AudioDestinationNodeInit.mdx +3 -0
- package/api-docs/type-aliases/AudioNodeConnectParam.mdx +3 -0
- package/api-docs/type-aliases/AudioNodeInitConnectParam.mdx +3 -0
- package/api-docs/type-aliases/AudioNodeInitConnection.mdx +3 -0
- package/api-docs/type-aliases/AudioNodeInitConnections.mdx +3 -0
- package/api-docs/type-aliases/AudioNodeParam.mdx +3 -0
- package/api-docs/type-aliases/AudioSamples.mdx +17 -0
- package/api-docs/type-aliases/BaseAudioNode.mdx +3 -0
- package/api-docs/type-aliases/Callback.mdx +20 -0
- package/api-docs/type-aliases/Canvas.mdx +3 -0
- package/api-docs/type-aliases/CanvasContext.mdx +3 -0
- package/api-docs/type-aliases/ChannelSplitterNodeInit.mdx +3 -0
- package/api-docs/type-aliases/Color.mdx +12 -0
- package/api-docs/type-aliases/ConnectInitParamBaseType.mdx +3 -0
- package/api-docs/type-aliases/ConnectInitParamType.mdx +3 -0
- package/api-docs/type-aliases/ConnectParamBase.mdx +11 -0
- package/api-docs/type-aliases/ConnectParamBaseType.mdx +3 -0
- package/api-docs/type-aliases/ConnectParamType.mdx +3 -0
- package/api-docs/type-aliases/DelayNodeInit.mdx +3 -0
- package/api-docs/type-aliases/Delegate.mdx +3 -0
- package/api-docs/type-aliases/DenoiseWorkletNodeInit.mdx +3 -0
- package/api-docs/type-aliases/ExtractMessageEventType.mdx +9 -0
- package/api-docs/type-aliases/GainNodeInit.mdx +3 -0
- package/api-docs/type-aliases/ImageMapIterable.mdx +3 -0
- package/api-docs/type-aliases/ImageType.mdx +6 -0
- package/api-docs/type-aliases/IncludeMessageEventDataType.mdx +10 -0
- package/api-docs/type-aliases/InputFrame.mdx +3 -0
- package/api-docs/type-aliases/IsVoice.mdx +19 -0
- package/api-docs/type-aliases/MediaElementAudioSourceNodeInit.mdx +3 -0
- package/api-docs/type-aliases/MediaStreamAudioDestinationNodeInit.mdx +3 -0
- package/api-docs/type-aliases/MediaStreamAudioSourceNodeInit.mdx +3 -0
- package/api-docs/type-aliases/Node.mdx +3 -0
- package/api-docs/type-aliases/NodeConnectionAction.mdx +3 -0
- package/api-docs/type-aliases/Nodes.mdx +3 -0
- package/api-docs/type-aliases/OptionalKeys.mdx +10 -0
- package/api-docs/type-aliases/ProcessEvents.mdx +17 -0
- package/api-docs/type-aliases/ProcessInputType.mdx +3 -0
- package/api-docs/type-aliases/ProcessStatus.mdx +3 -0
- package/api-docs/type-aliases/ProcessVideoTrack.mdx +15 -0
- package/api-docs/type-aliases/ProcessWorkerEvents.mdx +18 -0
- package/api-docs/type-aliases/ProcessorEvents.mdx +3 -0
- package/api-docs/type-aliases/ProcessorWorkerEvents.mdx +3 -0
- package/api-docs/type-aliases/RenderBackend.mdx +3 -0
- package/api-docs/type-aliases/RenderEffects.mdx +3 -0
- package/api-docs/type-aliases/RenderingEvents.mdx +3 -0
- package/api-docs/type-aliases/RunnerCreator.mdx +21 -0
- package/api-docs/type-aliases/SegmentationModel.mdx +3 -0
- package/api-docs/type-aliases/SegmentationTransform.mdx +3 -0
- package/api-docs/type-aliases/TupleOf.mdx +12 -0
- package/api-docs/type-aliases/UniversalAudioContextState.mdx +6 -0
- package/api-docs/type-aliases/Unsubscribe.mdx +9 -0
- package/api-docs/type-aliases/WasmPaths.mdx +3 -0
- package/api-docs/variables/BACKGROUND_BLUR_AMOUNT.mdx +3 -0
- package/api-docs/variables/BACKGROUND_THRESHOLD.mdx +3 -0
- package/api-docs/variables/CLIP_COUNT_THRESHOLD.mdx +6 -0
- package/api-docs/variables/CLIP_THRESHOLD.mdx +5 -0
- package/api-docs/variables/DOWN_SAMPLE_FACTOR.mdx +3 -0
- package/api-docs/variables/EDGE_BLUR_AMOUNT.mdx +3 -0
- package/api-docs/variables/EXCLUDE_BYSTANDERS.mdx +3 -0
- package/api-docs/variables/FOREGROUND_THRESHOLD.mdx +3 -0
- package/api-docs/variables/FRAME_RATE.mdx +3 -0
- package/api-docs/variables/LIGHT_WRAP_BLUR_AMOUNT.mdx +3 -0
- package/api-docs/variables/LIGHT_WRAP_EDGE_BAND.mdx +3 -0
- package/api-docs/variables/LIGHT_WRAP_INTENSITY.mdx +3 -0
- package/api-docs/variables/LIGHT_WRAP_TIGHTNESS.mdx +3 -0
- package/api-docs/variables/LOW_VOLUME_THRESHOLD.mdx +5 -0
- package/api-docs/variables/MASK_COMBINE_RATIO.mdx +3 -0
- package/api-docs/variables/MONO_THRESHOLD.mdx +6 -0
- package/api-docs/variables/MORPH_DILATE_RADIUS_PX.mdx +3 -0
- package/api-docs/variables/MORPH_ERODE_RADIUS_PX.mdx +3 -0
- package/api-docs/variables/MORPH_PASS.mdx +3 -0
- package/api-docs/variables/PERSON_CENTER.mdx +3 -0
- package/api-docs/variables/PLAY_VIDEO_TIMEOUT.mdx +3 -0
- package/api-docs/variables/PROCESSING_HEIGHT.mdx +3 -0
- package/api-docs/variables/PROCESSING_WIDTH.mdx +3 -0
- package/api-docs/variables/PROCESS_STATUS.mdx +22 -0
- package/api-docs/variables/RENDERING_EVENTS.mdx +14 -0
- package/api-docs/variables/RENDER_BACKEND.mdx +10 -0
- package/api-docs/variables/RENDER_EFFECT.mdx +11 -0
- package/api-docs/variables/SEGMENTATION_MODELS.mdx +10 -0
- package/api-docs/variables/SIGMA_RANGE.mdx +3 -0
- package/api-docs/variables/SIGMA_SPACE.mdx +3 -0
- package/api-docs/variables/SILENT_THRESHOLD.mdx +6 -0
- package/api-docs/variables/STRONG_EDGE_BLUR_AMOUNT.mdx +3 -0
- package/api-docs/variables/VOICE_PROBABILITY_THRESHOLD.mdx +5 -0
- package/api-docs/variables/calculateMaxBlurPass.mdx +13 -0
- package/api-docs/variables/clamping.mdx +26 -0
- package/api-docs/variables/createLazyProps.mdx +19 -0
- package/api-docs/variables/createObjectUpdater.mdx +31 -0
- package/api-docs/variables/createRemoteImageBitmap.mdx +21 -0
- package/api-docs/variables/getCanvasRenderingContext2D.mdx +14 -0
- package/api-docs/variables/getErrorMessage.mdx +13 -0
- package/api-docs/variables/getImageSize.mdx +13 -0
- package/api-docs/variables/handleWebGLContextLoss.mdx +21 -0
- package/api-docs/variables/isRenderingEvents.mdx +13 -0
- package/api-docs/variables/isSegmentationModel.mdx +13 -0
- package/api-docs/variables/resize.mdx +30 -0
- package/api-docs/variables/toRenderingEvents.mdx +13 -0
- package/api-docs/variables/urls.mdx +10 -0
- package/dist/common/backends/webgl/binaryMask.d.ts +8 -0
- package/dist/common/backends/webgl/binaryMask.js +95 -0
- package/dist/common/backends/webgl/blender.d.ts +2 -1
- package/dist/common/backends/webgl/blender.js +18 -15
- package/dist/common/backends/webgl/blur.d.ts +1 -1
- package/dist/common/backends/webgl/blur.js +23 -16
- package/dist/common/backends/webgl/canvas.js +9 -4
- package/dist/common/backends/webgl/diffAbsMask.d.ts +10 -0
- package/dist/common/backends/webgl/diffAbsMask.js +103 -0
- package/dist/common/backends/webgl/geodesicReconstruction.d.ts +13 -0
- package/dist/common/backends/webgl/geodesicReconstruction.js +176 -0
- package/dist/common/backends/webgl/jointBilateralFilter.d.ts +12 -0
- package/dist/common/backends/webgl/jointBilateralFilter.js +108 -0
- package/dist/common/backends/webgl/morphErode.d.ts +13 -0
- package/dist/common/backends/webgl/morphErode.js +121 -0
- package/dist/common/backends/webgl/renderer.js +194 -34
- package/dist/common/backends/webgl/replace.d.ts +17 -0
- package/dist/common/backends/webgl/replace.js +126 -0
- package/dist/common/backends/webgl/selectComponent.d.ts +15 -0
- package/dist/common/backends/webgl/selectComponent.js +102 -0
- package/dist/common/backends/webgl/smoothingMask.d.ts +3 -3
- package/dist/common/backends/webgl/smoothingMask.js +22 -21
- package/dist/common/backends/webgl/tentBlur.d.ts +9 -0
- package/dist/common/backends/webgl/tentBlur.js +118 -0
- package/dist/common/backends/webgl/texture.d.ts +1 -1
- package/dist/common/backends/webgl/texture.js +19 -17
- package/dist/common/backends/webgl/textureToTexture.d.ts +1 -1
- package/dist/common/backends/webgl/textureToTexture.js +19 -16
- package/dist/common/backends/webgl/webglUtils.d.ts +7 -1
- package/dist/common/backends/webgl/webglUtils.js +64 -6
- package/dist/common/backends/webgpu/blender.d.ts +6 -1
- package/dist/common/backends/webgpu/blender.js +96 -51
- package/dist/common/backends/webgpu/constants.js +1 -1
- package/dist/common/backends/webgpu/dualFilterBlur.d.ts +2 -3
- package/dist/common/backends/webgpu/dualFilterBlur.js +146 -140
- package/dist/common/backends/webgpu/jointBilateralFilter.d.ts +7 -0
- package/dist/common/backends/webgpu/jointBilateralFilter.js +162 -0
- package/dist/common/backends/webgpu/renderer.js +211 -39
- package/dist/common/backends/webgpu/replace.d.ts +11 -0
- package/dist/common/backends/webgpu/replace.js +175 -0
- package/dist/common/backends/webgpu/resize.d.ts +4 -0
- package/dist/common/backends/webgpu/resize.js +76 -0
- package/dist/common/backends/webgpu/smoothingMask.d.ts +2 -2
- package/dist/common/backends/webgpu/smoothingMask.js +74 -40
- package/dist/common/backends/webgpu/tentBlur.d.ts +5 -0
- package/dist/common/backends/webgpu/tentBlur.js +131 -0
- package/dist/common/backends/webgpu/texture.d.ts +2 -2
- package/dist/common/backends/webgpu/texture.js +24 -19
- package/dist/common/backends/webgpu/textureToTexture.d.ts +1 -1
- package/dist/common/backends/webgpu/textureToTexture.js +28 -15
- package/dist/common/backends/webgpu/types.d.ts +2 -0
- package/dist/common/backends/webgpu/webgpuUtils.d.ts +24 -4
- package/dist/common/backends/webgpu/webgpuUtils.js +60 -11
- package/dist/common/constants.d.ts +19 -5
- package/dist/common/constants.js +19 -5
- package/dist/common/index.d.ts +1 -0
- package/dist/common/index.js +1 -0
- package/dist/common/tsconfig.tsbuildinfo +1 -1
- package/dist/common/types/render.d.ts +142 -13
- package/dist/common/types/utils.d.ts +21 -0
- package/dist/common/utils.d.ts +16 -0
- package/dist/common/utils.js +152 -0
- package/dist/main/benchUtils.d.ts +33 -1
- package/dist/main/benchUtils.js +145 -11
- package/dist/main/tsconfig.tsbuildinfo +1 -1
- package/dist/main/video/canvasTransform.d.ts +1 -1
- package/dist/main/video/canvasTransform.js +135 -47
- package/dist/main/video/index.d.ts +1 -1
- package/dist/main/video/index.js +1 -1
- package/dist/workers/mediaWorker.js +35 -162
- package/dist/workers/tsconfig.tsbuildinfo +1 -1
- package/package.json +16 -10
- package/dist/common/backends/webgl/shaders/blendFragmentShader.d.ts +0 -1
- package/dist/common/backends/webgl/shaders/blendFragmentShader.js +0 -21
- package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.d.ts +0 -1
- package/dist/common/backends/webgl/shaders/dualFilterBlurFragmentShader.js +0 -45
- package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.d.ts +0 -1
- package/dist/common/backends/webgl/shaders/smoothingMaskFragmentShader.js +0 -57
- package/dist/common/backends/webgl/shaders/textureFragmentShader.d.ts +0 -1
- package/dist/common/backends/webgl/shaders/textureFragmentShader.js +0 -16
- package/dist/common/backends/webgl/shaders/vertexShader.d.ts +0 -1
- package/dist/common/backends/webgl/shaders/vertexShader.js +0 -19
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { assert } from '@pexip/utils';
|
|
2
|
+
import { m3 } from '../../m3';
|
|
3
|
+
import { RENDERING_EVENTS } from '../../constants';
|
|
4
|
+
import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
|
|
5
|
+
import vertexSrc from './shaders/matrix.vert';
|
|
6
|
+
import fragmentSrc from './shaders/diffAbs.frag';
|
|
7
|
+
/**
|
|
8
|
+
* Create absulute difference renderer to caculate the difference between 2 textures
|
|
9
|
+
*/
|
|
10
|
+
export const createDiffAbsRenderer = (processor, width, height) => {
|
|
11
|
+
const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
|
|
12
|
+
attribs: {
|
|
13
|
+
position: 'a_position',
|
|
14
|
+
texCoord: 'a_texCoord',
|
|
15
|
+
},
|
|
16
|
+
uniforms: {
|
|
17
|
+
matrix: 'u_matrix',
|
|
18
|
+
texA: 'u_tex_a',
|
|
19
|
+
texB: 'u_tex_b',
|
|
20
|
+
size: 'u_size',
|
|
21
|
+
},
|
|
22
|
+
vertexSrc,
|
|
23
|
+
fragmentSrc,
|
|
24
|
+
});
|
|
25
|
+
const vao = gl.createVertexArray();
|
|
26
|
+
gl.bindVertexArray(vao);
|
|
27
|
+
// Position
|
|
28
|
+
const positionBuffer = gl.createBuffer();
|
|
29
|
+
gl.enableVertexAttribArray(attribLocations.position);
|
|
30
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
31
|
+
gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
32
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
33
|
+
// prettier-ignore
|
|
34
|
+
new Float32Array([
|
|
35
|
+
0, 0,
|
|
36
|
+
0, 1,
|
|
37
|
+
1, 0,
|
|
38
|
+
1, 0,
|
|
39
|
+
0, 1,
|
|
40
|
+
1, 1,
|
|
41
|
+
]), gl.STATIC_DRAW);
|
|
42
|
+
// Texture coordinates for the rectangle
|
|
43
|
+
const texCoordBuffer = gl.createBuffer();
|
|
44
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
45
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
46
|
+
// prettier-ignore
|
|
47
|
+
new Float32Array([
|
|
48
|
+
0, 0,
|
|
49
|
+
0, 1,
|
|
50
|
+
1, 0,
|
|
51
|
+
1, 0,
|
|
52
|
+
0, 1,
|
|
53
|
+
1, 1,
|
|
54
|
+
]), gl.STATIC_DRAW);
|
|
55
|
+
gl.enableVertexAttribArray(attribLocations.texCoord);
|
|
56
|
+
gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
57
|
+
const outputTexturePing = createMaskTexture(gl, width, height, true, floatSingleChannelSupported, floatFullChannelSupported);
|
|
58
|
+
const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
|
|
59
|
+
return {
|
|
60
|
+
render: (textureA, textureB) => {
|
|
61
|
+
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
62
|
+
assert(textureA.width === textureB.width &&
|
|
63
|
+
textureA.height === textureB.height);
|
|
64
|
+
gl.useProgram(program);
|
|
65
|
+
gl.bindVertexArray(vao);
|
|
66
|
+
gl.uniform2i(uniformLocations.size, textureA.width, textureA.height);
|
|
67
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
68
|
+
gl.uniform1i(uniformLocations.texA, 0);
|
|
69
|
+
gl.bindTexture(gl.TEXTURE_2D, textureA.texture);
|
|
70
|
+
gl.activeTexture(gl.TEXTURE1);
|
|
71
|
+
gl.uniform1i(uniformLocations.texB, 1);
|
|
72
|
+
gl.bindTexture(gl.TEXTURE_2D, textureB.texture);
|
|
73
|
+
// No Flip Y Coordinate for drawing on the frame buffer
|
|
74
|
+
// Project, position and scale to full-size of the canvas
|
|
75
|
+
const matrix = m3.projection(1, 1, 1);
|
|
76
|
+
gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
|
|
77
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBufferPing);
|
|
78
|
+
gl.viewport(0, 0, width, height);
|
|
79
|
+
gl.clearColor(0, 0, 0, 0);
|
|
80
|
+
gl.clear(gl.COLOR_BUFFER_BIT);
|
|
81
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
82
|
+
// Cleanup
|
|
83
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
84
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
85
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
86
|
+
gl.bindVertexArray(null);
|
|
87
|
+
return {
|
|
88
|
+
texture: outputTexturePing,
|
|
89
|
+
frameBuffer: frameBufferPing,
|
|
90
|
+
width,
|
|
91
|
+
height,
|
|
92
|
+
};
|
|
93
|
+
},
|
|
94
|
+
release: () => {
|
|
95
|
+
gl.deleteBuffer(positionBuffer);
|
|
96
|
+
gl.deleteBuffer(texCoordBuffer);
|
|
97
|
+
gl.deleteTexture(outputTexturePing);
|
|
98
|
+
gl.deleteFramebuffer(frameBufferPing);
|
|
99
|
+
gl.deleteVertexArray(vao);
|
|
100
|
+
release();
|
|
101
|
+
},
|
|
102
|
+
};
|
|
103
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RendererOptions } from '../../types/render';
|
|
2
|
+
import type { TextureInfo } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* A Geodesic Reconstruction Renderer that performs geodesic dilation of
|
|
5
|
+
* a mask constrained by a reference mask.
|
|
6
|
+
*/
|
|
7
|
+
export declare const createGeodesicReconstructionRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
|
|
8
|
+
render: ({ mask, maskRef }: {
|
|
9
|
+
mask: TextureInfo;
|
|
10
|
+
maskRef: TextureInfo;
|
|
11
|
+
}, options: Pick<RendererOptions, "morphDilateRadiusPx" | "morphPass">) => TextureInfo;
|
|
12
|
+
release: () => void;
|
|
13
|
+
};
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { assert } from '@pexip/utils';
|
|
2
|
+
import { m3 } from '../../m3';
|
|
3
|
+
import { RENDERING_EVENTS } from '../../constants';
|
|
4
|
+
import { clamping } from '../../utils';
|
|
5
|
+
import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
|
|
6
|
+
import vertexSrc from './shaders/matrix.vert';
|
|
7
|
+
import fragmentSrc from './shaders/dilateClipToMask.frag';
|
|
8
|
+
import { createDiffAbsRenderer } from './diffAbsMask';
|
|
9
|
+
// Convergence threshold used when testing for geodesic reconstruction stability.
|
|
10
|
+
const CONVERGENCE_MAX_ITERATIONS = 64;
|
|
11
|
+
const MAX_RADIUS_PX = 64;
|
|
12
|
+
const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
|
|
13
|
+
/**
|
|
14
|
+
* A Geodesic Reconstruction Renderer that performs geodesic dilation of
|
|
15
|
+
* a mask constrained by a reference mask.
|
|
16
|
+
*/
|
|
17
|
+
export const createGeodesicReconstructionRenderer = (processor, width, height) => {
|
|
18
|
+
const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
|
|
19
|
+
attribs: {
|
|
20
|
+
position: 'a_position',
|
|
21
|
+
texCoord: 'a_texCoord',
|
|
22
|
+
},
|
|
23
|
+
uniforms: {
|
|
24
|
+
matrix: 'u_matrix',
|
|
25
|
+
mask: 'u_mask',
|
|
26
|
+
maskRef: 'u_ref_mask',
|
|
27
|
+
texelSize: 'u_texel_size',
|
|
28
|
+
radiusPx: 'u_radius_px',
|
|
29
|
+
direction: 'u_direction',
|
|
30
|
+
},
|
|
31
|
+
vertexSrc,
|
|
32
|
+
fragmentSrc,
|
|
33
|
+
});
|
|
34
|
+
const vao = gl.createVertexArray();
|
|
35
|
+
gl.bindVertexArray(vao);
|
|
36
|
+
// Position
|
|
37
|
+
const positionBuffer = gl.createBuffer();
|
|
38
|
+
gl.enableVertexAttribArray(attribLocations.position);
|
|
39
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
40
|
+
gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
41
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
42
|
+
// prettier-ignore
|
|
43
|
+
new Float32Array([
|
|
44
|
+
0, 0,
|
|
45
|
+
0, 1,
|
|
46
|
+
1, 0,
|
|
47
|
+
1, 0,
|
|
48
|
+
0, 1,
|
|
49
|
+
1, 1,
|
|
50
|
+
]), gl.STATIC_DRAW);
|
|
51
|
+
// Texture coordinates for the rectangle
|
|
52
|
+
const texCoordBuffer = gl.createBuffer();
|
|
53
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
54
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
55
|
+
// prettier-ignore
|
|
56
|
+
new Float32Array([
|
|
57
|
+
0, 0,
|
|
58
|
+
0, 1,
|
|
59
|
+
1, 0,
|
|
60
|
+
1, 0,
|
|
61
|
+
0, 1,
|
|
62
|
+
1, 1,
|
|
63
|
+
]), gl.STATIC_DRAW);
|
|
64
|
+
gl.enableVertexAttribArray(attribLocations.texCoord);
|
|
65
|
+
gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
66
|
+
const scaleFactor = processor.width / width;
|
|
67
|
+
const MIN_CCL_WIDTH = 32;
|
|
68
|
+
const MIN_CCL_HEIGHT = 18;
|
|
69
|
+
const convergenceWidth = Math.max(Math.floor(width / 4), MIN_CCL_WIDTH);
|
|
70
|
+
const convergenceHeight = Math.max(Math.floor(height / 4), MIN_CCL_HEIGHT);
|
|
71
|
+
const CONVERGENCE_EPSILON = convergenceWidth === MIN_CCL_WIDTH ? 1e-8 : 1e-4;
|
|
72
|
+
const convergenceSize = convergenceWidth * convergenceHeight;
|
|
73
|
+
const diffRenderer = createDiffAbsRenderer(processor, convergenceWidth, convergenceHeight);
|
|
74
|
+
const supportFloat = floatSingleChannelSupported || floatFullChannelSupported;
|
|
75
|
+
const readPixelsFormat = supportFloat ? gl.RGBA : gl.RED;
|
|
76
|
+
const readPixelsType = supportFloat ? gl.FLOAT : gl.UNSIGNED_BYTE;
|
|
77
|
+
const convergenceDiff = supportFloat
|
|
78
|
+
? new Float32Array(convergenceSize * 4)
|
|
79
|
+
: new Uint8Array(convergenceSize);
|
|
80
|
+
const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
|
|
81
|
+
const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
|
|
82
|
+
const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
|
|
83
|
+
const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
|
|
84
|
+
return {
|
|
85
|
+
render: ({ mask, maskRef }, options) => {
|
|
86
|
+
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
87
|
+
let write = frameBufferPing;
|
|
88
|
+
let read = mask.texture;
|
|
89
|
+
const dilate = () => {
|
|
90
|
+
gl.useProgram(program);
|
|
91
|
+
gl.bindVertexArray(vao);
|
|
92
|
+
gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphDilateRadiusPx / scaleFactor)));
|
|
93
|
+
gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
|
|
94
|
+
// Reference Mask
|
|
95
|
+
gl.activeTexture(gl.TEXTURE1);
|
|
96
|
+
gl.uniform1i(uniformLocations.maskRef, 1);
|
|
97
|
+
gl.bindTexture(gl.TEXTURE_2D, maskRef.texture);
|
|
98
|
+
// No Flip Y Coordinate for drawing on the frame buffer
|
|
99
|
+
// Project, position and scale to full-size of the canvas
|
|
100
|
+
const matrix = m3.projection(1, 1, 1);
|
|
101
|
+
gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
|
|
102
|
+
// Separable
|
|
103
|
+
for (let j = 0; j < 2; j++) {
|
|
104
|
+
// Textures
|
|
105
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
106
|
+
gl.uniform1i(uniformLocations.mask, 0);
|
|
107
|
+
gl.bindTexture(gl.TEXTURE_2D, read);
|
|
108
|
+
// Horizontal then Vertical
|
|
109
|
+
gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
|
|
110
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, write);
|
|
111
|
+
gl.viewport(0, 0, width, height);
|
|
112
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
113
|
+
// Swap
|
|
114
|
+
[read, write] =
|
|
115
|
+
write === frameBufferPing
|
|
116
|
+
? [outputTexturePing, frameBufferPong]
|
|
117
|
+
: [outputTexturePong, frameBufferPing];
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
for (let i = 0; i < options.morphPass; i++) {
|
|
121
|
+
dilate();
|
|
122
|
+
}
|
|
123
|
+
let convergent = false;
|
|
124
|
+
let count = 0;
|
|
125
|
+
while (!convergent) {
|
|
126
|
+
dilate();
|
|
127
|
+
const diff = diffRenderer.render({ texture: outputTexturePing, width, height }, { texture: outputTexturePong, width, height });
|
|
128
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, diff.frameBuffer);
|
|
129
|
+
gl.readPixels(0, 0, convergenceWidth, convergenceHeight, readPixelsFormat, readPixelsType, convergenceDiff);
|
|
130
|
+
let sum = 0;
|
|
131
|
+
let meanDiff = 0;
|
|
132
|
+
if (supportFloat) {
|
|
133
|
+
for (let i = 0, j = 0; i < convergenceSize; ++i, j += 4) {
|
|
134
|
+
//@ts-expect-error will not be undefined
|
|
135
|
+
sum += convergenceDiff[j];
|
|
136
|
+
}
|
|
137
|
+
meanDiff = sum / convergenceSize;
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
for (let i = 0; i < convergenceSize; ++i) {
|
|
141
|
+
//@ts-expect-error will not be undefined
|
|
142
|
+
sum += convergenceDiff[i];
|
|
143
|
+
}
|
|
144
|
+
meanDiff = sum / convergenceSize / 255.0;
|
|
145
|
+
}
|
|
146
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
147
|
+
if (meanDiff < CONVERGENCE_EPSILON ||
|
|
148
|
+
count >= CONVERGENCE_MAX_ITERATIONS) {
|
|
149
|
+
// console.log(
|
|
150
|
+
// `Geodesic reconstruction converged in ${count} iterations with ${meanDiff}`,
|
|
151
|
+
// {convergenceWidth, convergenceHeight, width, height},
|
|
152
|
+
// );
|
|
153
|
+
convergent = true;
|
|
154
|
+
}
|
|
155
|
+
count++;
|
|
156
|
+
}
|
|
157
|
+
// Cleanup
|
|
158
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
159
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
160
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
161
|
+
gl.bindVertexArray(null);
|
|
162
|
+
return { texture: read, width, height };
|
|
163
|
+
},
|
|
164
|
+
release: () => {
|
|
165
|
+
gl.deleteBuffer(positionBuffer);
|
|
166
|
+
gl.deleteBuffer(texCoordBuffer);
|
|
167
|
+
gl.deleteTexture(outputTexturePing);
|
|
168
|
+
gl.deleteTexture(outputTexturePong);
|
|
169
|
+
gl.deleteFramebuffer(frameBufferPing);
|
|
170
|
+
gl.deleteFramebuffer(frameBufferPong);
|
|
171
|
+
gl.deleteVertexArray(vao);
|
|
172
|
+
diffRenderer.release();
|
|
173
|
+
release();
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { RendererOptions } from '../../types/render';
|
|
2
|
+
import type { TextureInfo } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Joint Bilateral Filter
|
|
5
|
+
*/
|
|
6
|
+
export declare const createJointBilateralFilterRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
|
|
7
|
+
render: ({ input, guide }: {
|
|
8
|
+
input: TextureInfo;
|
|
9
|
+
guide: TextureInfo;
|
|
10
|
+
}, options: RendererOptions) => TextureInfo;
|
|
11
|
+
release: () => void;
|
|
12
|
+
};
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { assert } from '@pexip/utils';
|
|
2
|
+
import { RENDERING_EVENTS } from '../../constants';
|
|
3
|
+
import { m3 } from '../../m3';
|
|
4
|
+
import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
|
|
5
|
+
import vertexSrc from './shaders/matrix.vert';
|
|
6
|
+
import fragmentSrc from './shaders/jointBilateralFilter.frag';
|
|
7
|
+
/**
|
|
8
|
+
* Joint Bilateral Filter
|
|
9
|
+
*/
|
|
10
|
+
export const createJointBilateralFilterRenderer = (processor, width, height) => {
|
|
11
|
+
const { gl, program, attribLocations, uniformLocations, release } = createGLProgram(processor, {
|
|
12
|
+
attribs: {
|
|
13
|
+
position: 'a_position',
|
|
14
|
+
texCoord: 'a_texCoord',
|
|
15
|
+
},
|
|
16
|
+
uniforms: {
|
|
17
|
+
frameGuide: 'u_frame_guide',
|
|
18
|
+
frameInput: 'u_frame_input',
|
|
19
|
+
matrix: 'u_matrix',
|
|
20
|
+
texelSize: 'u_texel_size',
|
|
21
|
+
sigmaSpace: 'u_sigma_space',
|
|
22
|
+
sigmaRangeLo: 'u_sigma_range_lo',
|
|
23
|
+
sigmaRangeHi: 'u_sigma_range_hi',
|
|
24
|
+
foregroundThreshold: 'u_foreground_threshold',
|
|
25
|
+
backgroundThreshold: 'u_background_threshold',
|
|
26
|
+
},
|
|
27
|
+
vertexSrc,
|
|
28
|
+
fragmentSrc,
|
|
29
|
+
});
|
|
30
|
+
gl.enable(gl.BLEND);
|
|
31
|
+
const vao = gl.createVertexArray();
|
|
32
|
+
gl.bindVertexArray(vao);
|
|
33
|
+
// Position
|
|
34
|
+
const positionBuffer = gl.createBuffer();
|
|
35
|
+
gl.enableVertexAttribArray(attribLocations.position);
|
|
36
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
37
|
+
gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
38
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
39
|
+
// prettier-ignore
|
|
40
|
+
new Float32Array([
|
|
41
|
+
0, 0,
|
|
42
|
+
0, 1,
|
|
43
|
+
1, 0,
|
|
44
|
+
1, 0,
|
|
45
|
+
0, 1,
|
|
46
|
+
1, 1,
|
|
47
|
+
]), gl.STATIC_DRAW);
|
|
48
|
+
// Texture coordinates for the rectangle
|
|
49
|
+
const texCoordBuffer = gl.createBuffer();
|
|
50
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
51
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
52
|
+
// prettier-ignore
|
|
53
|
+
new Float32Array([
|
|
54
|
+
0, 0,
|
|
55
|
+
0, 1,
|
|
56
|
+
1, 0,
|
|
57
|
+
1, 0,
|
|
58
|
+
0, 1,
|
|
59
|
+
1, 1,
|
|
60
|
+
]), gl.STATIC_DRAW);
|
|
61
|
+
gl.enableVertexAttribArray(attribLocations.texCoord);
|
|
62
|
+
gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
63
|
+
const outputTexture = createMaskTexture(gl, width, height);
|
|
64
|
+
const frameBuffer = createFrameBuffer(gl, outputTexture);
|
|
65
|
+
const scaleFactor = processor.width / width;
|
|
66
|
+
return {
|
|
67
|
+
render: ({ input, guide }, options) => {
|
|
68
|
+
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
69
|
+
assert(options.foregroundThreshold >= options.backgroundThreshold, 'foregroundThreshold must be >= backgroundThreshold');
|
|
70
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer);
|
|
71
|
+
gl.viewport(0, 0, width, height);
|
|
72
|
+
gl.useProgram(program);
|
|
73
|
+
gl.bindVertexArray(vao);
|
|
74
|
+
// Textures
|
|
75
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
76
|
+
gl.uniform1i(uniformLocations.frameInput, 0);
|
|
77
|
+
gl.bindTexture(gl.TEXTURE_2D, input.texture);
|
|
78
|
+
gl.activeTexture(gl.TEXTURE1);
|
|
79
|
+
gl.uniform1i(uniformLocations.frameGuide, 1);
|
|
80
|
+
gl.bindTexture(gl.TEXTURE_2D, guide.texture);
|
|
81
|
+
gl.uniform2f(uniformLocations.texelSize, 1.0 / input.width, 1.0 / input.height);
|
|
82
|
+
gl.uniform1f(uniformLocations.sigmaSpace, options.sigmaSpace / scaleFactor);
|
|
83
|
+
gl.uniform1f(uniformLocations.sigmaRangeLo, options.sigmaRangeLo);
|
|
84
|
+
gl.uniform1f(uniformLocations.sigmaRangeHi, options.sigmaRangeHi);
|
|
85
|
+
gl.uniform1f(uniformLocations.backgroundThreshold, options.backgroundThreshold);
|
|
86
|
+
gl.uniform1f(uniformLocations.foregroundThreshold, options.foregroundThreshold);
|
|
87
|
+
// No Flip Y Coordinate for drawing on the frame buffer
|
|
88
|
+
// Project, position and scale to full-size of the canvas
|
|
89
|
+
const matrix = m3.projection(1, 1, 1);
|
|
90
|
+
gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
|
|
91
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
92
|
+
// Cleanup
|
|
93
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
94
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
95
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
96
|
+
gl.bindVertexArray(null);
|
|
97
|
+
return { texture: outputTexture, width, height };
|
|
98
|
+
},
|
|
99
|
+
release: () => {
|
|
100
|
+
gl.deleteBuffer(positionBuffer);
|
|
101
|
+
gl.deleteBuffer(texCoordBuffer);
|
|
102
|
+
gl.deleteTexture(outputTexture);
|
|
103
|
+
gl.deleteFramebuffer(frameBuffer);
|
|
104
|
+
gl.deleteVertexArray(vao);
|
|
105
|
+
release();
|
|
106
|
+
},
|
|
107
|
+
};
|
|
108
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RendererOptions } from '../../types/render';
|
|
2
|
+
import type { TextureInfo } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* Create a morphological erosion renderer that performs erosion of a mask with a
|
|
5
|
+
* given radius and number of passes.
|
|
6
|
+
* The erosion is implemented as a separable filter.
|
|
7
|
+
*/
|
|
8
|
+
export declare const createMorphErodeRenderer: (processor: OffscreenCanvas, width: number, height: number) => {
|
|
9
|
+
render: ({ mask }: {
|
|
10
|
+
mask: TextureInfo;
|
|
11
|
+
}, options: Pick<RendererOptions, "morphPass" | "morphErodeRadiusPx">) => TextureInfo;
|
|
12
|
+
release: () => void;
|
|
13
|
+
};
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { assert } from '@pexip/utils';
|
|
2
|
+
import { m3 } from '../../m3';
|
|
3
|
+
import { RENDERING_EVENTS } from '../../constants';
|
|
4
|
+
import { clamping } from '../../utils';
|
|
5
|
+
import { createGLProgram, createMaskTexture, createFrameBuffer, } from './webglUtils';
|
|
6
|
+
import vertexSrc from './shaders/matrix.vert';
|
|
7
|
+
import fragmentSrc from './shaders/erode.frag';
|
|
8
|
+
const MAX_RADIUS_PX = 64;
|
|
9
|
+
const clampRadiusPx = clamping(0, MAX_RADIUS_PX);
|
|
10
|
+
/**
|
|
11
|
+
* Create a morphological erosion renderer that performs erosion of a mask with a
|
|
12
|
+
* given radius and number of passes.
|
|
13
|
+
* The erosion is implemented as a separable filter.
|
|
14
|
+
*/
|
|
15
|
+
export const createMorphErodeRenderer = (processor, width, height) => {
|
|
16
|
+
const { gl, program, attribLocations, uniformLocations, floatSingleChannelSupported, floatFullChannelSupported, release, } = createGLProgram(processor, {
|
|
17
|
+
attribs: {
|
|
18
|
+
position: 'a_position',
|
|
19
|
+
texCoord: 'a_texCoord',
|
|
20
|
+
},
|
|
21
|
+
uniforms: {
|
|
22
|
+
matrix: 'u_matrix',
|
|
23
|
+
mask: 'u_mask',
|
|
24
|
+
texelSize: 'u_texel_size',
|
|
25
|
+
radiusPx: 'u_radius_px',
|
|
26
|
+
direction: 'u_direction',
|
|
27
|
+
},
|
|
28
|
+
vertexSrc,
|
|
29
|
+
fragmentSrc,
|
|
30
|
+
});
|
|
31
|
+
const vao = gl.createVertexArray();
|
|
32
|
+
gl.bindVertexArray(vao);
|
|
33
|
+
// Position
|
|
34
|
+
const positionBuffer = gl.createBuffer();
|
|
35
|
+
gl.enableVertexAttribArray(attribLocations.position);
|
|
36
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
|
37
|
+
gl.vertexAttribPointer(attribLocations.position, 2, gl.FLOAT, false, 0, 0);
|
|
38
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
39
|
+
// prettier-ignore
|
|
40
|
+
new Float32Array([
|
|
41
|
+
0, 0,
|
|
42
|
+
0, 1,
|
|
43
|
+
1, 0,
|
|
44
|
+
1, 0,
|
|
45
|
+
0, 1,
|
|
46
|
+
1, 1,
|
|
47
|
+
]), gl.STATIC_DRAW);
|
|
48
|
+
// Texture coordinates for the rectangle
|
|
49
|
+
const texCoordBuffer = gl.createBuffer();
|
|
50
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);
|
|
51
|
+
gl.bufferData(gl.ARRAY_BUFFER,
|
|
52
|
+
// prettier-ignore
|
|
53
|
+
new Float32Array([
|
|
54
|
+
0, 0,
|
|
55
|
+
0, 1,
|
|
56
|
+
1, 0,
|
|
57
|
+
1, 0,
|
|
58
|
+
0, 1,
|
|
59
|
+
1, 1,
|
|
60
|
+
]), gl.STATIC_DRAW);
|
|
61
|
+
gl.enableVertexAttribArray(attribLocations.texCoord);
|
|
62
|
+
gl.vertexAttribPointer(attribLocations.texCoord, 2, gl.FLOAT, false, 0, 0);
|
|
63
|
+
const outputTexturePing = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
|
|
64
|
+
const frameBufferPing = createFrameBuffer(gl, outputTexturePing);
|
|
65
|
+
const outputTexturePong = createMaskTexture(gl, width, height, false, floatSingleChannelSupported, floatFullChannelSupported);
|
|
66
|
+
const frameBufferPong = createFrameBuffer(gl, outputTexturePong);
|
|
67
|
+
const scaleFactor = processor.width / width;
|
|
68
|
+
return {
|
|
69
|
+
render: ({ mask }, options) => {
|
|
70
|
+
assert(!gl.isContextLost(), RENDERING_EVENTS.ContextLost);
|
|
71
|
+
gl.clearColor(0, 0, 0, 0);
|
|
72
|
+
gl.clear(gl.COLOR_BUFFER_BIT);
|
|
73
|
+
gl.useProgram(program);
|
|
74
|
+
gl.bindVertexArray(vao);
|
|
75
|
+
gl.uniform2f(uniformLocations.texelSize, 1 / mask.width, 1 / mask.height);
|
|
76
|
+
gl.uniform1i(uniformLocations.radiusPx, clampRadiusPx(Math.floor(options.morphErodeRadiusPx / scaleFactor)));
|
|
77
|
+
const matrix = m3.projection(1, 1, 1);
|
|
78
|
+
// No Flip Y Coordinate for drawing on the frame buffer
|
|
79
|
+
// Project, position and scale to full-size of the canvas
|
|
80
|
+
gl.uniformMatrix3fv(uniformLocations.matrix, false, matrix);
|
|
81
|
+
let write = frameBufferPing;
|
|
82
|
+
let read = mask.texture;
|
|
83
|
+
// Ping-pong framebuffers for multiple passes
|
|
84
|
+
for (let i = 0; i < options.morphPass; i++) {
|
|
85
|
+
// Separable
|
|
86
|
+
for (let j = 0; j < 2; j++) {
|
|
87
|
+
// Textures
|
|
88
|
+
gl.activeTexture(gl.TEXTURE0);
|
|
89
|
+
gl.uniform1i(uniformLocations.mask, 0);
|
|
90
|
+
gl.bindTexture(gl.TEXTURE_2D, read);
|
|
91
|
+
// Horizontal then Vertical
|
|
92
|
+
gl.uniform2f(uniformLocations.direction, 1.0 - j, 0.0 + j);
|
|
93
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, write);
|
|
94
|
+
gl.viewport(0, 0, mask.width, mask.height);
|
|
95
|
+
gl.drawArrays(gl.TRIANGLES, 0, 6);
|
|
96
|
+
// Swap
|
|
97
|
+
[read, write] =
|
|
98
|
+
write === frameBufferPing
|
|
99
|
+
? [outputTexturePing, frameBufferPong]
|
|
100
|
+
: [outputTexturePong, frameBufferPing];
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Cleanup
|
|
104
|
+
gl.bindBuffer(gl.ARRAY_BUFFER, null);
|
|
105
|
+
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
|
|
106
|
+
gl.bindTexture(gl.TEXTURE_2D, null);
|
|
107
|
+
gl.bindVertexArray(null);
|
|
108
|
+
return { texture: read, width, height };
|
|
109
|
+
},
|
|
110
|
+
release: () => {
|
|
111
|
+
gl.deleteBuffer(positionBuffer);
|
|
112
|
+
gl.deleteBuffer(texCoordBuffer);
|
|
113
|
+
gl.deleteTexture(outputTexturePing);
|
|
114
|
+
gl.deleteTexture(outputTexturePong);
|
|
115
|
+
gl.deleteFramebuffer(frameBufferPing);
|
|
116
|
+
gl.deleteFramebuffer(frameBufferPong);
|
|
117
|
+
gl.deleteVertexArray(vao);
|
|
118
|
+
release();
|
|
119
|
+
},
|
|
120
|
+
};
|
|
121
|
+
};
|