@snap/camera-kit 0.13.2 → 0.14.1

package/lib/media-sources/FunctionSource.js

@@ -4,9 +4,7 @@ const getYUVImageData = (canvas, lensCore) => __awaiter(void 0, void 0, void 0,
     const { width, height } = canvas;
     // A YUV buffer has lower-res UV channels, so the total number of bytes works out like so:
     const outputBuffer = new ArrayBuffer((width * height * 3) / 2);
-    yield new Promise((onSuccess, onFailure) => {
-        lensCore.imageToYuvBuffer({ image: canvas, width, height, outputBuffer, onSuccess, onFailure });
-    });
+    yield lensCore.imageToYuvBuffer({ image: canvas, width, height, outputBuffer });
     const pixels = new Uint8ClampedArray(outputBuffer);
     return new ImageData(pixels, width, height);
 });
@@ -65,6 +63,7 @@ export const createFunctionSource = (sourceFunction, options = {}) => {
                     return;
                 try {
                     yield sourceFunction(({ format, imageData, timestampMillis }) => {
+                        // TODO: with processFrame promisified, we don't really need to create a wrapper promise
                         const frameOutput = new Promise((resolve, reject) => __awaiter(void 0, void 0, void 0, function* () {
                             const inputFrame = yield getImageBitmap(imageData, format !== null && format !== void 0 ? format : "rgb");
                             if (inputFrame.width !== width || inputFrame.height !== height) {
@@ -75,39 +74,37 @@ export const createFunctionSource = (sourceFunction, options = {}) => {
                                 // frame.
                                 source.setRenderSize(width, height);
                             }
-                            lensCore.processFrame({
-                                inputFrame,
-                                timestampMillis,
-                                onSuccess: () => __awaiter(void 0, void 0, void 0, function* () {
-                                    // Closing releases graphics resources associated with the frame, now that is
-                                    // has been processed.
-                                    inputFrame.close();
-                                    switch (format !== null && format !== void 0 ? format : "rgb") {
-                                        case "nv12":
-                                        case "yuv":
-                                            const [live, capture] = yield Promise.all([
-                                                getYUVImageData(output.live, lensCore),
-                                                getYUVImageData(output.capture, lensCore),
-                                            ]).catch((error) => {
-                                                reject(error);
-                                                return [undefined, undefined];
-                                            });
-                                            // if either of these is undefined, we'll have already rejected
-                                            // the promise, so we can return.
-                                            if (!live || !capture)
-                                                return;
-                                            return resolve({ live, capture });
-                                        case "rgb":
-                                            return resolve({
-                                                live: getRGBImageData(output.live, imageReader2D),
-                                                capture: getRGBImageData(output.capture, imageReader2D),
-                                            });
-                                    }
-                                }),
-                                onFailure: (error) => {
-                                    inputFrame.close();
-                                    reject(error);
-                                },
+                            lensCore
+                                .processFrame({ inputFrame, timestampMillis })
+                                .then(() => __awaiter(void 0, void 0, void 0, function* () {
+                                // Closing releases graphics resources associated with the frame, now that is
+                                // has been processed.
+                                inputFrame.close();
+                                switch (format !== null && format !== void 0 ? format : "rgb") {
+                                    case "nv12":
+                                    case "yuv":
+                                        const [live, capture] = yield Promise.all([
+                                            getYUVImageData(output.live, lensCore),
+                                            getYUVImageData(output.capture, lensCore),
+                                        ]).catch((error) => {
+                                            reject(error);
+                                            return [undefined, undefined];
+                                        });
+                                        // if either of these is undefined, we'll have already rejected
+                                        // the promise, so we can return.
+                                        if (!live || !capture)
+                                            return;
+                                        return resolve({ live, capture });
+                                    case "rgb":
+                                        return resolve({
+                                            live: getRGBImageData(output.live, imageReader2D),
+                                            capture: getRGBImageData(output.capture, imageReader2D),
+                                        });
+                                }
+                            }))
+                                .catch((error) => {
+                                inputFrame.close();
+                                reject(error);
                             });
                         }));
                         // Even if there's an error processing the frame, we do want to attempt to process the next

package/lib/media-sources/FunctionSource.js.map

@@ -1 +1 @@
-{"version":3,"file":"FunctionSource.js","sourceRoot":"","sources":["../../src/media-sources/FunctionSource.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,eAAe,EAAqD,MAAM,mBAAmB,CAAC;AAEvG,MAAM,eAAe,GAAG,CAAO,MAAyB,EAAE,QAAwB,EAAsB,EAAE;IACtG,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,0FAA0F;IAC1F,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,CAAC,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,IAAI,OAAO,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE;QACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACpG,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACnD,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC,CAAA,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAyB,EAAE,aAA8C,EAAa,EAAE;IAC7G,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,OAAO,aAAa,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;AACzE,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAO,SAAoB,EAAE,MAAmB,EAAwB,EAAE;IAC7F,QAAQ,MAAM,EAAE;QACZ,KAAK,MAAM,CAAC;QACZ,KAAK,KAAK;YACN,IAAI,CAAC,MAAM,CAAC,UAAU;gBAClB,OAAO,OAAO,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,sCAAsC,MAAM,oCAAoC,CAAC,CAC9F,CAAC;YACN,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE;gBAChD,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,SAAS,CAAC,KAAK;gBAC3B,WAAW,EAAE,SAAS,CAAC,MAAM;gBAC7B,SAAS,EAAE,CAAC;aACf,CAAC,CAAC;YACH,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACpC,KAAK,KAAK;YACN,OAAO,iBAAiB,CAAC,SAAS,CAAC,CAAC;KAC3C;AACL,CAAC,CAAA,CAAC;AAuBF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,cAAmC,EACnC,UAAoD,EAAE,EACvC,EAAE;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAE9B,kHAAkH;IAClH,6EAA6E;IAC7E,MAAM,iBAAiB,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAEzD,MAAM,UAAU,GAA8B;QAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG;gBACX,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;gBAChD,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;aACtD,CAAC;YAEF,MAAM,YAAY,GAAG,CACjB,MAAuB,EACvB,QAAwB,EACxB,WAAmC,EACrC,EAAE,CACA,qBAAqB,CAAC,GAAS,EAAE;;gBAC7B,IAAI,CAAC,kBAAkB;oBAAE,OAAO;gBAChC,IAAI;oBACA,MAAM,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,EAAE;wBAC5D,MAAM,WAAW,GAAG,IAAI,OAAO,CAAc,CAAO,OAAO,EAAE,MAAM,EAAE,EAAE;4BACnE,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,KAAK,CAAC,CAAC;4BACpE,IAAI,UAAU,CAAC,KAAK,KAAK,KAAK,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,EAAE;gCAC5D,KAAK,GAAG,iBAAiB,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;gCACnD,MAAM,GAAG,iBAAiB,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;gCACtD,iFAAiF;gCACjF,kFAAkF;gCAClF,SAAS;gCACT,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;6BACvC;4BACD,QAAQ,CAAC,YAAY,CAAC;gCAClB,UAAU;gCACV,eAAe;gCACf,SAAS,EAAE,GAAS,EAAE;oCAClB,6EAA6E;oCAC7E,sBAAsB;oCACtB,UAAU,CAAC,KAAK,EAAE,CAAC;oCAEnB,QAAQ,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,KAAK,EAAE;wCACrB,KAAK,MAAM,CAAC;wCACZ,KAAK,KAAK;4CACN,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;gDACtC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC;gDACtC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;6CAC5C,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gDACf,MAAM,CAAC,KAAK,CAAC,CAAC;gDACd,OAAO,CAAC,SAAS,EAAE,SAAS,CAAU,CAAC;4CAC3C,CAAC,CAAC,CAAC;4CACH,+DAA+D;4CAC/D,iCAAiC;4CACjC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;gDAAE,OAAO;4CAC9B,OAAO,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;wCACtC,KAAK,KAAK;4CACN,OAAO,OAAO,CAAC;gDACX,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;gDACjD,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;6CAC1D,CAAC,CAAC;qCACV;gCACL,CAAC,CAAA;gCACD,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE;oCACjB,UAAU,CAAC,KAAK,EAAE,CAAC;oCACnB,MAAM,CAAC,KAAK,CAAC,CAAC;gCAClB,CAAC;6BACJ,CAAC,CAAC;wBACP,CAAC,CAAA,CAAC,CAAC;wBACH,2FAA2F;wBAC3F,gFAAgF;wBAChF,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;wBACvE,OAAO,WAAW,CAAC;oBACvB,CAAC,CAAC,CAAC;iBACN;gBAAC,OAAO,KAAK,EAAE;oBACZ,WAAW,CACP,IAAI,KAAK,CACL,kEAAkE;wBAC9D,uBAAuB,MAAA,cAAc,CAAC,IAAI,mCAAI,WAAW,GAAG,EAChE,EAAE,KAAK,EAAE,KAAK,EAAE,CACnB,CACJ,CAAC;iBACL;YACL,CAAC,CAAA,CAAC,CAAC;YACP,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChD,CAAC;QACD,QAAQ,EAAE,GAAG,EAAE;YACX,kBAAkB,GAAG,KAAK,CAAC;QAC/B,CAAC;KACJ,CAAC;IAEF,OAAO,IAAI,eAAe,CAAC,EAAE,wBAAwB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AACxF,CAAC,CAAC","sourcesContent":["import { LensCoreModule } from \"../lens-core-module/generated-types\";\nimport { CameraKitSource, CameraKitSourceOptions, CameraKitSourceSubscriber } from \"./CameraKitSource\";\n\nconst getYUVImageData = async (canvas: HTMLCanvasElement, lensCore: LensCoreModule): Promise<ImageData> => {\n    const { width, height } = canvas;\n    // A YUV buffer has lower-res UV channels, so the total number of bytes works out like so:\n    const outputBuffer = new ArrayBuffer((width * height * 3) / 2);\n    await new Promise((onSuccess, onFailure) => {\n        lensCore.imageToYuvBuffer({ image: canvas, width, height, outputBuffer, onSuccess, onFailure });\n    });\n    const pixels = new Uint8ClampedArray(outputBuffer);\n    return new ImageData(pixels, width, height);\n};\n\nconst getRGBImageData = (output: HTMLCanvasElement, imageReader2D: CanvasRenderingContext2D | null): ImageData => {\n    if (imageReader2D === null) return new ImageData(0, 0);\n    imageReader2D.drawImage(output, 0, 0);\n    return imageReader2D.getImageData(0, 0, output.width, output.height);\n};\n\nconst getImageBitmap = async (imageData: ImageData, format: FrameFormat): Promise<ImageBitmap> => {\n    switch (format) {\n        case \"nv12\":\n        case \"yuv\":\n            if (!window.VideoFrame)\n                return Promise.reject(\n                    new Error(`Cannot process frame. ImageData in ${format} is not supported by this browser.`)\n                );\n            const frame = new VideoFrame(imageData.data.buffer, {\n                format: \"NV12\",\n                codedWidth: imageData.width,\n                codedHeight: imageData.height,\n                timestamp: 0,\n            });\n            return createImageBitmap(frame);\n        case \"rgb\":\n            return createImageBitmap(imageData);\n    }\n};\n\n/** @internal */\nexport type FrameFormat = \"rgb\" | \"yuv\" | \"nv12\";\n\n/** @internal */\nexport interface FrameInput {\n    format?: FrameFormat;\n    imageData: ImageData;\n    timestampMillis: number;\n}\n\n/** @internal */\nexport interface FrameOutput {\n    live: ImageData;\n    capture: ImageData;\n}\n\n/** @internal */\nexport interface MediaSourceFunction {\n    (render: (frame: FrameInput) => Promise<FrameOutput>): Promise<void>;\n}\n\n/**\n * Creates a {@link CameraKitSource} from a function which provides per-frame pixel data to CameraKit.\n *\n * @param sourceFunction This function will be called in a requestAnimationFrame loop. Each time it is called, it is\n * passed a `render` function. It may call `render` and CameraKit will process the pixel data passed to `render` and\n * return a Promise of the processed pixels (along with rendering them to the normal output canvases).\n * @param options\n * @param options.cameraType By default we set this to 'user', which is the camera type most Lenses expect.\n *\n * @internal\n */\nexport const createFunctionSource = (\n    sourceFunction: MediaSourceFunction,\n    options: Omit<CameraKitSourceOptions, \"fpsLimit\"> = {}\n): CameraKitSource => {\n    let width = 0;\n    let height = 0;\n    let shouldProcessFrame = true;\n\n    // We require an auxiliary canvas that we can use to read back pixel data (unless we're in YUV mode, in which case\n    // we use LensCore to convert between formats and we don't need this canvas).\n    const imageReaderCanvas = document.createElement(\"canvas\");\n    const imageReader2D = imageReaderCanvas.getContext(\"2d\");\n\n    const subscriber: CameraKitSourceSubscriber = {\n        onAttach: (source, lensCore, reportError) => {\n            const outputs = lensCore.getOutputCanvases();\n            const output = {\n                live: outputs[lensCore.CanvasType.Preview.value],\n                capture: outputs[lensCore.CanvasType.Capture.value],\n            };\n\n            const processFrame = (\n                source: CameraKitSource,\n                lensCore: LensCoreModule,\n                reportError: (error: Error) => void\n            ) =>\n                requestAnimationFrame(async () => {\n                    if (!shouldProcessFrame) return;\n                    try {\n                        await sourceFunction(({ format, imageData, timestampMillis }) => {\n                            const frameOutput = new Promise<FrameOutput>(async (resolve, reject) => {\n                                const inputFrame = await getImageBitmap(imageData, format ?? \"rgb\");\n                                if (inputFrame.width !== width || inputFrame.height !== height) {\n                                    width = imageReaderCanvas.width = inputFrame.width;\n                                    height = imageReaderCanvas.height = inputFrame.height;\n                                    // We don't await this promise, because we want to continue to process frames and\n                                    // let LensCore manage the concurrency between setting resolution and processing a\n                                    // frame.\n                                    source.setRenderSize(width, height);\n                                }\n                                lensCore.processFrame({\n                                    inputFrame,\n                                    timestampMillis,\n                                    onSuccess: async () => {\n                                        // Closing releases graphics resources associated with the frame, now that is\n                                        // has been processed.\n                                        inputFrame.close();\n\n                                        switch (format ?? \"rgb\") {\n                                            case \"nv12\":\n                                            case \"yuv\":\n                                                const [live, capture] = await Promise.all([\n                                                    getYUVImageData(output.live, lensCore),\n                                                    getYUVImageData(output.capture, lensCore),\n                                                ]).catch((error) => {\n                                                    reject(error);\n                                                    return [undefined, undefined] as const;\n                                                });\n                                                // if either of these is undefined, we'll have already rejected\n                                                // the promise, so we can return.\n                                                if (!live || !capture) return;\n                                                return resolve({ live, capture });\n                                            case \"rgb\":\n                                                return resolve({\n                                                    live: getRGBImageData(output.live, imageReader2D),\n                                                    capture: getRGBImageData(output.capture, imageReader2D),\n                                                });\n                                        }\n                                    },\n                                    onFailure: (error) => {\n                                        inputFrame.close();\n                                        reject(error);\n                                    },\n                                });\n                            });\n                            // Even if there's an error processing the frame, we do want to attempt to process the next\n                            // frame. We expect `sourceFunction` to handle a rejected `frameOutput` Promise.\n                            frameOutput.finally(() => processFrame(source, lensCore, reportError));\n                            return frameOutput;\n                        });\n                    } catch (error) {\n                        reportError(\n                            new Error(\n                                \"Failure to process frame, which was not handled by the provided \" +\n                                    `MediaSourceFunction ${sourceFunction.name ?? \"anonymous\"}.`,\n                                { cause: error }\n                            )\n                        );\n                    }\n                });\n            processFrame(source, lensCore, reportError);\n        },\n        onDetach: () => {\n            shouldProcessFrame = false;\n        },\n    };\n\n    return new CameraKitSource({ useManualFrameProcessing: true }, subscriber, options);\n};\n"]}
+{"version":3,"file":"FunctionSource.js","sourceRoot":"","sources":["../../src/media-sources/FunctionSource.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,eAAe,EAAqD,MAAM,mBAAmB,CAAC;AAEvG,MAAM,eAAe,GAAG,CAAO,MAAyB,EAAE,QAAkB,EAAsB,EAAE;IAChG,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,0FAA0F;IAC1F,MAAM,YAAY,GAAG,IAAI,WAAW,CAAC,CAAC,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/D,MAAM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,IAAI,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACnD,OAAO,IAAI,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC,CAAA,CAAC;AAEF,MAAM,eAAe,GAAG,CAAC,MAAyB,EAAE,aAA8C,EAAa,EAAE;IAC7G,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO,IAAI,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,OAAO,aAAa,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;AACzE,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAO,SAAoB,EAAE,MAAmB,EAAwB,EAAE;IAC7F,QAAQ,MAAM,EAAE;QACZ,KAAK,MAAM,CAAC;QACZ,KAAK,KAAK;YACN,IAAI,CAAC,MAAM,CAAC,UAAU;gBAClB,OAAO,OAAO,CAAC,MAAM,CACjB,IAAI,KAAK,CAAC,sCAAsC,MAAM,oCAAoC,CAAC,CAC9F,CAAC;YACN,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE;gBAChD,MAAM,EAAE,MAAM;gBACd,UAAU,EAAE,SAAS,CAAC,KAAK;gBAC3B,WAAW,EAAE,SAAS,CAAC,MAAM;gBAC7B,SAAS,EAAE,CAAC;aACf,CAAC,CAAC;YACH,OAAO,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACpC,KAAK,KAAK;YACN,OAAO,iBAAiB,CAAC,SAAS,CAAC,CAAC;KAC3C;AACL,CAAC,CAAA,CAAC;AAuBF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAChC,cAAmC,EACnC,UAAoD,EAAE,EACvC,EAAE;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,kBAAkB,GAAG,IAAI,CAAC;IAE9B,kHAAkH;IAClH,6EAA6E;IAC7E,MAAM,iBAAiB,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IAC3D,MAAM,aAAa,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAEzD,MAAM,UAAU,GAA8B;QAC1C,QAAQ,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE;YACxC,MAAM,OAAO,GAAG,QAAQ,CAAC,iBAAiB,EAAE,CAAC;YAC7C,MAAM,MAAM,GAAG;gBACX,IAAI,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;gBAChD,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;aACtD,CAAC;YAEF,MAAM,YAAY,GAAG,CAAC,MAAuB,EAAE,QAAkB,EAAE,WAAmC,EAAE,EAAE,CACtG,qBAAqB,CAAC,GAAS,EAAE;;gBAC7B,IAAI,CAAC,kBAAkB;oBAAE,OAAO;gBAChC,IAAI;oBACA,MAAM,cAAc,CAAC,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,eAAe,EAAE,EAAE,EAAE;wBAC5D,wFAAwF;wBACxF,MAAM,WAAW,GAAG,IAAI,OAAO,CAAc,CAAO,OAAO,EAAE,MAAM,EAAE,EAAE;4BACnE,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,SAAS,EAAE,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,KAAK,CAAC,CAAC;4BACpE,IAAI,UAAU,CAAC,KAAK,KAAK,KAAK,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,EAAE;gCAC5D,KAAK,GAAG,iBAAiB,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;gCACnD,MAAM,GAAG,iBAAiB,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;gCACtD,iFAAiF;gCACjF,kFAAkF;gCAClF,SAAS;gCACT,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;6BACvC;4BACD,QAAQ;iCACH,YAAY,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC;iCAC7C,IAAI,CAAC,GAAS,EAAE;gCACb,6EAA6E;gCAC7E,sBAAsB;gCACtB,UAAU,CAAC,KAAK,EAAE,CAAC;gCAEnB,QAAQ,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,KAAK,EAAE;oCACrB,KAAK,MAAM,CAAC;oCACZ,KAAK,KAAK;wCACN,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;4CACtC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC;4CACtC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC;yCAC5C,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;4CACf,MAAM,CAAC,KAAK,CAAC,CAAC;4CACd,OAAO,CAAC,SAAS,EAAE,SAAS,CAAU,CAAC;wCAC3C,CAAC,CAAC,CAAC;wCACH,+DAA+D;wCAC/D,iCAAiC;wCACjC,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO;4CAAE,OAAO;wCAC9B,OAAO,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;oCACtC,KAAK,KAAK;wCACN,OAAO,OAAO,CAAC;4CACX,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,CAAC;4CACjD,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC;yCAC1D,CAAC,CAAC;iCACV;4BACL,CAAC,CAAA,CAAC;iCACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gCACb,UAAU,CAAC,KAAK,EAAE,CAAC;gCACnB,MAAM,CAAC,KAAK,CAAC,CAAC;4BAClB,CAAC,CAAC,CAAC;wBACX,CAAC,CAAA,CAAC,CAAC;wBACH,2FAA2F;wBAC3F,gFAAgF;wBAChF,WAAW,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;wBACvE,OAAO,WAAW,CAAC;oBACvB,CAAC,CAAC,CAAC;iBACN;gBAAC,OAAO,KAAK,EAAE;oBACZ,WAAW,CACP,IAAI,KAAK,CACL,kEAAkE;wBAC9D,uBAAuB,MAAA,cAAc,CAAC,IAAI,mCAAI,WAAW,GAAG,EAChE,EAAE,KAAK,EAAE,KAAK,EAAE,CACnB,CACJ,CAAC;iBACL;YACL,CAAC,CAAA,CAAC,CAAC;YACP,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QAChD,CAAC;QACD,QAAQ,EAAE,GAAG,EAAE;YACX,kBAAkB,GAAG,KAAK,CAAC;QAC/B,CAAC;KACJ,CAAC;IAEF,OAAO,IAAI,eAAe,CAAC,EAAE,wBAAwB,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;AACxF,CAAC,CAAC","sourcesContent":["import { LensCore } from \"../lens-core-module\";\nimport { CameraKitSource, CameraKitSourceOptions, CameraKitSourceSubscriber } from \"./CameraKitSource\";\n\nconst getYUVImageData = async (canvas: HTMLCanvasElement, lensCore: LensCore): Promise<ImageData> => {\n    const { width, height } = canvas;\n    // A YUV buffer has lower-res UV channels, so the total number of bytes works out like so:\n    const outputBuffer = new ArrayBuffer((width * height * 3) / 2);\n    await lensCore.imageToYuvBuffer({ image: canvas, width, height, outputBuffer });\n    const pixels = new Uint8ClampedArray(outputBuffer);\n    return new ImageData(pixels, width, height);\n};\n\nconst getRGBImageData = (output: HTMLCanvasElement, imageReader2D: CanvasRenderingContext2D | null): ImageData => {\n    if (imageReader2D === null) return new ImageData(0, 0);\n    imageReader2D.drawImage(output, 0, 0);\n    return imageReader2D.getImageData(0, 0, output.width, output.height);\n};\n\nconst getImageBitmap = async (imageData: ImageData, format: FrameFormat): Promise<ImageBitmap> => {\n    switch (format) {\n        case \"nv12\":\n        case \"yuv\":\n            if (!window.VideoFrame)\n                return Promise.reject(\n                    new Error(`Cannot process frame. ImageData in ${format} is not supported by this browser.`)\n                );\n            const frame = new VideoFrame(imageData.data.buffer, {\n                format: \"NV12\",\n                codedWidth: imageData.width,\n                codedHeight: imageData.height,\n                timestamp: 0,\n            });\n            return createImageBitmap(frame);\n        case \"rgb\":\n            return createImageBitmap(imageData);\n    }\n};\n\n/** @internal */\nexport type FrameFormat = \"rgb\" | \"yuv\" | \"nv12\";\n\n/** @internal */\nexport interface FrameInput {\n    format?: FrameFormat;\n    imageData: ImageData;\n    timestampMillis: number;\n}\n\n/** @internal */\nexport interface FrameOutput {\n    live: ImageData;\n    capture: ImageData;\n}\n\n/** @internal */\nexport interface MediaSourceFunction {\n    (render: (frame: FrameInput) => Promise<FrameOutput>): Promise<void>;\n}\n\n/**\n * Creates a {@link CameraKitSource} from a function which provides per-frame pixel data to CameraKit.\n *\n * @param sourceFunction This function will be called in a requestAnimationFrame loop. Each time it is called, it is\n * passed a `render` function. It may call `render` and CameraKit will process the pixel data passed to `render` and\n * return a Promise of the processed pixels (along with rendering them to the normal output canvases).\n * @param options\n * @param options.cameraType By default we set this to 'user', which is the camera type most Lenses expect.\n *\n * @internal\n */\nexport const createFunctionSource = (\n    sourceFunction: MediaSourceFunction,\n    options: Omit<CameraKitSourceOptions, \"fpsLimit\"> = {}\n): CameraKitSource => {\n    let width = 0;\n    let height = 0;\n    let shouldProcessFrame = true;\n\n    // We require an auxiliary canvas that we can use to read back pixel data (unless we're in YUV mode, in which case\n    // we use LensCore to convert between formats and we don't need this canvas).\n    const imageReaderCanvas = document.createElement(\"canvas\");\n    const imageReader2D = imageReaderCanvas.getContext(\"2d\");\n\n    const subscriber: CameraKitSourceSubscriber = {\n        onAttach: (source, lensCore, reportError) => {\n            const outputs = lensCore.getOutputCanvases();\n            const output = {\n                live: outputs[lensCore.CanvasType.Preview.value],\n                capture: outputs[lensCore.CanvasType.Capture.value],\n            };\n\n            const processFrame = (source: CameraKitSource, lensCore: LensCore, reportError: (error: Error) => void) =>\n                requestAnimationFrame(async () => {\n                    if (!shouldProcessFrame) return;\n                    try {\n                        await sourceFunction(({ format, imageData, timestampMillis }) => {\n                            // TODO: with processFrame promisified, we don't really need to create a wrapper promise\n                            const frameOutput = new Promise<FrameOutput>(async (resolve, reject) => {\n                                const inputFrame = await getImageBitmap(imageData, format ?? \"rgb\");\n                                if (inputFrame.width !== width || inputFrame.height !== height) {\n                                    width = imageReaderCanvas.width = inputFrame.width;\n                                    height = imageReaderCanvas.height = inputFrame.height;\n                                    // We don't await this promise, because we want to continue to process frames and\n                                    // let LensCore manage the concurrency between setting resolution and processing a\n                                    // frame.\n                                    source.setRenderSize(width, height);\n                                }\n                                lensCore\n                                    .processFrame({ inputFrame, timestampMillis })\n                                    .then(async () => {\n                                        // Closing releases graphics resources associated with the frame, now that is\n                                        // has been processed.\n                                        inputFrame.close();\n\n                                        switch (format ?? \"rgb\") {\n                                            case \"nv12\":\n                                            case \"yuv\":\n                                                const [live, capture] = await Promise.all([\n                                                    getYUVImageData(output.live, lensCore),\n                                                    getYUVImageData(output.capture, lensCore),\n                                                ]).catch((error) => {\n                                                    reject(error);\n                                                    return [undefined, undefined] as const;\n                                                });\n                                                // if either of these is undefined, we'll have already rejected\n                                                // the promise, so we can return.\n                                                if (!live || !capture) return;\n                                                return resolve({ live, capture });\n                                            case \"rgb\":\n                                                return resolve({\n                                                    live: getRGBImageData(output.live, imageReader2D),\n                                                    capture: getRGBImageData(output.capture, imageReader2D),\n                                                });\n                                        }\n                                    })\n                                    .catch((error) => {\n                                        inputFrame.close();\n                                        reject(error);\n                                    });\n                            });\n                            // Even if there's an error processing the frame, we do want to attempt to process the next\n                            // frame. We expect `sourceFunction` to handle a rejected `frameOutput` Promise.\n                            frameOutput.finally(() => processFrame(source, lensCore, reportError));\n                            return frameOutput;\n                        });\n                    } catch (error) {\n                        reportError(\n                            new Error(\n                                \"Failure to process frame, which was not handled by the provided \" +\n                                    `MediaSourceFunction ${sourceFunction.name ?? \"anonymous\"}.`,\n                                { cause: error }\n                            )\n                        );\n                    }\n                });\n            processFrame(source, lensCore, reportError);\n        },\n        onDetach: () => {\n            shouldProcessFrame = false;\n        },\n    };\n\n    return new CameraKitSource({ useManualFrameProcessing: true }, subscriber, options);\n};\n"]}

package/lib/media-sources/MediaStreamSource.js

@@ -63,9 +63,13 @@ export function createUserMediaSource(constraints = { video: true }, options = {
  * @category Rendering
  */
 export function createMediaStreamSource(stream, options = {}) {
-    const optionsNotNull = Object.assign(Object.assign({}, defaultOptions), options);
-    const { width, height } = stream.getVideoTracks().length > 0 ? stream.getVideoTracks()[0].getSettings() : { width: 0, height: 0 };
-    const enableSourceAudio = stream.getAudioTracks().length > 0 && !optionsNotNull.disableSourceAudio;
+    var _a;
+    const { width, height, facingMode } = stream.getVideoTracks().length > 0
+        ? stream.getVideoTracks()[0].getSettings()
+        : { width: 0, height: 0, facingMode: undefined };
+    const detectedCameraType = facingMode === "user" || facingMode === "environment" ? facingMode : undefined;
+    const optionsWithDefaults = Object.assign(Object.assign(Object.assign({}, defaultOptions), options), { cameraType: (_a = options.cameraType) !== null && _a !== void 0 ? _a : detectedCameraType });
+    const enableSourceAudio = stream.getAudioTracks().length > 0 && !optionsWithDefaults.disableSourceAudio;
     const simulateStereoAudio = true;
     const sampleRate = 44100;
     let audioContext = undefined;
@@ -94,17 +98,14 @@ export function createMediaStreamSource(stream, options = {}) {
         onAttach: (source, lensCore, reportError) => __awaiter(this, void 0, void 0, function* () {
             if (width !== undefined && height !== undefined)
                 yield source.setRenderSize(width, height);
-            yield source.setTransform(optionsNotNull.transform);
+            yield source.setTransform(optionsWithDefaults.transform);
             if (enableSourceAudio) {
                 // Audio paramters set has to be called before lens is applied
-                lensCore.setAudioParameters({
+                yield lensCore.setAudioParameters({
                     parameters: {
                         numChannels: simulateStereoAudio ? 2 : 1,
                         sampleRate,
                     },
-                    onFailure: (error) => {
-                        reportError(error);
-                    },
                 });
                 try {
                     // There is a possibility of the onAttach method being called twice in a row due to a bug.
@@ -146,12 +147,7 @@ export function createMediaStreamSource(stream, options = {}) {
                                     const rightSamples = e.data.buffer[0].length > 1 ? e.data.buffer[0][1] : leftSamples.slice();
                                     inputBuffers.push(rightSamples);
                                 }
-                                lensCore.processAudioSampleBuffer({
-                                    input: inputBuffers,
-                                    onFailure: (error) => {
-                                        reportError(error);
-                                    },
-                                });
+                                lensCore.processAudioSampleBuffer({ input: inputBuffers }).catch(reportError);
                             }
                         };
                     }
@@ -175,6 +171,6 @@ export function createMediaStreamSource(stream, options = {}) {
                 audioContext = undefined;
             }
         }),
-    }, options);
+    }, optionsWithDefaults);
 }
 //# sourceMappingURL=MediaStreamSource.js.map

package/lib/media-sources/MediaStreamSource.js.map

@@ -1 +1 @@
-{"version":3,"file":"MediaStreamSource.js","sourceRoot":"","sources":["../../src/media-sources/MediaStreamSource.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,eAAe,EAA0B,MAAM,mBAAmB,CAAC;AAE5E,MAAM,cAAc,GAA6B;IAC7C,SAAS,EAAE,WAAW,CAAC,QAAQ;IAC/B,kBAAkB,EAAE,KAAK;CAC5B,CAAC;AAEF,SAAS,YAAY,CAAC,OAAqC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,CAAC,UAAU,EAAE,CAAC;AACzB,CAAC;AAED,SAAe,iBAAiB,CAAC,YAAsC;;QACnE,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO;QAC7D,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CAAA;AAQD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAgB,qBAAqB,CACvC,cAAsC,EAAE,KAAK,EAAE,IAAI,EAAE,EACrD,UAA4D,EAAE;;QAE9D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACtE,OAAO,uBAAuB,CAAC,MAAM,kBAAI,SAAS,EAAE,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,IAAK,OAAO,EAAG,CAAC;IAChH,CAAC;CAAA;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,uBAAuB,CACnC,MAAmB,EACnB,UAA4D,EAAE;IAE9D,MAAM,cAAc,mCACb,cAAc,GACd,OAAO,CACb,CAAC;IAEF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GACnB,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC5G,MAAM,iBAAiB,GAAY,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,kBAAkB,CAAC;IAE5G,MAAM,mBAAmB,GAAG,IAAI,CAAC;IACjC,MAAM,UAAU,GAAW,KAAK,CAAC;IAEjC,IAAI,YAAY,GAA6B,SAAS,CAAC;IACvD,IAAI,WAAW,GAA2C,SAAS,CAAC;IACpE,IAAI,OAAO,GAAiC,SAAS,CAAC;IACtD,IAAI,qBAA6B,CAAC;IAElC,IAAI,iBAAiB,EAAE;QACnB,oFAAoF;QACpF,MAAM,yBAAyB,GAAG;;;;;;;;;;6EAUmC,CAAC;QACtE,MAAM,sBAAsB,GAAG,IAAI,IAAI,CAAC,CAAC,yBAAyB,CAAC,EAAE;YACjE,IAAI,EAAE,wBAAwB;SACjC,CAAC,CAAC;QACH,qBAAqB,GAAG,GAAG,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC;KACvE;IAED,OAAO,IAAI,eAAe,CACtB,EAAE,KAAK,EAAE,MAAM,EAAE,EACjB;QACI,QAAQ,EAAE,CAAO,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE;YAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC3F,MAAM,MAAM,CAAC,YAAY,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;YAEpD,IAAI,iBAAiB,EAAE;gBACnB,8DAA8D;gBAC9D,QAAQ,CAAC,kBAAkB,CAAC;oBACxB,UAAU,EAAE;wBACR,WAAW,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBACxC,UAAU;qBACb;oBACD,SAAS,EAAE,CAAC,KAAY,EAAE,EAAE;wBACxB,WAAW,CAAC,KAAK,CAAC,CAAC;oBACvB,CAAC;iBACJ,CAAC,CAAC;gBAEH,IAAI;oBACA,0FAA0F;oBAC1F,iFAAiF;oBACjF,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,UAAU,EAAE,CAAC;oBAC1B,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;iBACzC;gBAAC,OAAO,KAAK,EAAE;oBACZ,qDAAqD;oBACrD,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;iBACnC;gBAED,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;gBAClC,WAAW,GAAG,YAAY,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;gBAC3D,MAAM,iBAAiB,GAAG,WAAW,CAAC;gBACtC,YAAY,CAAC,YAAY;qBACpB,SAAS,CAAC,qBAAqB,CAAC;qBAChC,IAAI,CAAC,GAAG,EAAE;oBACP,IAAI,YAAY,EAAE;wBACd,OAAO,GAAG,IAAI,gBAAgB,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;wBACnE,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBACnC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;wBAC1C,wEAAwE;wBACxE,8EAA8E;wBAC9E,0EAA0E;wBAC1E,mDAAmD;wBACnD,iCAAiC;wBACjC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,EAAE;4BAC3B,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,KAAK,MAAM,EAAE;gCAC7B,yEAAyE;gCACzE,6EAA6E;gCAC7E,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAiB,CAAC;gCAExD,4CAA4C;gCAC5C,8CAA8C;gCAC9C,IAAI,CAAC,WAAW;oCAAE,OAAO;gCAEzB,IAAI,YAAY,GAAG,CAAC,WAAW,CAAC,CAAC;gCACjC,IAAI,mBAAmB,EAAE;oCACrB,MAAM,YAAY,GACd,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;oCAC5E,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;iCACnC;gCAED,QAAQ,CAAC,wBAAwB,CAAC;oCAC9B,KAAK,EAAE,YAAY;oCACnB,SAAS,EAAE,CAAC,KAAY,EAAE,EAAE;wCACxB,WAAW,CAAC,KAAK,CAAC,CAAC;oCACvB,CAAC;iCACJ,CAAC,CAAC;6BACN;wBACL,CAAC,CAAC;qBACL;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;oBACpB,WAAW,CAAC,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAA;QACD,QAAQ,EAAE,CAAO,WAAW,EAAE,EAAE;YAC5B,IAAI,OAAO,EAAE;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,OAAO,GAAG,SAAS,CAAC;aACvB;YACD,IAAI,WAAW,EAAE;gBACb,WAAW,CAAC,UAAU,EAAE,CAAC;gBACzB,WAAW,GAAG,SAAS,CAAC;aAC3B;YACD,IAAI,YAAY,EAAE;gBACd,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACzD,YAAY,GAAG,SAAS,CAAC;aAC5B;QACL,CAAC,CAAA;KACJ,EACD,OAAO,CACV,CAAC;AACN,CAAC","sourcesContent":["import { ensureError } from \"../common/errorHelpers\";\nimport { Transform2D } from \"../transforms\";\nimport { CameraKitSource, CameraKitSourceOptions } from \"./CameraKitSource\";\n\nconst defaultOptions: MediaStreamSourceOptions = {\n    transform: Transform2D.Identity,\n    disableSourceAudio: false,\n};\n\nfunction closeWorklet(worklet: AudioWorkletNode | undefined) {\n    if (!worklet) return;\n    worklet.port.close();\n    worklet.port.onmessage = null;\n    worklet.disconnect();\n}\n\nasync function closeAudioContext(audioContext: AudioContext | undefined) {\n    if (!audioContext || audioContext.state === \"closed\") return;\n    return audioContext.close();\n}\n\n/** @category Rendering */\nexport interface MediaStreamSourceOptions {\n    transform: Transform2D;\n    disableSourceAudio: boolean; // defaults to false\n}\n\n/**\n * Create a {@link CameraKitSource} from a user's media device -- this calls\n * [MediaDevices.getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) to get a\n * MediaStream and then calls {@link createMediaStreamSource}.\n *\n * @param constraints Specify contraints used to get a MediaStream from a media device. By default we simply request\n * a video stream.\n * @param options\n * @param options.transform By default we horizontally mirror the video stream. The most common use-case is to obtain a\n * stream from a front-facing web cam, which requires mirroring to be viewed naturally.\n * @param options.cameraType By default we set this to 'front' to indicate a camera pointed at the user (e.g. a webcam).\n * @param options.fpsLimit By default we set no limit on FPS – if the source device has a known FPS setting this limit\n * may prevent CameraKit from using more compute resources than strictly necessary.\n * @returns A Promise, resolving to {@link CameraKitSource}\n *\n * @category Rendering\n *\n * @deprecated The helper will be removed in one of the future releases.\n * Consumer apps are responsible for acquiring a media stream,\n * which can then be supplied to {@link createMediaStreamSource}.\n */\nexport async function createUserMediaSource(\n    constraints: MediaStreamConstraints = { video: true },\n    options: CameraKitSourceOptions<MediaStreamSourceOptions> = {}\n): Promise<CameraKitSource> {\n    const stream = await navigator.mediaDevices.getUserMedia(constraints);\n    return createMediaStreamSource(stream, { transform: Transform2D.MirrorX, cameraType: \"front\", ...options });\n}\n\n/**\n * Create a {@link CameraKitSource} from any\n * [MediaStream](https://developer.mozilla.org/en-US/docs/Web/API/MediaStream).\n *\n * @param stream Any MediaStream, such as obtained via `canvas.captureStream()` or `mediaDevices.getUserMedia()`.\n * @param options\n * @param options.transform We apply no transformation by default.\n * @param options.disableSourceAudio By default we pass audio to lens. Settings this to true will disable sending audio\n * to the lens.\n * @param options.cameraType By default we set this to 'user', which is the camera type most Lenses expect.\n * @param options.fpsLimit By default we set no limit on FPS – if the MediaStream has a known FPS setting this limit\n * may prevent CameraKit from using more compute resources than strictly necessary.\n *\n * @category Rendering\n */\nexport function createMediaStreamSource(\n    stream: MediaStream,\n    options: CameraKitSourceOptions<MediaStreamSourceOptions> = {}\n): CameraKitSource {\n    const optionsNotNull = {\n        ...defaultOptions,\n        ...options,\n    };\n\n    const { width, height } =\n        stream.getVideoTracks().length > 0 ? stream.getVideoTracks()[0].getSettings() : { width: 0, height: 0 };\n    const enableSourceAudio: boolean = stream.getAudioTracks().length > 0 && !optionsNotNull.disableSourceAudio;\n\n    const simulateStereoAudio = true;\n    const sampleRate: number = 44100;\n\n    let audioContext: AudioContext | undefined = undefined;\n    let audioSource: MediaStreamAudioSourceNode | undefined = undefined;\n    let worklet: AudioWorkletNode | undefined = undefined;\n    let microphoneRecorderUrl: string;\n\n    if (enableSourceAudio) {\n        // https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Using_AudioWorklet\n        const microphoneRecorderWorklet = `\n        class MicrophoneWorkletProcessor extends AudioWorkletProcessor {\n            process(inputs, outputs, parameters) {\n                this.port.postMessage({\n                    eventType: 'data',\n                    buffer: inputs\n                });\n                return true;\n            }\n        }\n        registerProcessor('microphone-worklet', MicrophoneWorkletProcessor);`;\n        const microphoneRecorderBlob = new Blob([microphoneRecorderWorklet], {\n            type: \"application/javascript\",\n        });\n        microphoneRecorderUrl = URL.createObjectURL(microphoneRecorderBlob);\n    }\n\n    return new CameraKitSource(\n        { media: stream },\n        {\n            onAttach: async (source, lensCore, reportError) => {\n                if (width !== undefined && height !== undefined) await source.setRenderSize(width, height);\n                await source.setTransform(optionsNotNull.transform);\n\n                if (enableSourceAudio) {\n                    // Audio paramters set has to be called before lens is applied\n                    lensCore.setAudioParameters({\n                        parameters: {\n                            numChannels: simulateStereoAudio ? 2 : 1,\n                            sampleRate,\n                        },\n                        onFailure: (error: Error) => {\n                            reportError(error);\n                        },\n                    });\n\n                    try {\n                        // There is a possibility of the onAttach method being called twice in a row due to a bug.\n                        // To ensure there are not leaks, it is better to close any existing connections.\n                        closeWorklet(worklet);\n                        audioSource?.disconnect();\n                        await closeAudioContext(audioContext);\n                    } catch (error) {\n                        // We still want to continue if anything above failed\n                        reportError(ensureError(error));\n                    }\n\n                    audioContext = new AudioContext();\n                    audioSource = audioContext.createMediaStreamSource(stream);\n                    const scopedAudioSource = audioSource;\n                    audioContext.audioWorklet\n                        .addModule(microphoneRecorderUrl)\n                        .then(() => {\n                            if (audioContext) {\n                                worklet = new AudioWorkletNode(audioContext, \"microphone-worklet\");\n                                scopedAudioSource.connect(worklet);\n                                worklet.connect(audioContext.destination);\n                                // NOTE: We subscribe to messages here, and they will continue to arrive\n                                // even after audioContext.close() is called. To disconnect the audio worklets\n                                // created here, we need to track two variables - worklet and audioSource.\n                                // By calling disconnect() on them, we can properly\n                                // disconnect the audio worklets.\n                                worklet.port.onmessage = (e) => {\n                                    if (e.data.eventType === \"data\") {\n                                        // developer.mozilla.org/en-US/docs/Web/API/AudioWorkletProcessor/process\n                                        // inputs[n][m] is the list of samples in the n-th input at the m-th channel.\n                                        const leftSamples = e.data.buffer[0][0] as Float32Array;\n\n                                        // Firefox might have leftSamples undefined:\n                                        // https://jira.sc-corp.net/browse/CAMKIT-5189\n                                        if (!leftSamples) return;\n\n                                        let inputBuffers = [leftSamples];\n                                        if (simulateStereoAudio) {\n                                            const rightSamples =\n                                                e.data.buffer[0].length > 1 ? e.data.buffer[0][1] : leftSamples.slice();\n                                            inputBuffers.push(rightSamples);\n                                        }\n\n                                        lensCore.processAudioSampleBuffer({\n                                            input: inputBuffers,\n                                            onFailure: (error: Error) => {\n                                                reportError(error);\n                                            },\n                                        });\n                                    }\n                                };\n                            }\n                        })\n                        .catch((error: Error) => {\n                            reportError(error);\n                        });\n                }\n            },\n            onDetach: async (reportError) => {\n                if (worklet) {\n                    closeWorklet(worklet);\n                    worklet = undefined;\n                }\n                if (audioSource) {\n                    audioSource.disconnect();\n                    audioSource = undefined;\n                }\n                if (audioContext) {\n                    await closeAudioContext(audioContext).catch(reportError);\n                    audioContext = undefined;\n                }\n            },\n        },\n        options\n    );\n}\n"]}
+{"version":3,"file":"MediaStreamSource.js","sourceRoot":"","sources":["../../src/media-sources/MediaStreamSource.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,eAAe,EAA0B,MAAM,mBAAmB,CAAC;AAE5E,MAAM,cAAc,GAA6B;IAC7C,SAAS,EAAE,WAAW,CAAC,QAAQ;IAC/B,kBAAkB,EAAE,KAAK;CAC5B,CAAC;AAEF,SAAS,YAAY,CAAC,OAAqC;IACvD,IAAI,CAAC,OAAO;QAAE,OAAO;IACrB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAC9B,OAAO,CAAC,UAAU,EAAE,CAAC;AACzB,CAAC;AAED,SAAe,iBAAiB,CAAC,YAAsC;;QACnE,IAAI,CAAC,YAAY,IAAI,YAAY,CAAC,KAAK,KAAK,QAAQ;YAAE,OAAO;QAC7D,OAAO,YAAY,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;CAAA;AAQD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAgB,qBAAqB,CACvC,cAAsC,EAAE,KAAK,EAAE,IAAI,EAAE,EACrD,UAA4D,EAAE;;QAE9D,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACtE,OAAO,uBAAuB,CAAC,MAAM,kBAAI,SAAS,EAAE,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,IAAK,OAAO,EAAG,CAAC;IAChH,CAAC;CAAA;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,uBAAuB,CACnC,MAAmB,EACnB,UAA4D,EAAE;;IAE9D,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,GAC/B,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,CAAC;QAC9B,CAAC,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QAC1C,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC;IAEzD,MAAM,kBAAkB,GAAG,UAAU,KAAK,MAAM,IAAI,UAAU,KAAK,aAAa,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAE1G,MAAM,mBAAmB,iDAClB,cAAc,GACd,OAAO,KACV,UAAU,EAAE,MAAA,OAAO,CAAC,UAAU,mCAAI,kBAAkB,GACvD,CAAC;IAEF,MAAM,iBAAiB,GAAY,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC;IAEjH,MAAM,mBAAmB,GAAG,IAAI,CAAC;IACjC,MAAM,UAAU,GAAW,KAAK,CAAC;IAEjC,IAAI,YAAY,GAA6B,SAAS,CAAC;IACvD,IAAI,WAAW,GAA2C,SAAS,CAAC;IACpE,IAAI,OAAO,GAAiC,SAAS,CAAC;IACtD,IAAI,qBAA6B,CAAC;IAElC,IAAI,iBAAiB,EAAE;QACnB,oFAAoF;QACpF,MAAM,yBAAyB,GAAG;;;;;;;;;;6EAUmC,CAAC;QACtE,MAAM,sBAAsB,GAAG,IAAI,IAAI,CAAC,CAAC,yBAAyB,CAAC,EAAE;YACjE,IAAI,EAAE,wBAAwB;SACjC,CAAC,CAAC;QACH,qBAAqB,GAAG,GAAG,CAAC,eAAe,CAAC,sBAAsB,CAAC,CAAC;KACvE;IAED,OAAO,IAAI,eAAe,CACtB,EAAE,KAAK,EAAE,MAAM,EAAE,EACjB;QACI,QAAQ,EAAE,CAAO,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE;YAC9C,IAAI,KAAK,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC3F,MAAM,MAAM,CAAC,YAAY,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;YAEzD,IAAI,iBAAiB,EAAE;gBACnB,8DAA8D;gBAC9D,MAAM,QAAQ,CAAC,kBAAkB,CAAC;oBAC9B,UAAU,EAAE;wBACR,WAAW,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBACxC,UAAU;qBACb;iBACJ,CAAC,CAAC;gBAEH,IAAI;oBACA,0FAA0F;oBAC1F,iFAAiF;oBACjF,YAAY,CAAC,OAAO,CAAC,CAAC;oBACtB,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,UAAU,EAAE,CAAC;oBAC1B,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;iBACzC;gBAAC,OAAO,KAAK,EAAE;oBACZ,qDAAqD;oBACrD,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;iBACnC;gBAED,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;gBAClC,WAAW,GAAG,YAAY,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;gBAC3D,MAAM,iBAAiB,GAAG,WAAW,CAAC;gBACtC,YAAY,CAAC,YAAY;qBACpB,SAAS,CAAC,qBAAqB,CAAC;qBAChC,IAAI,CAAC,GAAG,EAAE;oBACP,IAAI,YAAY,EAAE;wBACd,OAAO,GAAG,IAAI,gBAAgB,CAAC,YAAY,EAAE,oBAAoB,CAAC,CAAC;wBACnE,iBAAiB,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBACnC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;wBAC1C,wEAAwE;wBACxE,8EAA8E;wBAC9E,0EAA0E;wBAC1E,mDAAmD;wBACnD,iCAAiC;wBACjC,OAAO,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAAE,EAAE;4BAC3B,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,KAAK,MAAM,EAAE;gCAC7B,yEAAyE;gCACzE,6EAA6E;gCAC7E,MAAM,WAAW,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAiB,CAAC;gCAExD,4CAA4C;gCAC5C,8CAA8C;gCAC9C,IAAI,CAAC,WAAW;oCAAE,OAAO;gCAEzB,IAAI,YAAY,GAAG,CAAC,WAAW,CAAC,CAAC;gCACjC,IAAI,mBAAmB,EAAE;oCACrB,MAAM,YAAY,GACd,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;oCAC5E,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;iCACnC;gCAED,QAAQ,CAAC,wBAAwB,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;6BACjF;wBACL,CAAC,CAAC;qBACL;gBACL,CAAC,CAAC;qBACD,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;oBACpB,WAAW,CAAC,KAAK,CAAC,CAAC;gBACvB,CAAC,CAAC,CAAC;aACV;QACL,CAAC,CAAA;QACD,QAAQ,EAAE,CAAO,WAAW,EAAE,EAAE;YAC5B,IAAI,OAAO,EAAE;gBACT,YAAY,CAAC,OAAO,CAAC,CAAC;gBACtB,OAAO,GAAG,SAAS,CAAC;aACvB;YACD,IAAI,WAAW,EAAE;gBACb,WAAW,CAAC,UAAU,EAAE,CAAC;gBACzB,WAAW,GAAG,SAAS,CAAC;aAC3B;YACD,IAAI,YAAY,EAAE;gBACd,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACzD,YAAY,GAAG,SAAS,CAAC;aAC5B;QACL,CAAC,CAAA;KACJ,EACD,mBAAmB,CACtB,CAAC;AACN,CAAC","sourcesContent":["import { ensureError } from \"../common/errorHelpers\";\nimport { Transform2D } from \"../transforms\";\nimport { CameraKitSource, CameraKitSourceOptions } from \"./CameraKitSource\";\n\nconst defaultOptions: MediaStreamSourceOptions = {\n    transform: Transform2D.Identity,\n    disableSourceAudio: false,\n};\n\nfunction closeWorklet(worklet: AudioWorkletNode | undefined) {\n    if (!worklet) return;\n    worklet.port.close();\n    worklet.port.onmessage = null;\n    worklet.disconnect();\n}\n\nasync function closeAudioContext(audioContext: AudioContext | undefined) {\n    if (!audioContext || audioContext.state === \"closed\") return;\n    return audioContext.close();\n}\n\n/** @category Rendering */\nexport interface MediaStreamSourceOptions {\n    transform: Transform2D;\n    disableSourceAudio: boolean; // defaults to false\n}\n\n/**\n * Create a {@link CameraKitSource} from a user's media device -- this calls\n * [MediaDevices.getUserMedia](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia) to get a\n * MediaStream and then calls {@link createMediaStreamSource}.\n *\n * @param constraints Specify contraints used to get a MediaStream from a media device. By default we simply request\n * a video stream.\n * @param options\n * @param options.transform By default we horizontally mirror the video stream. The most common use-case is to obtain a\n * stream from a front-facing web cam, which requires mirroring to be viewed naturally.\n * @param options.cameraType By default we set this to 'front' to indicate a camera pointed at the user (e.g. a webcam).\n * @param options.fpsLimit By default we set no limit on FPS – if the source device has a known FPS setting this limit\n * may prevent CameraKit from using more compute resources than strictly necessary.\n * @returns A Promise, resolving to {@link CameraKitSource}\n *\n * @category Rendering\n *\n * @deprecated The helper will be removed in one of the future releases.\n * Consumer apps are responsible for acquiring a media stream,\n * which can then be supplied to {@link createMediaStreamSource}.\n */\nexport async function createUserMediaSource(\n    constraints: MediaStreamConstraints = { video: true },\n    options: CameraKitSourceOptions<MediaStreamSourceOptions> = {}\n): Promise<CameraKitSource> {\n    const stream = await navigator.mediaDevices.getUserMedia(constraints);\n    return createMediaStreamSource(stream, { transform: Transform2D.MirrorX, cameraType: \"front\", ...options });\n}\n\n/**\n * Create a {@link CameraKitSource} from any\n * [MediaStream](https://developer.mozilla.org/en-US/docs/Web/API/MediaStream).\n *\n * @param stream Any MediaStream, such as obtained via `canvas.captureStream()` or `mediaDevices.getUserMedia()`.\n * @param options\n * @param options.transform We apply no transformation by default.\n * @param options.disableSourceAudio By default we pass audio to lens. Settings this to true will disable sending audio\n * to the lens.\n * @param options.cameraType By default we set this to 'user', which is the camera type most Lenses expect.\n * @param options.fpsLimit By default we set no limit on FPS – if the MediaStream has a known FPS setting this limit\n * may prevent CameraKit from using more compute resources than strictly necessary.\n *\n * @category Rendering\n */\nexport function createMediaStreamSource(\n    stream: MediaStream,\n    options: CameraKitSourceOptions<MediaStreamSourceOptions> = {}\n): CameraKitSource {\n    const { width, height, facingMode } =\n        stream.getVideoTracks().length > 0\n            ? stream.getVideoTracks()[0].getSettings()\n            : { width: 0, height: 0, facingMode: undefined };\n\n    const detectedCameraType = facingMode === \"user\" || facingMode === \"environment\" ? facingMode : undefined;\n\n    const optionsWithDefaults = {\n        ...defaultOptions,\n        ...options,\n        cameraType: options.cameraType ?? detectedCameraType,\n    };\n\n    const enableSourceAudio: boolean = stream.getAudioTracks().length > 0 && !optionsWithDefaults.disableSourceAudio;\n\n    const simulateStereoAudio = true;\n    const sampleRate: number = 44100;\n\n    let audioContext: AudioContext | undefined = undefined;\n    let audioSource: MediaStreamAudioSourceNode | undefined = undefined;\n    let worklet: AudioWorkletNode | undefined = undefined;\n    let microphoneRecorderUrl: string;\n\n    if (enableSourceAudio) {\n        // https://developer.mozilla.org/en-US/docs/Web/API/Web_Audio_API/Using_AudioWorklet\n        const microphoneRecorderWorklet = `\n        class MicrophoneWorkletProcessor extends AudioWorkletProcessor {\n            process(inputs, outputs, parameters) {\n                this.port.postMessage({\n                    eventType: 'data',\n                    buffer: inputs\n                });\n                return true;\n            }\n        }\n        registerProcessor('microphone-worklet', MicrophoneWorkletProcessor);`;\n        const microphoneRecorderBlob = new Blob([microphoneRecorderWorklet], {\n            type: \"application/javascript\",\n        });\n        microphoneRecorderUrl = URL.createObjectURL(microphoneRecorderBlob);\n    }\n\n    return new CameraKitSource(\n        { media: stream },\n        {\n            onAttach: async (source, lensCore, reportError) => {\n                if (width !== undefined && height !== undefined) await source.setRenderSize(width, height);\n                await source.setTransform(optionsWithDefaults.transform);\n\n                if (enableSourceAudio) {\n                    // Audio paramters set has to be called before lens is applied\n                    await lensCore.setAudioParameters({\n                        parameters: {\n                            numChannels: simulateStereoAudio ? 2 : 1,\n                            sampleRate,\n                        },\n                    });\n\n                    try {\n                        // There is a possibility of the onAttach method being called twice in a row due to a bug.\n                        // To ensure there are not leaks, it is better to close any existing connections.\n                        closeWorklet(worklet);\n                        audioSource?.disconnect();\n                        await closeAudioContext(audioContext);\n                    } catch (error) {\n                        // We still want to continue if anything above failed\n                        reportError(ensureError(error));\n                    }\n\n                    audioContext = new AudioContext();\n                    audioSource = audioContext.createMediaStreamSource(stream);\n                    const scopedAudioSource = audioSource;\n                    audioContext.audioWorklet\n                        .addModule(microphoneRecorderUrl)\n                        .then(() => {\n                            if (audioContext) {\n                                worklet = new AudioWorkletNode(audioContext, \"microphone-worklet\");\n                                scopedAudioSource.connect(worklet);\n                                worklet.connect(audioContext.destination);\n                                // NOTE: We subscribe to messages here, and they will continue to arrive\n                                // even after audioContext.close() is called. To disconnect the audio worklets\n                                // created here, we need to track two variables - worklet and audioSource.\n                                // By calling disconnect() on them, we can properly\n                                // disconnect the audio worklets.\n                                worklet.port.onmessage = (e) => {\n                                    if (e.data.eventType === \"data\") {\n                                        // developer.mozilla.org/en-US/docs/Web/API/AudioWorkletProcessor/process\n                                        // inputs[n][m] is the list of samples in the n-th input at the m-th channel.\n                                        const leftSamples = e.data.buffer[0][0] as Float32Array;\n\n                                        // Firefox might have leftSamples undefined:\n                                        // https://jira.sc-corp.net/browse/CAMKIT-5189\n                                        if (!leftSamples) return;\n\n                                        let inputBuffers = [leftSamples];\n                                        if (simulateStereoAudio) {\n                                            const rightSamples =\n                                                e.data.buffer[0].length > 1 ? e.data.buffer[0][1] : leftSamples.slice();\n                                            inputBuffers.push(rightSamples);\n                                        }\n\n                                        lensCore.processAudioSampleBuffer({ input: inputBuffers }).catch(reportError);\n                                    }\n                                };\n                            }\n                        })\n                        .catch((error: Error) => {\n                            reportError(error);\n                        });\n                }\n            },\n            onDetach: async (reportError) => {\n                if (worklet) {\n                    closeWorklet(worklet);\n                    worklet = undefined;\n                }\n                if (audioSource) {\n                    audioSource.disconnect();\n                    audioSource = undefined;\n                }\n                if (audioContext) {\n                    await closeAudioContext(audioContext).catch(reportError);\n                    audioContext = undefined;\n                }\n            },\n        },\n        optionsWithDefaults\n    );\n}\n"]}

package/lib/metrics/businessEventsReporter.js

@@ -1,7 +1,6 @@
 import { __awaiter } from "tslib";
 import { v4 } from "uuid";
-import { catchError, from, of, switchMap, take } from "rxjs";
-import { cameraKitUserAgent as userAgent } from "../common/cameraKitUserAgent";
+import { catchError, combineLatestWith, from, fromEvent, map, merge, of, switchMap, take } from "rxjs";
 import { entries } from "../common/entries";
 import { Injectable } from "../dependency-injection/Injectable";
 import { createBatchingHandler } from "../handlers/batchingHandler";
@@ -15,6 +14,7 @@ import { remoteConfigurationFactory } from "../remote-configuration/remoteConfig
 import { IndexedDBPersistence } from "../persistence/IndexedDBPersistence";
 import { ExpiringPersistence } from "../persistence/ExpiringPersistence";
 import { convertDaysToSeconds } from "../common/time";
+import { getPlatformInfo } from "../platform/platformInfo";
 import { metricsHandlerFactory } from "./metricsHandler";
 import { metricsEventTargetFactory } from "./metricsEventTarget";
 const logger = getLogger("BusinessEventsReporter");
@@ -59,7 +59,7 @@ const getOrGenerateVendorUuid = (persistence) => __awaiter(void 0, void 0, void
         throw new Error("Failed to generate vendor UUID");
     }
 });
-function listenAndReport(metricsEventTarget, metricsHandler, pageVisibility, eventHandlers, apiHostname, appVendorUuid) {
+function listenAndReport(metricsEventTarget, metricsHandler, pageVisibility, eventHandlers, apiHostname, appVendorAndPartnerUuid) {
     const sessionId = v4();
     logger.log(`Session ID: ${sessionId}`);
     // Blizzard convention is to start the sequenceId at 1.
@@ -92,54 +92,75 @@ function listenAndReport(metricsEventTarget, metricsHandler, pageVisibility, eve
         maxBatchAge: BUSINESS_EVENT_BATCH_MAX_AGE_MS,
         pageVisibility,
     })).handler;
-    const makeBlizzardEvent = (event) => {
+    const makeBlizzardEvent = (event, appVendorUuid, partnerUuid) => {
         var _a;
-        const deviceConnectivity = (_a = connectivityTypeMapping[userAgent.connectionType]) !== null && _a !== void 0 ? _a : blizzard.CameraKitConnectivityType.CAMERA_KIT_CONNECTIVITY_TYPE_UNKNOWN;
+        const { sdkShortVersion, sdkLongVersion, lensCore, locale, origin, deviceModel, connectionType } = getPlatformInfo();
+        const deviceConnectivity = (_a = connectivityTypeMapping[connectionType]) !== null && _a !== void 0 ? _a : blizzard.CameraKitConnectivityType.CAMERA_KIT_CONNECTIVITY_TYPE_UNKNOWN;
         return Object.assign(Object.assign({}, event), { cameraKitEventBase: blizzard.CameraKitEventBase.fromPartial({
                 kitEventBase: blizzard.KitEventBase.fromPartial({
-                    locale: userAgent.locale,
+                    locale,
                     kitVariant: blizzard.KitType.CAMERA_KIT_WEB,
-                    kitVariantVersion: userAgent.sdkShortVersion,
+                    kitVariantVersion: sdkShortVersion,
                     kitClientTimestampMillis: Date.now(),
                 }),
                 deviceCluster: 0,
-                cameraKitVersion: userAgent.sdkLongVersion,
-                lensCoreVersion: userAgent.lensCoreVersion,
-                deviceModel: userAgent.deviceModel,
+                cameraKitVersion: sdkLongVersion,
+                lensCoreVersion: lensCore.version,
+                deviceModel,
                 cameraKitVariant: blizzard.CameraKitVariant.CAMERA_KIT_VARIANT_PARTNER,
                 cameraKitFlavor: blizzard.CameraKitFlavor.CAMERA_KIT_FLAVOR_DEBUG,
                 // We overload appId, using the origin instead because it's nice and human-readable (our backed adds
                 // the true appId as oauth_client_id before forwarding events to Blizzard).
-                appId: userAgent.origin,
+                appId: origin,
                 deviceConnectivity,
-                sessionId: sessionId,
+                sessionId,
                 appVendorUuid,
+                partnerUuid,
             }) });
     };
     const sendServerEvent = (eventName, eventData) => {
+        const { osName: osType, osVersion } = getPlatformInfo();
         return handler(blizzard.ServerEvent.fromPartial({
             eventName,
-            osType: userAgent.osType,
-            osVersion: userAgent.osVersion,
+            osType,
+            osVersion,
             maxSequenceIdOnInstance: 0,
             sequenceId: sequenceId++,
             eventData,
         }));
     };
-    entries(eventHandlers).forEach(([eventType, createEventData]) => {
-        metricsEventTarget.addEventListener(eventType, ({ detail }) => {
-            // Safety: When iterating over object keys in a mapped type, we lose the association between the key type
-            // and the value type – at each iteration, the key type is a union of all possible keys and the value type
-            // is a union of all possible values. When the value is a function with an argument, and that argument
-            // depends on the key type (which is a union), the contravariance of the argument type means that the union
-            // becomes an intersection. In our case here, this means the compiler expects each argument to contain all
-            // properties from all event types. The cast is safe because the mapped `EventHandlers` type ensures that
-            // `createEventData` takes an argument of the type corresponding its key's `eventType`'s event detail.
-            const [eventName, eventData] = createEventData(makeBlizzardEvent(detail));
-            sendServerEvent(eventName, eventData);
-        });
+    // Add event listeners for each event type and turn those listeners into Observables
+    const metricsEvents = entries(eventHandlers).map(([eventType, createEventData]) => fromEvent(metricsEventTarget, eventType).pipe(map((event) => ({ event, createEventData }))));
+    // Subscribe to all the metrics events and combine them with the app/partner IDs obtained
+    // from remote configuration -- this means we'll queue up any metrics events that occur
+    // before remote config is downloaded, and send them once that config is available.
+    merge(...metricsEvents)
+        .pipe(combineLatestWith(appVendorAndPartnerUuid))
+        .subscribe(([{ event, createEventData }, { appVendorUuid, partnerUuid }]) => {
+        // Safety: When iterating over object keys in a mapped type, we lose the association between the key type
+        // and the value type – at each iteration, the key type is a union of all possible keys and the value type
+        // is a union of all possible values. When the value is a function with an argument, and that argument
+        // depends on the key type (which is a union), the contravariance of the argument type means that the union
+        // becomes an intersection. In our case here, this means the compiler expects each argument to contain all
+        // properties from all event types. The cast is safe because the mapped `EventHandlers` type ensures that
+        // `createEventData` takes an argument of the type corresponding its key's `eventType`'s event detail.
+        const [eventName, eventData] = createEventData(makeBlizzardEvent(event.detail, appVendorUuid, partnerUuid));
+        sendServerEvent(eventName, eventData);
     });
 }
+function getAppVendorAndPartnerUuid(configuration, remoteConfiguration) {
+    const vendorAnalyticsPersistence = new ExpiringPersistence(() => vendorUuidExpiry, new IndexedDBPersistence({ databaseName: "SessionHistory" }));
+    return remoteConfiguration.getInitializationConfig().pipe(take(1), switchMap(({ appVendorUuidOptIn }) => {
+        const partnerUuid = configuration.analyticsId;
+        if (appVendorUuidOptIn) {
+            return from(getOrGenerateVendorUuid(vendorAnalyticsPersistence)).pipe(map((appVendorUuid) => ({ appVendorUuid, partnerUuid })));
+        }
+        return of({ appVendorUuid: undefined, partnerUuid });
+    }), catchError((error) => {
+        logger.warn(`Failed to retrieve or generate vendor UUID.`, error);
+        return of({ appVendorUuid: undefined, partnerUuid: configuration.analyticsId });
+    }));
+}
 export const businessEventsReporterFactory = Injectable("businessEventsReporter", [
     metricsEventTargetFactory.token,
     metricsHandlerFactory.token,
@@ -147,87 +168,72 @@ export const businessEventsReporterFactory = Injectable("businessEventsReporter"
     configurationToken,
     remoteConfigurationFactory.token,
 ], (metricsEventTarget, metricsHandler, pageVisibility, configuration, remoteConfiguration) => {
-    const vendorAnalyticsPersistence = new ExpiringPersistence(() => vendorUuidExpiry, new IndexedDBPersistence({ databaseName: "SessionHistory" }));
-    remoteConfiguration
-        .getInitializationConfig()
-        .pipe(take(1), switchMap(({ appVendorUuidOptIn }) => {
-        if (appVendorUuidOptIn) {
-            return from(getOrGenerateVendorUuid(vendorAnalyticsPersistence));
-        }
-        return of(undefined);
-    }), catchError((error) => {
-        logger.warn(`Failed to retrieve or generate vendor UUID.`, error);
-        return of(undefined);
-    }))
-        .subscribe({
-        next: (appVendorUuid) => {
-            /**
-             * This defines a mapping from a business event's external name (the name we document in public
-             * API docs), to its internal representation as a Blizzard ServerEvent.
-             *
-             * It is important that we do this, since the naming of these internal business events are
-             * unintuitive and will not make sense to SDK users.
-             *
-             * To specify the internal event, we must give the ServerEvent's eventName, the specific property
-             *  name which contains the event data (this is a "oneof" property on ServerEvent), and use the
-             * correct event type's `fromPartial` method (this is generated from the ServerEvent protobuf).
-             *
-             * These events are documented here:
-             * https://docs.google.com/document/d/1-kSzFWCWw9Qo3D08FR1_cqeHTsUtk9p3p3uOptzWDTY/
-             *
-             * They are defined in code here:
-             * https://github.sc-corp.net/Snapchat/snapchat/tree/master/blizzard/schema/blizzard-schema/
-             *  codeGen/src/main/java/com/snapchat/analytics/schema/events/cameraKit
-             */
-            listenAndReport(metricsEventTarget, metricsHandler, pageVisibility, {
-                assetDownload: (event) => [
-                    "CAMERA_KIT_ASSET_DOWNLOAD",
-                    { cameraKitAssetDownload: blizzard.CameraKitAssetDownload.fromPartial(event) },
-                ],
-                assetValidationFailed: (event) => [
-                    "CAMERA_KIT_ASSET_VALIDATION_FAILED",
-                    {
-                        cameraKitAssetValidationFailed: blizzard.CameraKitAssetValidationFailed.fromPartial(event),
-                    },
-                ],
-                benchmarkComplete: (event) => [
-                    "CAMERA_KIT_WEB_BENCHMARK_COMPLETE",
-                    {
-                        cameraKitWebBenchmarkComplete: blizzard.CameraKitWebBenchmarkComplete.fromPartial(event),
-                    },
-                ],
-                exception: (event) => [
-                    "CAMERA_KIT_EXCEPTION",
-                    { cameraKitException: blizzard.CameraKitException.fromPartial(event) },
-                ],
-                legalPrompt: (event) => [
-                    "CAMERA_KIT_LEGAL_PROMPT",
-                    { cameraKitLegalPrompt: blizzard.CameraKitLegalPrompt.fromPartial(event) },
-                ],
-                lensDownload: (event) => [
-                    "CAMERA_KIT_LENS_DOWNLOAD",
-                    { cameraKitLensDownload: blizzard.CameraKitLensDownload.fromPartial(event) },
-                ],
-                lensView: (event) => [
-                    "CAMERA_KIT_WEB_LENS_SWIPE",
-                    { cameraKitWebLensSwipe: blizzard.CameraKitWebLensSwipe.fromPartial(event) },
-                ],
-                lensWait: (event) => [
-                    "CAMERA_KIT_LENS_SPIN",
-                    { cameraKitLensSpin: blizzard.CameraKitLensSpin.fromPartial(event) },
-                ],
-                lensContentValidationFailed: (event) => [
-                    "CAMERA_KIT_LENS_CONTENT_VALIDATION_FAILED",
-                    {
-                        cameraKitLensContentValidationFailed: blizzard.CameraKitLensContentValidationFailed.fromPartial(event),
-                    },
-                ],
-                session: (event) => [
-                    "CAMERA_KIT_SESSION",
-                    { cameraKitSession: blizzard.CameraKitSession.fromPartial(event) },
-                ],
-            }, configuration.apiHostname, appVendorUuid);
-        },
-    });
+    const appVendorAndPartnerUuid = getAppVendorAndPartnerUuid(configuration, remoteConfiguration);
+    /**
+     * This defines a mapping from a business event's external name (the name we document in public
+     * API docs), to its internal representation as a Blizzard ServerEvent.
+     *
+     * It is important that we do this, since the naming of these internal business events are
+     * unintuitive and will not make sense to SDK users.
+     *
+     * To specify the internal event, we must give the ServerEvent's eventName, the specific property
+     *  name which contains the event data (this is a "oneof" property on ServerEvent), and use the
+     * correct event type's `fromPartial` method (this is generated from the ServerEvent protobuf).
+     *
+     * These events are documented here:
+     * https://docs.google.com/document/d/1-kSzFWCWw9Qo3D08FR1_cqeHTsUtk9p3p3uOptzWDTY/
+     *
+     * They are defined in code here:
+     * https://github.sc-corp.net/Snapchat/snapchat/tree/master/blizzard/schema/blizzard-schema/
+     *  codeGen/src/main/java/com/snapchat/analytics/schema/events/cameraKit
+     */
+    listenAndReport(metricsEventTarget, metricsHandler, pageVisibility, {
+        assetDownload: (event) => [
+            "CAMERA_KIT_ASSET_DOWNLOAD",
+            { cameraKitAssetDownload: blizzard.CameraKitAssetDownload.fromPartial(event) },
+        ],
+        assetValidationFailed: (event) => [
+            "CAMERA_KIT_ASSET_VALIDATION_FAILED",
+            {
+                cameraKitAssetValidationFailed: blizzard.CameraKitAssetValidationFailed.fromPartial(event),
+            },
+        ],
+        benchmarkComplete: (event) => [
+            "CAMERA_KIT_WEB_BENCHMARK_COMPLETE",
+            {
+                cameraKitWebBenchmarkComplete: blizzard.CameraKitWebBenchmarkComplete.fromPartial(event),
+            },
+        ],
+        exception: (event) => [
+            "CAMERA_KIT_EXCEPTION",
+            { cameraKitException: blizzard.CameraKitException.fromPartial(event) },
+        ],
+        legalPrompt: (event) => [
+            "CAMERA_KIT_LEGAL_PROMPT",
+            { cameraKitLegalPrompt: blizzard.CameraKitLegalPrompt.fromPartial(event) },
+        ],
+        lensDownload: (event) => [
+            "CAMERA_KIT_LENS_DOWNLOAD",
+            { cameraKitLensDownload: blizzard.CameraKitLensDownload.fromPartial(event) },
+        ],
+        lensView: (event) => [
+            "CAMERA_KIT_WEB_LENS_SWIPE",
+            { cameraKitWebLensSwipe: blizzard.CameraKitWebLensSwipe.fromPartial(event) },
+        ],
+        lensWait: (event) => [
+            "CAMERA_KIT_LENS_SPIN",
+            { cameraKitLensSpin: blizzard.CameraKitLensSpin.fromPartial(event) },
+        ],
+        lensContentValidationFailed: (event) => [
+            "CAMERA_KIT_LENS_CONTENT_VALIDATION_FAILED",
+            {
+                cameraKitLensContentValidationFailed: blizzard.CameraKitLensContentValidationFailed.fromPartial(event),
+            },
+        ],
+        session: (event) => [
+            "CAMERA_KIT_SESSION",
+            { cameraKitSession: blizzard.CameraKitSession.fromPartial(event) },
+        ],
+    }, configuration.apiHostname, appVendorAndPartnerUuid);
 });
 //# sourceMappingURL=businessEventsReporter.js.map