brilliantsole 0.0.27 → 0.0.29

package/examples/camera/barcode-detector.js

@@ -0,0 +1,109 @@
+import { registerModel } from "./utils.js";
+
+let barcodeDetector;
+let drawBarcode = false;
+
+registerModel(
+  "barcode detector",
+  () => {
+    if (!barcodeDetector) {
+      BarcodeDetector.getSupportedFormats().then((supportedFormats) => {
+        console.log({ supportedFormats });
+        // create new detector
+        barcodeDetector = new BarcodeDetector({
+          formats: supportedFormats,
+        });
+        console.log("barcodeDetector", barcodeDetector);
+      });
+    }
+  },
+  async (image, canvas, context, modelResultsElement) => {
+    if (!barcodeDetector) {
+      console.log("barcodeDetector not created");
+      return;
+    }
+
+    const barcodes = await barcodeDetector.detect(image);
+
+    console.log("barcodes", barcodes);
+
+    const barcode = barcodes[0];
+    if (barcode) {
+      const { boundingBox, cornerPoints, rawValue, format } = sampleBarcodes[0];
+
+      modelResultsElement.innerText = `${format}: ${rawValue}`;
+
+      const { x, y, width, height, top, right, bottom, left } = boundingBox;
+
+      const _x = (x / image.naturalWidth) * canvas.width;
+      const _y = (y / image.naturalHeight) * canvas.height;
+
+      const _width = (width / image.naturalWidth) * canvas.width;
+      const _height = (height / image.naturalHeight) * canvas.height;
+
+      if (drawBarcode) {
+        context.fillStyle = "rgba(0, 191, 255, 0.4)";
+        context.fillRect(_x, _y, _width, _height);
+
+        context.strokeStyle = "white";
+        context.lineWidth = 2;
+        context.strokeRect(_x, _y, _width, _height);
+      }
+
+      context.beginPath();
+      cornerPoints.forEach((cornerPoint, index) => {
+        const { x, y } = cornerPoint;
+
+        const _x = (x / image.naturalWidth) * canvas.width;
+        const _y = (y / image.naturalHeight) * canvas.height;
+
+        if (drawBarcode) {
+          if (index == 0) {
+            context.moveTo(_x, _y);
+          } else {
+            context.lineTo(_x, _y);
+          }
+        }
+      });
+
+      context.closePath();
+      context.fillStyle = "rgba(0, 191, 255, 0.4)";
+      context.fill();
+    }
+  }
+);
+
+const sampleBarcodes = [
+  {
+    boundingBox: {
+      x: 155.0115203857422,
+      y: 288.3844299316406,
+      width: 301.869873046875,
+      height: 298.4500427246094,
+      top: 288.3844299316406,
+      right: 456.8813934326172,
+      bottom: 586.83447265625,
+      left: 155.0115203857422,
+    },
+    cornerPoints: [
+      {
+        x: 384.11309814453125,
+        y: 288.3844299316406,
+      },
+      {
+        x: 456.8813781738281,
+        y: 512.93701171875,
+      },
+      {
+        x: 213.1260986328125,
+        y: 586.83447265625,
+      },
+      {
+        x: 155.0115203857422,
+        y: 341.5698547363281,
+      },
+    ],
+    format: "qr_code",
+    rawValue: "http://en.m.wikipedia.org",
+  },
+];

package/examples/camera/depth-estimation.js

@@ -0,0 +1,71 @@
+import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.2";
+
+import {
+  drawBox,
+  drawGreyscaleImage,
+  registerModel,
+  resizeImage,
+} from "./utils.js";
+
+let depthEstimator = undefined;
+let isRunning = false;
+
+const createDepthEstimator = async () => {
+  depthEstimator = await pipeline(
+    "depth-estimation",
+    "onnx-community/depth-anything-v2-small"
+  );
+  console.log("created depthEstimator", depthEstimator);
+};
+
+registerModel(
+  "depth estimator",
+  () => {
+    if (!depthEstimator) {
+      createDepthEstimator();
+    }
+  },
+  async (
+    image,
+    canvas,
+    context,
+    mediaResultsElement,
+    generatedImageCanvas,
+    generatedImageContext
+  ) => {
+    if (!depthEstimator) {
+      console.error("depthEstimator not created yet");
+      return;
+    }
+    if (isRunning) {
+      return;
+    }
+    try {
+      isRunning = true;
+
+      console.log("running depthEstimator");
+      const src = resizeImage(image, 128, 128);
+      const output = await depthEstimator(src);
+      console.log("depthEstimator output", output);
+
+      if (output.depth) {
+        const { channels, data, height, width } = output.depth;
+        console.log({ channels, data, height, width });
+        if (channels == 1) {
+          generatedImageCanvas.style.display = "";
+          drawGreyscaleImage(
+            data,
+            height,
+            width,
+            generatedImageCanvas,
+            generatedImageContext
+          );
+        }
+      }
+    } catch (error) {
+      console.error("error running depthEstimator", error);
+    } finally {
+      isRunning = false;
+    }
+  }
+);

package/examples/camera/face-detector.js

@@ -0,0 +1,119 @@
+import {
+  FaceDetector,
+  FilesetResolver,
+  DrawingUtils,
+} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
+
+import { drawBox, drawPoint, registerModel } from "./utils.js";
+
+let faceDetector = undefined;
+let runningMode = "VIDEO";
+
+registerModel(
+  "face detector",
+  () => {
+    if (!faceDetector) {
+      createFaceDetector();
+    }
+  },
+  async (image, canvas, context) => {
+    if (!faceDetector) {
+      console.error("faceDetector not created yet");
+      return;
+    }
+
+    const faceDetectorResult = faceDetector.detectForVideo(
+      image,
+      performance.now()
+    );
+
+    //console.log("faceDetectorResult", faceDetectorResult);
+
+    context.save();
+    for (const detection of faceDetectorResult.detections) {
+      const { boundingBox, keypoints } = detection;
+
+      drawBox(boundingBox, image, canvas, context);
+
+      keypoints.forEach((keypoint) => {
+        drawPoint(keypoint, image, canvas, context);
+      });
+    }
+    context.restore();
+  }
+);
+
+const createFaceDetector = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"
+  );
+  faceDetector = await FaceDetector.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_detector/blaze_face_short_range/float16/1/blaze_face_short_range.tflite`,
+      delegate: "GPU",
+    },
+    runningMode: runningMode,
+    minDetectionConfidence: 0.5,
+    minSuppressionThreshold: 0.3,
+  });
+  console.log("created faceDetector", faceDetector);
+};
+
+const sampleResult = {
+  detections: [
+    {
+      categories: [
+        {
+          score: 0.9460012316703796,
+          index: 0,
+          categoryName: "",
+          displayName: "",
+        },
+      ],
+      boundingBox: {
+        originX: 196,
+        originY: 284,
+        width: 290,
+        height: 290,
+      },
+      keypoints: [
+        {
+          x: 0.3776719570159912,
+          y: 0.48935234546661377,
+          score: 0,
+          label: "",
+        },
+        {
+          x: 0.5415752530097961,
+          y: 0.48879799246788025,
+          score: 0,
+          label: "",
+        },
+        {
+          x: 0.4468824565410614,
+          y: 0.5775673985481262,
+          score: 0,
+          label: "",
+        },
+        {
+          x: 0.4519948363304138,
+          y: 0.6720198392868042,
+          score: 0,
+          label: "",
+        },
+        {
+          x: 0.3088231682777405,
+          y: 0.5479761958122253,
+          score: 0,
+          label: "",
+        },
+        {
+          x: 0.6484290957450867,
+          y: 0.549033522605896,
+          score: 0,
+          label: "",
+        },
+      ],
+    },
+  ],
+};

package/examples/camera/face-landmark.js

@@ -0,0 +1,111 @@
+import vision from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
+const { FaceLandmarker, FilesetResolver, DrawingUtils } = vision;
+
+import { registerModel } from "./utils.js";
+
+let faceLandmarker = undefined;
+let runningMode = "VIDEO";
+let drawingUtils;
+
+registerModel(
+  "face landmark",
+  () => {
+    if (!faceLandmarker) {
+      createFaceLandmarker();
+    }
+  },
+  async (image, canvas, context, mediaResultsElement) => {
+    if (!faceLandmarker) {
+      console.error("faceLandmarker not created yet");
+      return;
+    }
+
+    drawingUtils = drawingUtils || new DrawingUtils(context);
+
+    const results = faceLandmarker.detectForVideo(image, performance.now());
+
+    console.log("faceLandmarkerResult", results);
+    const blendShapeCategories = results?.faceBlendshapes?.[0]?.categories;
+    if (blendShapeCategories) {
+      const blendShapeCategoriesString = {};
+      blendShapeCategories.forEach(({ categoryName, score }) => {
+        blendShapeCategoriesString[categoryName] = score;
+      });
+      mediaResultsElement.textContent = JSON.stringify(
+        blendShapeCategoriesString,
+        null,
+        2
+      );
+    }
+
+    if (results.faceLandmarks) {
+      for (const landmarks of results.faceLandmarks) {
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_TESSELATION,
+          { color: "#C0C0C070", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_RIGHT_EYE,
+          { color: "#FF3030", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_RIGHT_EYEBROW,
+          { color: "#FF3030", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_LEFT_EYE,
+          { color: "#30FF30", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_LEFT_EYEBROW,
+          { color: "#30FF30", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_FACE_OVAL,
+          { color: "#E0E0E0", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_LIPS,
+          { color: "#E0E0E0", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_RIGHT_IRIS,
+          { color: "#FF3030", lineWidth: 1 }
+        );
+        drawingUtils.drawConnectors(
+          landmarks,
+          FaceLandmarker.FACE_LANDMARKS_LEFT_IRIS,
+          { color: "#30FF30", lineWidth: 1 }
+        );
+      }
+    }
+  }
+);
+
+const createFaceLandmarker = async () => {
+  const filesetResolver = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm"
+  );
+  faceLandmarker = await FaceLandmarker.createFromOptions(filesetResolver, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task`,
+      delegate: "GPU",
+    },
+    outputFaceBlendshapes: true,
+    runningMode,
+    numFaces: 1,
+    minFaceDetectionConfidence: 0.5,
+    minFacePresenceConfidence: 0.5,
+    minTrackingConfidence: 0.5,
+    outputFacialTransformationMatrixes: false,
+  });
+  console.log("created faceLandmarker", faceLandmarker);
+};

package/examples/camera/gesture-recognition.js

@@ -0,0 +1,97 @@
+import {
+  GestureRecognizer,
+  FilesetResolver,
+  DrawingUtils,
+} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3";
+
+import { registerModel } from "./utils.js";
+
+let gestureRecognizer = undefined;
+let drawingUtils;
+let runningMode = "VIDEO";
+
+registerModel(
+  "gesture recognition",
+  () => {
+    if (!gestureRecognizer) {
+      createGestureRecognizer();
+    }
+  },
+  async (image, canvas, context, modelResultsElement) => {
+    if (!gestureRecognizer) {
+      console.error("gestureRecognizer not created yet");
+      return;
+    }
+
+    const gestureRecognizerResult = gestureRecognizer.recognizeForVideo(
+      image,
+      performance.now()
+    );
+    console.log("gestureRecognizerResult", gestureRecognizerResult);
+
+    if (gestureRecognizerResult.landmarks) {
+      drawingUtils = drawingUtils || new DrawingUtils(context);
+
+      for (const landmarks of gestureRecognizerResult.landmarks) {
+        drawingUtils.drawConnectors(
+          landmarks,
+          GestureRecognizer.HAND_CONNECTIONS,
+          {
+            color: "#00FF00",
+            lineWidth: 3,
+          }
+        );
+        drawingUtils.drawLandmarks(landmarks, {
+          color: "#FF0000",
+          lineWidth: 0,
+          radius: 3,
+        });
+      }
+    }
+    if (gestureRecognizerResult.gestures.length > 0) {
+      const categoryName = gestureRecognizerResult.gestures[0][0].categoryName;
+      const categoryScore = parseFloat(
+        gestureRecognizerResult.gestures[0][0].score * 100
+      ).toFixed(2);
+      const handedness = gestureRecognizerResult.handednesses[0][0].displayName;
+      console.log(
+        `GestureRecognizer: ${categoryName}\n Confidence: ${categoryScore} %\n Handedness: ${handedness}`
+      );
+      modelResultsElement.innerText = `GestureRecognizer: ${categoryName}\n Confidence: ${categoryScore} %\n Handedness: ${handedness}`;
+    }
+  }
+);
+
+const createGestureRecognizer = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.3/wasm"
+  );
+  gestureRecognizer = await GestureRecognizer.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath:
+        "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task",
+      delegate: "GPU",
+    },
+    runningMode: runningMode,
+    numHands: 1,
+
+    /**
+     * The minimum confidence score for the hand detection to be considered
+     * successful. Defaults to 0.5.
+     */
+    minHandDetectionConfidence: 0.5,
+
+    /**
+     * The minimum confidence score of hand presence score in the hand landmark
+     * detection.
+     */
+    minHandPresenceConfidence: 0.5,
+
+    /**
+     * The minimum confidence score for the hand tracking to be considered
+     * successful.
+     */
+    minTrackingConfidence: 0.5,
+  });
+  console.log("created gestureRecognizer", gestureRecognizer);
+};

package/examples/camera/hand-landmark.js

@@ -0,0 +1,74 @@
+import {
+  HandLandmarker,
+  FilesetResolver,
+} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0";
+
+import { registerModel } from "./utils.js";
+
+let handLandmarker = undefined;
+let runningMode = "VIDEO";
+
+registerModel(
+  "hand landmark",
+  () => {
+    if (!handLandmarker) {
+      createHandLandmarker();
+    }
+  },
+  async (image, canvas, context) => {
+    if (!handLandmarker) {
+      console.error("handLandmarker not created yet");
+      return;
+    }
+
+    const handLandmarkerResult = handLandmarker.detectForVideo(
+      image,
+      performance.now()
+    );
+
+    for (const landmarks of handLandmarkerResult.landmarks) {
+      drawConnectors(context, landmarks, HAND_CONNECTIONS, {
+        color: "#00FF00",
+        lineWidth: 3,
+      });
+      drawLandmarks(context, landmarks, {
+        color: "#FF0000",
+        lineWidth: 0,
+        radius: 3,
+      });
+    }
+  }
+);
+
+const createHandLandmarker = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.0/wasm"
+  );
+  handLandmarker = await HandLandmarker.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: `https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task`,
+      delegate: "GPU",
+    },
+    runningMode: runningMode,
+    numHands: 1,
+    /**
+     * The minimum confidence score for the hand detection to be considered successful in palm detection model.
+     */
+    minHandDetectionConfidence: 0.5,
+    /**
+     * The minimum confidence score for the hand presence score in the hand landmark detection model.
+     * In Video mode and Live stream mode, if the hand presence confidence score from the hand landmark model is below this threshold,
+     * Hand Landmarker triggers the palm detection model.
+     * Otherwise, a lightweight hand tracking algorithm determines the location of the hand(s) for subsequent landmark detections.
+     */
+    minHandPresenceConfidence: 0.5,
+    /**
+     * The minimum confidence score for the hand tracking to be considered successful.
+     * This is the bounding box IoU threshold between hands in the current frame and the last frame.
+     * In Video mode and Stream mode of Hand Landmarker, if the tracking fails, Hand Landmarker triggers hand detection.
+     * Otherwise, it skips the hand detection.
+     */
+    minTrackingConfidence: 0.5,
+  });
+  console.log("created handLandmarker", handLandmarker);
+};

package/examples/camera/image-segmentation.js

@@ -0,0 +1,98 @@
+import {
+  ImageSegmenter,
+  FilesetResolver,
+} from "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.2";
+
+import { registerModel } from "./utils.js";
+
+let imageSegmenter = undefined;
+let runningMode = "LIVE_STREAM";
+let labels;
+
+const legendColors = [
+  [255, 197, 0, 255], // Vivid Yellow
+  [128, 62, 117, 255], // Strong Purple
+  [255, 104, 0, 255], // Vivid Orange
+  [166, 189, 215, 255], // Very Light Blue
+  [193, 0, 32, 255], // Vivid Red
+  [206, 162, 98, 255], // Grayish Yellow
+  [129, 112, 102, 255], // Medium Gray
+  [0, 125, 52, 255], // Vivid Green
+  [246, 118, 142, 255], // Strong Purplish Pink
+  [0, 83, 138, 255], // Strong Blue
+  [255, 112, 92, 255], // Strong Yellowish Pink
+  [83, 55, 112, 255], // Strong Violet
+  [255, 142, 0, 255], // Vivid Orange Yellow
+  [179, 40, 81, 255], // Strong Purplish Red
+  [244, 200, 0, 255], // Vivid Greenish Yellow
+  [127, 24, 13, 255], // Strong Reddish Brown
+  [147, 170, 0, 255], // Vivid Yellowish Green
+  [89, 51, 21, 255], // Deep Yellowish Brown
+  [241, 58, 19, 255], // Vivid Reddish Orange
+  [35, 44, 22, 255], // Dark Olive Green
+  [0, 161, 194, 255], // Vivid Blue
+];
+
+registerModel(
+  "image segmenter",
+  () => {
+    if (!imageSegmenter) {
+      createImageSegmenter();
+    }
+  },
+  async (image, canvas, context, mediaResultsElement) => {
+    if (!imageSegmenter) {
+      console.error("imageSegmenter not created yet");
+      return;
+    }
+
+    const result = imageSegmenter.segmentForVideo(image, performance.now());
+
+    console.log("imageSegmenter result", result);
+
+    const { width, height } = result.categoryMask;
+    let imageData = context.getImageData(0, 0, width, height).data;
+    canvas.width = width;
+    canvas.height = height;
+    const mask = result.categoryMask.getAsUint8Array();
+    for (let i in mask) {
+      const legendColor = legendColors[mask[i] % legendColors.length];
+      imageData[i * 4] = (legendColor[0] + imageData[i * 4]) / 2;
+      imageData[i * 4 + 1] = (legendColor[1] + imageData[i * 4 + 1]) / 2;
+      imageData[i * 4 + 2] = (legendColor[2] + imageData[i * 4 + 2]) / 2;
+      imageData[i * 4 + 3] = (legendColor[3] + imageData[i * 4 + 3]) / 2;
+    }
+    const uint8Array = new Uint8ClampedArray(imageData.buffer);
+    const dataNew = new ImageData(uint8Array, width, height);
+    context.putImageData(dataNew, 0, 0);
+  }
+);
+
+const modelAssetPaths = {
+  selfieMulticlass:
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_multiclass_256x256/float32/latest/selfie_multiclass_256x256.tflite",
+  hairSegmenter:
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/hair_segmenter/float32/latest/hair_segmenter.tflite",
+  selfieSegmenter:
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite",
+  deeplab:
+    "https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/latest/deeplab_v3.tflite",
+};
+const createImageSegmenter = async () => {
+  const vision = await FilesetResolver.forVisionTasks(
+    "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.2/wasm"
+  );
+
+  imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
+    baseOptions: {
+      modelAssetPath: modelAssetPaths.selfieMulticlass,
+      delegate: "GPU",
+    },
+
+    runningMode: runningMode,
+    outputCategoryMask: true,
+    outputConfidenceMasks: false,
+  });
+  labels = imageSegmenter.getLabels();
+  console.log("created imageSegmenter", imageSegmenter, labels);
+};