npm - brilliantsole - Versions diffs - 0.0.28 → 0.0.29 - Mend

brilliantsole 0.0.28 → 0.0.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (69) hide show

package/build/brilliantsole.cjs +656 -158
package/build/brilliantsole.cjs.map +1 -1
package/build/brilliantsole.js +656 -158
package/build/brilliantsole.js.map +1 -1
package/build/brilliantsole.ls.js +656 -158
package/build/brilliantsole.ls.js.map +1 -1
package/build/brilliantsole.min.js +1 -1
package/build/brilliantsole.min.js.map +1 -1
package/build/brilliantsole.module.d.ts +75 -15
package/build/brilliantsole.module.js +654 -159
package/build/brilliantsole.module.js.map +1 -1
package/build/brilliantsole.module.min.d.ts +75 -15
package/build/brilliantsole.module.min.js +1 -1
package/build/brilliantsole.module.min.js.map +1 -1
package/build/brilliantsole.node.module.d.ts +75 -15
package/build/brilliantsole.node.module.js +654 -159
package/build/brilliantsole.node.module.js.map +1 -1
package/build/dts/BS.d.ts +1 -0
package/build/dts/Device.d.ts +25 -7
package/build/dts/MicrophoneManager.d.ts +88 -0
package/build/dts/connection/BaseConnectionManager.d.ts +2 -2
package/build/dts/devicePair/DevicePair.d.ts +5 -5
package/build/dts/sensor/SensorDataManager.d.ts +3 -3
package/build/dts/utils/AudioUtils.d.ts +2 -0
package/build/index.d.ts +75 -15
package/build/index.node.d.ts +75 -15
package/examples/basic/index.html +108 -53
package/examples/basic/script.js +248 -21
package/examples/camera/barcode-detector.js +109 -0
package/examples/camera/depth-estimation.js +71 -0
package/examples/camera/face-detector.js +119 -0
package/examples/camera/face-landmark.js +111 -0
package/examples/camera/gesture-recognition.js +97 -0
package/examples/camera/hand-landmark.js +74 -0
package/examples/camera/image-segmentation.js +98 -0
package/examples/camera/image-to-text.js +43 -0
package/examples/camera/image-upscale.js +75 -0
package/examples/camera/index.html +129 -0
package/examples/camera/object-detection.js +98 -0
package/examples/camera/pose-landmark.js +60 -0
package/examples/camera/script.js +316 -0
package/examples/camera/utils.js +165 -0
package/examples/camera/yolo-tiny.js +54 -0
package/examples/camera/yolo.js +119 -0
package/examples/edge-impulse/script.js +23 -5
package/examples/glasses-gestures/README.md +11 -0
package/examples/glasses-gestures/edge-impulse-standalone.js +7228 -0
package/examples/glasses-gestures/edge-impulse-standalone.wasm +0 -0
package/examples/glasses-gestures/index.html +69 -0
package/examples/glasses-gestures/run-impulse.js +135 -0
package/examples/glasses-gestures/script.js +226 -0
package/examples/microphone/gender.js +54 -0
package/examples/microphone/index.html +102 -0
package/examples/microphone/script.js +394 -0
package/examples/microphone/utils.js +45 -0
package/examples/microphone/whisper-realtime.js +166 -0
package/examples/microphone/whisper.js +132 -0
package/examples/ukaton-firmware-update/merged-firmware.bin +0 -0
package/examples/webxr-3/components/bs-camera.js +65 -0
package/examples/webxr-3/index.html +134 -0
package/examples/webxr-3/script.js +432 -0
package/package.json +1 -1
package/src/BS.ts +9 -0
package/src/CameraManager.ts +4 -6
package/src/Device.ts +110 -0
package/src/MicrophoneManager.ts +599 -0
package/src/connection/BaseConnectionManager.ts +2 -0
package/src/sensor/SensorDataManager.ts +5 -0
package/src/utils/AudioUtils.ts +65 -0

package/examples/glasses-gestures/edge-impulse-standalone.wasm ADDED Viewed

Binary file

package/examples/glasses-gestures/index.html ADDED Viewed

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Glasses Gestures</title>
+    <style>
+      #features {
+        width: 50%;
+        font-size: 18px;
+      }
+      #results {
+        font-family: monospace;
+        white-space: pre;
+      }
+    </style>
+  </head>
+  <body>
+    <nav>
+      <a href="../../">home</a>
+    </nav>
+    <h1 id="title"></h1>
+    <button id="toggleConnection">connect</button>
+    <h1 id="gesture"></h1>
+    <p id="results"></p>
+    <script src="edge-impulse-standalone.js"></script>
+    <script src="run-impulse.js"></script>
+    <script src="./script.js" type="module"></script>
+    <script>
+      (async () => {
+        return;
+        var classifier = new EdgeImpulseClassifier();
+        await classifier.init();
+        let project = classifier.getProjectInfo();
+        document.querySelector("h1").textContent =
+          project.owner +
+          " / " +
+          project.name +
+          " (version " +
+          project.deploy_version +
+          ")";
+        document.querySelector("#run-inference").onclick = () => {
+          try {
+            let features = document
+              .querySelector("#features")
+              .value.split(",")
+              .map((x) => Number(x.trim()));
+            console.log("classify", features);
+            let res = classifier.classify(features);
+            document.querySelector("#results").textContent = JSON.stringify(
+              res,
+              null,
+              4
+            );
+          } catch (ex) {
+            alert("Failed to classify: " + (ex.message || ex.toString()));
+          }
+        };
+      })();
+    </script>
+  </body>
+</html>

package/examples/glasses-gestures/run-impulse.js ADDED Viewed

@@ -0,0 +1,135 @@
+// Classifier module
+let classifierInitialized = false;
+Module.onRuntimeInitialized = function() {
+    classifierInitialized = true;
+};
+class EdgeImpulseClassifier {
+    _initialized = false;
+    init() {
+        if (classifierInitialized === true) return Promise.resolve();
+        return new Promise((resolve) => {
+            Module.onRuntimeInitialized = () => {
+                classifierInitialized = true;
+                Module.init();
+                resolve();
+            };
+        });
+    }
+    getProjectInfo() {
+        if (!classifierInitialized) throw new Error('Module is not initialized');
+        return this._convertToOrdinaryJsObject(Module.get_project(), Module.emcc_classification_project_t.prototype);
+    }
+    classify(rawData, debug = false) {
+        if (!classifierInitialized) throw new Error('Module is not initialized');
+        const obj = this._arrayToHeap(rawData);
+        let ret = Module.run_classifier(obj.buffer.byteOffset, rawData.length, debug);
+        Module._free(obj.ptr);
+        if (ret.result !== 0) {
+            throw new Error('Classification failed (err code: ' + ret.result + ')');
+        }
+        return this._fillResultStruct(ret);
+    }
+    classifyContinuous(rawData, enablePerfCal = true) {
+        if (!classifierInitialized) throw new Error('Module is not initialized');
+        const obj = this._arrayToHeap(rawData);
+        let ret = Module.run_classifier_continuous(obj.buffer.byteOffset, rawData.length, false, enablePerfCal);
+        Module._free(obj.ptr);
+        if (ret.result !== 0) {
+            throw new Error('Classification failed (err code: ' + ret.result + ')');
+        }
+        return this._fillResultStruct(ret);
+    }
+    getProperties() {
+        if (!classifierInitialized) throw new Error('Module is not initialized');
+        return this._convertToOrdinaryJsObject(Module.get_properties(), Module.emcc_classification_properties_t.prototype);
+    }
+    /**
+     * Override the threshold on a learn block (you can find thresholds via getProperties().thresholds)
+     * @param {*} obj, e.g. { id: 16, min_score: 0.2 } to set min. object detection threshold to 0.2 for block ID 16
+     */
+    setThreshold(obj) {
+        const ret = Module.set_threshold(obj);
+        if (!ret.success) {
+            throw new Error(ret.error);
+        }
+    }
+    _arrayToHeap(data) {
+        let typedArray = new Float32Array(data);
+        let numBytes = typedArray.length * typedArray.BYTES_PER_ELEMENT;
+        let ptr = Module._malloc(numBytes);
+        let heapBytes = new Uint8Array(Module.HEAPU8.buffer, ptr, numBytes);
+        heapBytes.set(new Uint8Array(typedArray.buffer));
+        return { ptr: ptr, buffer: heapBytes };
+    }
+    _convertToOrdinaryJsObject(emboundObj, prototype) {
+        let newObj = { };
+        for (const key of Object.getOwnPropertyNames(prototype)) {
+            const descriptor = Object.getOwnPropertyDescriptor(prototype, key);
+            if (descriptor && typeof descriptor.get === 'function') {
+                newObj[key] = emboundObj[key]; // Evaluates the getter and assigns as an own property.
+            }
+        }
+        return newObj;
+    }
+    _fillResultStruct(ret) {
+        let props = Module.get_properties();
+        let jsResult = {
+            anomaly: ret.anomaly,
+            results: []
+        };
+        for (let cx = 0; cx < ret.size(); cx++) {
+            let c = ret.get(cx);
+            if (props.model_type === 'object_detection' || props.model_type === 'constrained_object_detection') {
+                jsResult.results.push({ label: c.label, value: c.value, x: c.x, y: c.y, width: c.width, height: c.height });
+            }
+            else {
+                jsResult.results.push({ label: c.label, value: c.value });
+            }
+            c.delete();
+        }
+        if (props.has_object_tracking) {
+            jsResult.object_tracking_results = [];
+            for (let cx = 0; cx < ret.object_tracking_size(); cx++) {
+                let c = ret.object_tracking_get(cx);
+                jsResult.object_tracking_results.push({ object_id: c.object_id, label: c.label, value: c.value, x: c.x, y: c.y, width: c.width, height: c.height });
+                c.delete();
+            }
+        }
+        if (props.has_visual_anomaly_detection) {
+            jsResult.visual_ad_max = ret.visual_ad_max;
+            jsResult.visual_ad_mean = ret.visual_ad_mean;
+            jsResult.visual_ad_grid_cells = [];
+            for (let cx = 0; cx < ret.visual_ad_grid_cells_size(); cx++) {
+                let c = ret.visual_ad_grid_cells_get(cx);
+                jsResult.visual_ad_grid_cells.push({ label: c.label, value: c.value, x: c.x, y: c.y, width: c.width, height: c.height });
+                c.delete();
+            }
+        }
+        ret.delete();
+        return jsResult;
+    }
+}

package/examples/glasses-gestures/script.js ADDED Viewed

@@ -0,0 +1,226 @@
+import * as BS from "../../build/brilliantsole.module.js";
+window.BS = BS;
+console.log(BS);
+// MODEL
+var classifier = new EdgeImpulseClassifier();
+await classifier.init();
+window.classifier = classifier;
+let project = classifier.getProjectInfo();
+document.querySelector("#title").textContent =
+  project.owner +
+  " / " +
+  project.name +
+  " (version " +
+  project.deploy_version +
+  ")";
+const gestureTitle = document.getElementById("gesture");
+let threshold = 0.5;
+/** @param {number[]} features */
+function classify(features) {
+  try {
+    let res = classifier.classify(features);
+    document.querySelector("#results").textContent = JSON.stringify(
+      res,
+      null,
+      4
+    );
+    res.results.forEach(({ label, value }, index) => {
+      if (index == 0) {
+        return;
+      }
+      if (value > threshold) {
+        const gesture = label.split("_")[1];
+        console.log({ gesture });
+        if (true) {
+          gestureTitle.innerText = gesture;
+        } else {
+          gestureTitle.innerText = `detected "${gesture}" gesture (score: ${value.toFixed(
+            2
+          )})`;
+        }
+        setTimeout(() => {
+          gestureTitle.innerText = "";
+        }, gestureDelay - 100);
+        lastTimeGestureRecognized = Date.now();
+      }
+    });
+  } catch (ex) {
+    alert("Failed to classify: " + (ex.message || ex.toString()));
+  }
+}
+// DEVICE
+const device = new BS.Device();
+console.log({ device });
+window.device = device;
+// CONNECT
+const toggleConnectionButton = document.getElementById("toggleConnection");
+toggleConnectionButton.addEventListener("click", () =>
+  device.toggleConnection()
+);
+device.addEventListener("connectionStatus", () => {
+  let disabled = false;
+  let innerText = device.connectionStatus;
+  switch (device.connectionStatus) {
+    case "notConnected":
+      innerText = "connect";
+      break;
+    case "connected":
+      innerText = "disconnect";
+      break;
+  }
+  toggleConnectionButton.disabled = disabled;
+  toggleConnectionButton.innerText = innerText;
+});
+// SENSOR CONFIG
+const sensorRate = 20;
+/** @type {BS.SensorType[]} */
+const sensorTypes = ["acceleration"];
+/** @type {BS.SensorConfiguration} */
+const sensorConfiguration = {};
+sensorTypes.forEach((sensorType) => {
+  sensorConfiguration[sensorType] = sensorRate;
+  device.addEventListener(sensorType, (event) => {
+    let data = [];
+    switch (event.message.sensorType) {
+      case "pressure":
+        data = event.message.pressure.sensors.map((sensor) => sensor.rawValue);
+        break;
+      case "acceleration":
+        {
+          const { x, y, z } = event.message.acceleration;
+          data = [x, y, z];
+        }
+        break;
+      case "linearAcceleration":
+        {
+          const { x, y, z } = event.message.linearAcceleration;
+          data = [x, y, z];
+        }
+        break;
+      case "gyroscope":
+        {
+          const { x, y, z } = event.message.gyroscope;
+          data = [x, y, z];
+        }
+        break;
+      case "magnetometer":
+        {
+          const { x, y, z } = event.message.magnetometer;
+          data = [x, y, z];
+        }
+        break;
+    }
+    data = data.map((value) => value * sensorScalars[sensorType]);
+    appendData(event.message.timestamp, sensorType, data);
+  });
+});
+device.addEventListener("connected", () => {
+  device.setSensorConfiguration(sensorConfiguration);
+});
+const sensorScalars = {
+  pressure: 1 / (2 ** 16 - 1),
+  acceleration: 1 / 4,
+  linearAcceleration: 1 / 4,
+  gyroscope: 1 / 720,
+  magnetometer: 1 / 2500,
+};
+// BUFFER
+const time = 600; // ms
+const numberOfSamples = time / sensorRate;
+const numberOfFeaturesInEachSensorType = {};
+BS.SensorTypes.forEach((sensorType) => {
+  switch (sensorType) {
+    case "pressure":
+      numberOfFeaturesInEachSensorType[sensorType] = 8; // change to 16 for ukaton
+      break;
+    case "linearAcceleration":
+    case "acceleration":
+    case "gyroscope":
+    case "magnetometer":
+      numberOfFeaturesInEachSensorType[sensorType] = 3;
+      break;
+  }
+});
+let numberOfFeaturesInOneSample = 0;
+sensorTypes.forEach((sensorType) => {
+  numberOfFeaturesInOneSample += numberOfFeaturesInEachSensorType[sensorType];
+});
+const numberOfFeatures = numberOfFeaturesInOneSample * numberOfSamples;
+console.log({
+  time,
+  numberOfSamples,
+  numberOfFeaturesInOneSample,
+  numberOfFeatures,
+});
+const samples = [];
+let pendingSample;
+let lastTimeClassified = 0;
+let lastTimeGestureRecognized = 0;
+let classificationDelay = 0;
+let gestureDelay = 1000;
+let isClassifying = false;
+/**
+ * @param {number} timestamp
+ * @param {BS.SensorType} sensorType
+ * @param {number[]} data
+ */
+function appendData(timestamp, sensorType, data) {
+  //console.log({ timestamp, sensorType, data });
+  if (!pendingSample || timestamp != pendingSample.timestamp) {
+    pendingSample = { timestamp };
+    //console.log("pendingSample", pendingSample);
+  }
+  pendingSample[sensorType] = data;
+  const gotAllSensorSamples = sensorTypes.every(
+    (sensorType) => sensorType in pendingSample
+  );
+  if (gotAllSensorSamples) {
+    //console.log("got all samples");
+    samples.push(pendingSample);
+    pendingSample = undefined;
+  }
+  //console.log(`collected ${samples.length} samples`);
+  while (samples.length > numberOfSamples) {
+    samples.shift();
+  }
+  if (!isClassifying && samples.length == numberOfSamples) {
+    const now = Date.now();
+    if (
+      now - lastTimeGestureRecognized < gestureDelay ||
+      now - lastTimeClassified < classificationDelay
+    ) {
+      return;
+    }
+    const features = [];
+    samples.forEach((sample) => {
+      const _features = [];
+      sensorTypes.forEach((sensorType) => {
+        _features.push(...sample[sensorType]);
+        features.push(..._features);
+      });
+    });
+    isClassifying = true;
+    //console.log("classifying", features);
+    classify(features);
+    isClassifying = false;
+    lastTimeClassified = now;
+  }
+}

package/examples/microphone/gender.js ADDED Viewed

@@ -0,0 +1,54 @@
+import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.2.4";
+import { registerModel } from "./utils.js";
+let classifier = undefined;
+let isRunning = false;
+const createClassifier = async () => {
+  console.log("creating classifier");
+  classifier = await pipeline(
+    "audio-classification",
+    "Xenova/wav2vec2-large-xlsr-53-gender-recognition-librispeech"
+  );
+  console.log("created classifier", classifier);
+};
+registerModel(
+  "gender",
+  () => {
+    if (!classifier) {
+      createClassifier();
+    }
+  },
+  () => {},
+  async (microphoneRecordingAudio, mediaResultsElement) => {
+    if (!classifier) {
+      console.error("classifier not created yet");
+      return;
+    }
+    if (isRunning) {
+      return;
+    }
+    isRunning = true;
+    const output = await classifier(microphoneRecordingAudio.src);
+    isRunning = false;
+    console.log("output", output);
+    let maxLabel, maxScore;
+    output.forEach(({ label, score }) => {
+      maxLabel ??= label;
+      maxScore ??= score;
+      if (score > maxScore) {
+        maxScore = score;
+        maxLabel = label;
+      }
+    });
+    mediaResultsElement.innerText = `${maxLabel}: ${maxScore.toFixed(3)}`;
+  }
+);
+const sampleOutput = [
+  { label: "male", score: 0.9976564049720764 },
+  { label: "female", score: 0.002343568252399564 },
+];

package/examples/microphone/index.html ADDED Viewed

@@ -0,0 +1,102 @@
+<html>
+  <head>
+    <title>Microphone | BrilliantSole JavaScript SDK</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <link rel="shortcut icon" type="image/x-icon" href="/favicon.ico" />
+    <script src="https://unpkg.com/peaks.js/dist/peaks.js"></script>
+    <script type="module" src="./script.js"></script>
+    <script type="module" src="./whisper.js"></script>
+    <script type="module" src="./whisper-realtime.js"></script>
+    <script type="module" src="./gender.js"></script>
+  </head>
+  <style>
+    #zoomview-container,
+    #overview-container {
+      width: 1000px;
+      height: 100px;
+    }
+    .highlighted {
+      color: green;
+    }
+  </style>
+  <body>
+    <nav>
+      <a href="../../">home</a>
+    </nav>
+    <h1>Microphone | BrilliantSole JavaScript SDK</h1>
+    <button id="toggleConnection">connect</button>
+    <label>
+      model type
+      <select id="modelType">
+        <optgroup label="model type">
+          <option>none</option>
+        </optgroup>
+      </select>
+    </label>
+    <br />
+    <label><b>status:</b> <span id="microphoneStatus"></span></label>
+    <button disabled hidden id="toggleMicrophone">start microphone</button>
+    <button disabled id="startMicrophone">start microphone</button>
+    <button disabled id="stopMicrophone">stop microphone</button>
+    <button disabled id="enableMicrphoneVad">
+      enable microphone voice activity detection
+    </button>
+    <br />
+    <button disabled id="toggleMicrophoneRecording">start recording</button>
+    <br />
+    <label>
+      <b>microphone configuration: </b>
+      <pre id="microphoneConfigurationPre"></pre>
+    </label>
+    <br />
+    <div id="microphoneConfiguration">
+      <template id="microphoneConfigurationTypeTemplate">
+        <label class="microphoneConfigurationType">
+          <b class="type"></b> <span hidden></span>
+          <select disabled>
+            <optgroup></optgroup>
+          </select>
+        </label>
+      </template>
+    </div>
+    <div>
+      <b>stream</b>
+      <br />
+      <audio id="microphoneStream" controls autoplay></audio>
+    </div>
+    <label>
+      visualization type
+      <select id="audioVisualizationType">
+        <optgroup label="type">
+          <option>waveform</option>
+          <option>fft</option>
+        </optgroup>
+      </select>
+    </label>
+    <br />
+    <canvas id="audioVisualizer" width="1000" height="200"></canvas>
+    <h1 id="modelResults"></h1>
+    <div>
+      <b>recording</b>
+      <label>
+        auto play
+        <input type="checkbox" id="autoPlayMicrphoneRecording" />
+      </label>
+      <br />
+      <audio id="microphoneRecording" controls></audio>
+    </div>
+    <div id="zoomview-container"></div>
+    <div id="overview-container"></div>
+  </body>
+</html>