@sorisdk/web-audio 0.4.0 → 0.5.1

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { MatcherEphemeralAuthOptions, InitMatcherOptions, MatcherEphemeralAuthResult, PackSource, MatchResult, AudioFingerprintType } from '@sorisdk/matcher';
-export { AudioFingerprintType, PackSource } from '@sorisdk/matcher';
+import { MatcherEphemeralAuthOptions, InitMatcherOptions, MatcherEphemeralAuthResult, AudioMarkerDetection, PackSource, MatchResult, AudioFingerprintType } from '@sorisdk/matcher';
+export { AudioFingerprintType, AudioMarkerConfig, AudioMarkerDetection, PackSource } from '@sorisdk/matcher';
 import { InitAfpgenOptions } from '@sorisdk/afpgen';
 
 declare const DEFAULT_BROWSER_AUDIOPACK_VERSION = "20220401000000";
@@ -52,7 +52,13 @@ interface InitMatchWindowWasmOptions {
     wasmInitInput?: unknown;
 }
 
-type DetectedMatch = Pick<MatchResult, "afpType" | "name" | "position" | "score">;
+interface DetectedMatch extends Pick<MatchResult, "afpType" | "name" | "position" | "score"> {
+    audioMarker?: AudioMarkerDetection | null;
+}
+interface AudioMarkerEvent {
+    marker: string | null;
+    detection: AudioMarkerDetection;
+}
 interface MaterialActivityRequestPayload {
     type: "material";
     material_id: string;
@@ -82,6 +88,7 @@ interface CreateMicrophoneMatcherOptions {
     mediaConstraints?: MediaStreamConstraints;
     stopTracksOnStop?: boolean;
     suspendContextOnStop?: boolean;
+    audiomarker?: boolean;
     activityReporter?: MicrophoneMatcherActivityReporterOptions;
     wasm?: CreateMicrophoneMatcherWasmOptions;
 }
@@ -97,6 +104,7 @@ interface MicrophoneMatcherEventMap extends Record<string, unknown> {
     match: {
         best: DetectedMatch;
     };
+    audiomarker: AudioMarkerEvent;
     campaign: MicrophoneMatcherCampaignEvent;
     nomatch: {};
     stop: {};
@@ -181,6 +189,8 @@ declare class MicrophoneMatcher {
     private readonly audioContext;
     private readonly stopTracksOnStop;
     private readonly suspendContextOnStop;
+    private readonly audiomarker;
+    private readonly captureSampleRate;
     private readonly audioGraphFactory;
     private readonly packSource;
     private readonly options;
@@ -195,8 +205,12 @@ declare class MicrophoneMatcher {
     private pendingMatch;
     private captureGeneration;
     private pcmBuffer;
+    private markerPcmBuffer;
     private resampleBuffer;
     private resampleOffset;
+    private latestAudioMarker;
+    private latestAudioMarkerDetection;
+    private audioMarkerRequestId;
     private hopSize;
     private diagnostics;
     constructor(options: CreateMicrophoneMatcherOptions);
@@ -226,6 +240,12 @@ declare class MicrophoneMatcher {
     private queueMatch;
     private runMatchLoop;
     private resetCaptureState;
+    private appendAudioMarkerSamples;
+    private beginAudioMarkerDetection;
+    private detectAudioMarkerForRequest;
+    private matchConfig;
+    private withAudioMarkerForRequest;
+    private emitAudioMarkerIfChanged;
     private normalizeInputSamples;
     private resampleToTargetRate;
     private cleanupAfterStartFailure;
@@ -236,6 +256,8 @@ declare class MicrophoneMatcher {
 }
 declare function createMicrophoneMatcher(options?: CreateMicrophoneMatcherOptions): Promise<MicrophoneMatcher>;
 
-declare function requestMicrophoneStream(mediaDevices: MediaDevices, mediaConstraints?: MediaStreamConstraints): Promise<MediaStream>;
+declare function requestMicrophoneStream(mediaDevices: MediaDevices, mediaConstraints?: MediaStreamConstraints, options?: {
+    preferredSampleRate?: number;
+}): Promise<MediaStream>;
 
-export { AudioRecognizer, type AudioRecognizerAuthOptions, type AudioRecognizerOptions, type AuthenticateAndLoadAudioPackOptions, type AuthenticateAndLoadAudioPackResult, type BrowserAudioPackLoader, type BrowserAudioPackState, type BrowserAudioPackStateStore, type BrowserSessionManager, type CreateLocalStorageAudioPackStoreOptions, type CreateLocalStorageSessionManagerOptions, type CreateMicrophoneMatcherOptions, type CreateMicrophoneMatcherWasmOptions, DEFAULT_BROWSER_AUDIOPACK_VERSION, DEFAULT_SORI_API_ENDPOINT, type DetectedMatch, type MaterialActivityRequestPayload, MicrophoneMatcher, type MicrophoneMatcherActivityReporterOptions, type MicrophoneMatcherCampaignEvent, type MicrophoneMatcherEventMap, type MicrophoneMatcherEventName, type MicrophoneMatcherListener, authenticateAndLoadAudioPack, createLocalStorageAudioPackStore, createLocalStorageSessionManager, createMicrophoneMatcher, requestMicrophoneStream };
+export { type AudioMarkerEvent, AudioRecognizer, type AudioRecognizerAuthOptions, type AudioRecognizerOptions, type AuthenticateAndLoadAudioPackOptions, type AuthenticateAndLoadAudioPackResult, type BrowserAudioPackLoader, type BrowserAudioPackState, type BrowserAudioPackStateStore, type BrowserSessionManager, type CreateLocalStorageAudioPackStoreOptions, type CreateLocalStorageSessionManagerOptions, type CreateMicrophoneMatcherOptions, type CreateMicrophoneMatcherWasmOptions, DEFAULT_BROWSER_AUDIOPACK_VERSION, DEFAULT_SORI_API_ENDPOINT, type DetectedMatch, type MaterialActivityRequestPayload, MicrophoneMatcher, type MicrophoneMatcherActivityReporterOptions, type MicrophoneMatcherCampaignEvent, type MicrophoneMatcherEventMap, type MicrophoneMatcherEventName, type MicrophoneMatcherListener, authenticateAndLoadAudioPack, createLocalStorageAudioPackStore, createLocalStorageSessionManager, createMicrophoneMatcher, requestMicrophoneStream };

package/dist/index.js

@@ -144,6 +144,7 @@ import {
 // src/internal.ts
 import { AudioFingerprintType } from "@sorisdk/matcher";
 var TARGET_SAMPLE_RATE = 16e3;
+var AUDIO_MARKER_SAMPLE_RATE = 48e3;
 var DEFAULT_RUNTIME_CONFIG = {
   captureBufferSize: 4096,
   matchWindowMs: 3250,
@@ -157,14 +158,14 @@ var DEFAULT_RUNTIME_CONFIG = {
   matchConfig: void 0,
   initialFingerprintType: AudioFingerprintType.Livestream
 };
-function createBrowserAudioContext() {
-  const AudioContextConstructor = globalThis.AudioContext;
+function createBrowserAudioContext(sampleRate = TARGET_SAMPLE_RATE) {
+  const AudioContextConstructor = globalThis.AudioContext ?? globalThis.webkitAudioContext;
   if (typeof AudioContextConstructor !== "function") {
     throw new Error("AudioContext is not available in this environment");
   }
   try {
     return new AudioContextConstructor({
-      sampleRate: TARGET_SAMPLE_RATE,
+      sampleRate,
       latencyHint: "interactive"
     });
   } catch {
@@ -491,7 +492,7 @@ async function createScriptProcessorGraph({
     }
   };
 }
-async function requestMicrophoneStream(mediaDevices, mediaConstraints) {
+async function requestMicrophoneStream(mediaDevices, mediaConstraints, options) {
   if (!mediaDevices?.getUserMedia) {
     throw new Error("MediaDevices.getUserMedia is not available");
   }
@@ -499,7 +500,7 @@ async function requestMicrophoneStream(mediaDevices, mediaConstraints) {
     mediaConstraints ?? {
       audio: {
         channelCount: 1,
-        sampleRate: 16e3,
+        sampleRate: options?.preferredSampleRate ?? TARGET_SAMPLE_RATE,
         echoCancellation: false,
         noiseSuppression: false,
         autoGainControl: false
@@ -534,6 +535,8 @@ var MicrophoneMatcher = class {
   audioContext;
   stopTracksOnStop;
   suspendContextOnStop;
+  audiomarker;
+  captureSampleRate;
   audioGraphFactory;
   packSource;
   options;
@@ -548,8 +551,12 @@ var MicrophoneMatcher = class {
   pendingMatch = false;
   captureGeneration = 0;
   pcmBuffer = new Float32Array(0);
+  markerPcmBuffer = new Float32Array(0);
   resampleBuffer = new Float32Array(0);
   resampleOffset = 0;
+  latestAudioMarker = null;
+  latestAudioMarkerDetection = null;
+  audioMarkerRequestId = 0;
   hopSize = 0;
   diagnostics = {
     sampleCallbacks: 0,
@@ -564,7 +571,9 @@ var MicrophoneMatcher = class {
     this.options = options;
     this.matcherSession = new MatcherSession(options.wasm?.matcher);
     this.mediaDevices = getBrowserMediaDevices();
-    this.audioContext = createBrowserAudioContext();
+    this.audiomarker = options.audiomarker === true;
+    this.captureSampleRate = this.audiomarker ? AUDIO_MARKER_SAMPLE_RATE : TARGET_SAMPLE_RATE;
+    this.audioContext = createBrowserAudioContext(this.captureSampleRate);
     this.stopTracksOnStop = options.stopTracksOnStop ?? true;
     this.suspendContextOnStop = options.suspendContextOnStop ?? true;
     this.audioGraphFactory = defaultAudioGraphFactory;
@@ -668,7 +677,9 @@ var MicrophoneMatcher = class {
       if (this.audioContext.state === "suspended") {
         await this.audioContext.resume();
       }
-      const stream = await requestMicrophoneStream(this.mediaDevices, this.options.mediaConstraints);
+      const stream = await requestMicrophoneStream(this.mediaDevices, this.options.mediaConstraints, {
+        preferredSampleRate: this.captureSampleRate
+      });
       this.stream = stream;
       this.events.emit("microphoneready", {
         stream,
@@ -738,6 +749,9 @@ var MicrophoneMatcher = class {
     const captureGeneration = this.captureGeneration;
     this.diagnostics.sampleCallbacks += 1;
     this.diagnostics.inputSamples += samples.length;
+    if (this.audiomarker) {
+      this.appendAudioMarkerSamples(samples);
+    }
     const normalizedSamples = this.normalizeInputSamples(samples);
     if (normalizedSamples.length === 0) {
       return;
@@ -782,24 +796,28 @@ var MicrophoneMatcher = class {
           continue;
         }
         this.diagnostics.readyQueries += 1;
-        let distinctBest = null;
+        let matchedBest = null;
         let shouldEmitNoMatch = false;
         const handleMatch = (event) => {
           if (!event.best) {
             return;
           }
-          distinctBest = event.best;
+          matchedBest = event.best;
           shouldEmitNoMatch = false;
         };
         const handleNoMatch = () => {
-          distinctBest = null;
+          matchedBest = null;
           shouldEmitNoMatch = true;
         };
         this.matcherSession.on("match", handleMatch);
         this.matcherSession.on("nomatch", handleNoMatch);
         try {
           this.diagnostics.matchRequests += 1;
-          await this.matcherSession.bestMatch(query, DEFAULT_RUNTIME_CONFIG.matchConfig);
+          const bestMatchPromise = this.matcherSession.bestMatch(query, this.matchConfig());
+          const audioMarkerRequestId = this.beginAudioMarkerDetection(captureGeneration);
+          await bestMatchPromise;
+          const distinctBest = this.withAudioMarkerForRequest(matchedBest, audioMarkerRequestId);
+          matchedBest = distinctBest;
         } finally {
           this.matcherSession.off("match", handleMatch);
           this.matcherSession.off("nomatch", handleNoMatch);
@@ -807,9 +825,9 @@ var MicrophoneMatcher = class {
         if (captureGeneration !== this.captureGeneration || !this.running) {
           return;
         }
-        if (distinctBest) {
-          this.events.emit("match", { best: distinctBest });
-          void this.reportCampaign(distinctBest, captureGeneration);
+        if (matchedBest) {
+          this.events.emit("match", { best: matchedBest });
+          void this.reportCampaign(matchedBest, captureGeneration);
         } else if (shouldEmitNoMatch) {
           this.events.emit("nomatch", {});
         }
@@ -825,11 +843,76 @@ var MicrophoneMatcher = class {
     this.pendingMatch = false;
     this.matchInFlight = false;
     this.pcmBuffer = new Float32Array(0);
+    this.markerPcmBuffer = new Float32Array(0);
     this.resampleBuffer = new Float32Array(0);
     this.resampleOffset = 0;
+    this.latestAudioMarker = null;
+    this.latestAudioMarkerDetection = null;
+    this.audioMarkerRequestId += 1;
     this.matchWindowOrThrow().reset();
     await this.extractor?.reset();
   }
+  appendAudioMarkerSamples(samples) {
+    const maxSamples = Math.max(
+      1,
+      Math.ceil(this.audioContext.sampleRate * DEFAULT_RUNTIME_CONFIG.matchWindowMs / 1e3)
+    );
+    const merged = appendFloat32(this.markerPcmBuffer, samples);
+    this.markerPcmBuffer = merged.length > maxSamples ? merged.slice(merged.length - maxSamples) : merged;
+  }
+  beginAudioMarkerDetection(captureGeneration) {
+    if (!this.audiomarker || this.markerPcmBuffer.length === 0) {
+      return null;
+    }
+    const requestId = this.audioMarkerRequestId + 1;
+    this.audioMarkerRequestId = requestId;
+    const markerPcm = float32ToPcm16Le(this.markerPcmBuffer);
+    void this.detectAudioMarkerForRequest(captureGeneration, requestId, markerPcm);
+    return requestId;
+  }
+  async detectAudioMarkerForRequest(captureGeneration, requestId, markerPcm) {
+    try {
+      const detection = await this.matcherSession.detectAudioMarker(
+        markerPcm,
+        Math.round(this.audioContext.sampleRate)
+      );
+      if (requestId !== this.audioMarkerRequestId || captureGeneration !== this.captureGeneration || !this.running) {
+        return null;
+      }
+      this.latestAudioMarkerDetection = { requestId, detection };
+      this.emitAudioMarkerIfChanged(detection);
+      return detection;
+    } catch (error) {
+      emitError(this.events, "audiomarker", error);
+      return null;
+    }
+  }
+  matchConfig() {
+    if (!this.audiomarker) {
+      return DEFAULT_RUNTIME_CONFIG.matchConfig;
+    }
+    return {
+      ...DEFAULT_RUNTIME_CONFIG.matchConfig,
+      audiomarker: true
+    };
+  }
+  withAudioMarkerForRequest(match, requestId) {
+    if (!match || requestId === null || this.latestAudioMarkerDetection?.requestId !== requestId) {
+      return match;
+    }
+    return {
+      ...match,
+      audioMarker: this.latestAudioMarkerDetection.detection
+    };
+  }
+  emitAudioMarkerIfChanged(detection) {
+    const marker = detection.code ?? null;
+    if (marker === this.latestAudioMarker) {
+      return;
+    }
+    this.latestAudioMarker = marker;
+    this.events.emit("audiomarker", { marker, detection });
+  }
   normalizeInputSamples(samples) {
     if (this.audioContext.sampleRate === TARGET_SAMPLE_RATE) {
       return samples;
@@ -916,6 +999,16 @@ function appendFloat32(left, right) {
   merged.set(right, left.length);
   return merged;
 }
+function float32ToPcm16Le(samples) {
+  const bytes = new Uint8Array(samples.length * 2);
+  const view = new DataView(bytes.buffer);
+  for (let index = 0; index < samples.length; index += 1) {
+    const sample = Math.max(-1, Math.min(1, samples[index] ?? 0));
+    const value = sample < 0 ? sample * 32768 : sample * 32767;
+    view.setInt16(index * 2, Math.round(value), true);
+  }
+  return bytes;
+}
 async function createMicrophoneMatcher(options = {}) {
   const matcher = new MicrophoneMatcher(options);
   await matcher.prepare();
@@ -964,6 +1057,7 @@ var FORWARDED_EVENTS = [
   "microphoneready",
   "start",
   "match",
+  "audiomarker",
   "campaign",
   "nomatch",
   "stop",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sorisdk/web-audio",
-  "version": "0.4.0",
+  "version": "0.5.1",
   "description": "Web SDK for browser-based audio recognition with SORI API",
   "type": "module",
   "main": "./dist/index.js",
@@ -34,8 +34,8 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "homepage": "https://docs.soriapi.com/ko/integration/web",
   "dependencies": {
-    "@sorisdk/afpgen": "0.4.0",
-    "@sorisdk/matcher": "0.4.0"
+    "@sorisdk/afpgen": "0.5.1",
+    "@sorisdk/matcher": "0.5.1"
   },
   "devDependencies": {
     "tsup": "^8.5.1",