@siteed/expo-audio-stream 1.16.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -1
- package/README.md +1 -1
- package/android/src/main/java/net/siteed/audiostream/AudioAnalysisData.kt +68 -22
- package/android/src/main/java/net/siteed/audiostream/AudioFormatUtils.kt +24 -0
- package/android/src/main/java/net/siteed/audiostream/AudioProcessor.kt +836 -386
- package/android/src/main/java/net/siteed/audiostream/AudioRecorderManager.kt +134 -23
- package/android/src/main/java/net/siteed/audiostream/AudioRecordingService.kt +35 -29
- package/android/src/main/java/net/siteed/audiostream/Constants.kt +1 -0
- package/android/src/main/java/net/siteed/audiostream/ExpoAudioStreamModule.kt +236 -96
- package/android/src/main/java/net/siteed/audiostream/FFT.kt +55 -0
- package/android/src/main/java/net/siteed/audiostream/Features.kt +49 -7
- package/android/src/main/java/net/siteed/audiostream/RecordingConfig.kt +4 -4
- package/build/AudioAnalysis/AudioAnalysis.types.d.ts +55 -47
- package/build/AudioAnalysis/AudioAnalysis.types.d.ts.map +1 -1
- package/build/AudioAnalysis/AudioAnalysis.types.js.map +1 -1
- package/build/AudioAnalysis/extractAudioAnalysis.d.ts +60 -13
- package/build/AudioAnalysis/extractAudioAnalysis.d.ts.map +1 -1
- package/build/AudioAnalysis/extractAudioAnalysis.js +147 -162
- package/build/AudioAnalysis/extractAudioAnalysis.js.map +1 -1
- package/build/ExpoAudioStream.types.d.ts +49 -3
- package/build/ExpoAudioStream.types.d.ts.map +1 -1
- package/build/ExpoAudioStream.types.js.map +1 -1
- package/build/ExpoAudioStream.web.d.ts +2 -0
- package/build/ExpoAudioStream.web.d.ts.map +1 -1
- package/build/ExpoAudioStream.web.js +8 -1
- package/build/ExpoAudioStream.web.js.map +1 -1
- package/build/ExpoAudioStreamModule.d.ts.map +1 -1
- package/build/ExpoAudioStreamModule.js +216 -12
- package/build/ExpoAudioStreamModule.js.map +1 -1
- package/build/WebRecorder.web.d.ts +67 -13
- package/build/WebRecorder.web.d.ts.map +1 -1
- package/build/WebRecorder.web.js +178 -173
- package/build/WebRecorder.web.js.map +1 -1
- package/build/index.d.ts +3 -3
- package/build/index.d.ts.map +1 -1
- package/build/index.js +2 -2
- package/build/index.js.map +1 -1
- package/build/useAudioRecorder.d.ts.map +1 -1
- package/build/useAudioRecorder.js +12 -8
- package/build/useAudioRecorder.js.map +1 -1
- package/build/utils/audioProcessing.d.ts +24 -0
- package/build/utils/audioProcessing.d.ts.map +1 -0
- package/build/utils/audioProcessing.js +133 -0
- package/build/utils/audioProcessing.js.map +1 -0
- package/build/workers/InlineFeaturesExtractor.web.d.ts +1 -1
- package/build/workers/InlineFeaturesExtractor.web.d.ts.map +1 -1
- package/build/workers/InlineFeaturesExtractor.web.js +692 -175
- package/build/workers/InlineFeaturesExtractor.web.js.map +1 -1
- package/build/workers/inlineAudioWebWorker.web.d.ts +1 -1
- package/build/workers/inlineAudioWebWorker.web.d.ts.map +1 -1
- package/build/workers/inlineAudioWebWorker.web.js +3 -2
- package/build/workers/inlineAudioWebWorker.web.js.map +1 -1
- package/ios/AudioAnalysisData.swift +51 -16
- package/ios/AudioProcessingHelpers.swift +710 -26
- package/ios/AudioProcessor.swift +334 -185
- package/ios/AudioStreamManager.swift +66 -22
- package/ios/DataPoint.swift +25 -12
- package/ios/DecodingConfig.swift +47 -0
- package/ios/ExpoAudioStreamModule.swift +189 -104
- package/ios/FFT.swift +62 -0
- package/ios/Features.swift +24 -3
- package/ios/RecordingSettings.swift +9 -7
- package/package.json +2 -1
- package/plugin/build/index.d.ts +2 -0
- package/plugin/build/index.js +10 -3
- package/plugin/src/index.ts +10 -1
- package/src/AudioAnalysis/AudioAnalysis.types.ts +68 -52
- package/src/AudioAnalysis/extractAudioAnalysis.ts +223 -219
- package/src/ExpoAudioStream.types.ts +57 -7
- package/src/ExpoAudioStream.web.ts +8 -1
- package/src/ExpoAudioStreamModule.ts +255 -10
- package/src/WebRecorder.web.ts +231 -243
- package/src/index.ts +5 -3
- package/src/useAudioRecorder.tsx +14 -10
- package/src/utils/audioProcessing.ts +205 -0
- package/src/workers/InlineFeaturesExtractor.web.tsx +692 -175
- package/src/workers/inlineAudioWebWorker.web.tsx +3 -2
|
@@ -12,7 +12,6 @@ export declare class WebRecorder {
|
|
|
12
12
|
private audioWorkletNode;
|
|
13
13
|
private featureExtractorWorker?;
|
|
14
14
|
private source;
|
|
15
|
-
private audioWorkletUrl;
|
|
16
15
|
private emitAudioEventCallback;
|
|
17
16
|
private emitAudioAnalysisCallback;
|
|
18
17
|
private config;
|
|
@@ -28,38 +27,93 @@ export declare class WebRecorder {
|
|
|
28
27
|
private compressedSize;
|
|
29
28
|
private pendingCompressedChunk;
|
|
30
29
|
private readonly wavMimeType;
|
|
31
|
-
|
|
30
|
+
private dataPointIdCounter;
|
|
31
|
+
/**
|
|
32
|
+
* Initializes a new WebRecorder instance for audio recording and processing
|
|
33
|
+
* @param audioContext - The AudioContext to use for recording
|
|
34
|
+
* @param source - The MediaStreamAudioSourceNode providing the audio input
|
|
35
|
+
* @param recordingConfig - Configuration options for the recording
|
|
36
|
+
* @param emitAudioEventCallback - Callback function for audio data events
|
|
37
|
+
* @param emitAudioAnalysisCallback - Callback function for audio analysis events
|
|
38
|
+
* @param logger - Optional logger for debugging information
|
|
39
|
+
*/
|
|
40
|
+
constructor({ audioContext, source, recordingConfig, emitAudioEventCallback, emitAudioAnalysisCallback, logger, }: {
|
|
32
41
|
audioContext: AudioContext;
|
|
33
42
|
source: MediaStreamAudioSourceNode;
|
|
34
43
|
recordingConfig: RecordingConfig;
|
|
35
|
-
audioWorkletUrl: string;
|
|
36
44
|
emitAudioEventCallback: EmitAudioEventFunction;
|
|
37
45
|
emitAudioAnalysisCallback: EmitAudioAnalysisFunction;
|
|
38
46
|
logger?: ConsoleLike;
|
|
39
47
|
});
|
|
48
|
+
/**
|
|
49
|
+
* Initializes the audio worklet using an inline script
|
|
50
|
+
* Creates and connects the audio processing pipeline
|
|
51
|
+
*/
|
|
40
52
|
init(): Promise<void>;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
53
|
+
/**
|
|
54
|
+
* Initializes the feature extractor worker for audio analysis
|
|
55
|
+
* Creates an inline worker from a blob for audio feature extraction
|
|
56
|
+
*/
|
|
57
|
+
initFeatureExtractorWorker(): void;
|
|
58
|
+
/**
|
|
59
|
+
* Processes audio analysis results from the feature extractor worker
|
|
60
|
+
* Updates the audio analysis data and emits events
|
|
61
|
+
* @param event - The event containing audio analysis results
|
|
62
|
+
*/
|
|
44
63
|
handleFeatureExtractorMessage(event: AudioFeaturesEvent): void;
|
|
64
|
+
/**
|
|
65
|
+
* Resets the data point ID counter
|
|
66
|
+
* Used when starting a new recording
|
|
67
|
+
*/
|
|
68
|
+
resetDataPointCounter(): void;
|
|
69
|
+
/**
|
|
70
|
+
* Starts the audio recording process
|
|
71
|
+
* Connects the audio nodes and begins capturing audio data
|
|
72
|
+
*/
|
|
45
73
|
start(): void;
|
|
74
|
+
/**
|
|
75
|
+
* Stops the audio recording process and returns the recorded data
|
|
76
|
+
* @returns Promise resolving to an object containing PCM data and optional compressed blob
|
|
77
|
+
*/
|
|
46
78
|
stop(): Promise<{
|
|
47
79
|
pcmData: Float32Array;
|
|
48
80
|
compressedBlob?: Blob;
|
|
49
81
|
}>;
|
|
82
|
+
/**
|
|
83
|
+
* Cleans up resources when recording is stopped
|
|
84
|
+
* Closes audio context and disconnects nodes
|
|
85
|
+
*/
|
|
50
86
|
private cleanup;
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
87
|
+
/**
|
|
88
|
+
* Pauses the audio recording process
|
|
89
|
+
* Disconnects audio nodes and pauses the media recorder
|
|
90
|
+
*/
|
|
54
91
|
pause(): void;
|
|
92
|
+
/**
|
|
93
|
+
* Stops all media stream tracks to release hardware resources
|
|
94
|
+
* Ensures recording indicators (like microphone icon) are turned off
|
|
95
|
+
*/
|
|
55
96
|
stopMediaStreamTracks(): void;
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
97
|
+
/**
|
|
98
|
+
* Determines the audio format capabilities of the current audio context
|
|
99
|
+
* @param sampleRate - The sample rate to check
|
|
100
|
+
* @returns Object containing format information (sample rate, bit depth, channels)
|
|
101
|
+
*/
|
|
60
102
|
private checkAudioContextFormat;
|
|
103
|
+
/**
|
|
104
|
+
* Resumes a paused recording
|
|
105
|
+
* Reconnects audio nodes and resumes the media recorder
|
|
106
|
+
*/
|
|
61
107
|
resume(): void;
|
|
108
|
+
/**
|
|
109
|
+
* Initializes the compressed media recorder if compression is enabled
|
|
110
|
+
* Sets up event handlers for compressed audio data
|
|
111
|
+
*/
|
|
62
112
|
private initializeCompressedRecorder;
|
|
113
|
+
/**
|
|
114
|
+
* Processes features if enabled
|
|
115
|
+
*/
|
|
116
|
+
processFeatures(chunk: Float32Array, sampleRate: number, chunkPosition: number, startPosition: number, endPosition: number, samples: number): void;
|
|
63
117
|
}
|
|
64
118
|
export {};
|
|
65
119
|
//# sourceMappingURL=WebRecorder.web.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"WebRecorder.web.d.ts","sourceRoot":"","sources":["../src/WebRecorder.web.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACnE,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACtE,OAAO,EACH,yBAAyB,EACzB,sBAAsB,EACzB,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"WebRecorder.web.d.ts","sourceRoot":"","sources":["../src/WebRecorder.web.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACnE,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACtE,OAAO,EACH,yBAAyB,EACzB,sBAAsB,EACzB,MAAM,uBAAuB,CAAA;AAa9B,UAAU,kBAAkB;IACxB,IAAI,EAAE;QACF,OAAO,EAAE,MAAM,CAAA;QACf,MAAM,EAAE,aAAa,CAAA;KACxB,CAAA;CACJ;AASD,qBAAa,WAAW;IACpB,OAAO,CAAC,YAAY,CAAc;IAClC,OAAO,CAAC,gBAAgB,CAAmB;IAC3C,OAAO,CAAC,sBAAsB,CAAC,CAAQ;IACvC,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,sBAAsB,CAAwB;IACtD,OAAO,CAAC,yBAAyB,CAA2B;IAC5D,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,QAAQ,CAAY;IAC5B,OAAO,CAAC,gBAAgB,CAAQ;IAChC,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,cAAc,CAAQ;IAC9B,OAAO,CAAC,iBAAiB,CAAe;IACxC,OAAO,CAAC,WAAW,CAAY;IAC/B,OAAO,CAAC,MAAM,CAAC,CAAa;IAC5B,OAAO,CAAC,uBAAuB,CAA6B;IAC5D,OAAO,CAAC,gBAAgB,CAAa;IACrC,OAAO,CAAC,cAAc,CAAY;IAClC,OAAO,CAAC,sBAAsB,CAAoB;IAClD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,kBAAkB,CAAY;IAEtC;;;;;;;;OAQG;gBACS,EACR,YAAY,EACZ,MAAM,EACN,eAAe,EACf,sBAAsB,EACtB,yBAAyB,EACzB,MAAM,GACT,EAAE;QACC,YAAY,EAAE,YAAY,CAAA;QAC1B,MAAM,EAAE,0BAA0B,CAAA;QAClC,eAAe,EAAE,eAAe,CAAA;QAChC,sBAAsB,EAAE,sBAAsB,CAAA;QAC9C,yBAAyB,EAAE,yBAAyB,CAAA;QACpD,MAAM,CAAC,EAAE,WAAW,CAAA;KACvB;IAmDD;;;OAGG;IACG,IAAI;IAqHV;;;OAGG;IACH,0BAA0B;IAuB1B;;;;OAIG;IACH,6BAA6B,CAAC,KAAK,EAAE,kBAAkB;IA+FvD;;;OAGG;IACH,qBAAqB;IAYrB;;;OAGG;IACH,KAAK;IAaL;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC;QAAE,OAAO,EAAE,YAAY,CAAC;QAAC,cAAc,CAAC,EAAE,IAAI,CAAA;KAAE,CAAC;IAqBvE;;;OAGG;IACH,OAAO,CAAC,OAAO;IAaf;;;OAGG;IACH,KAAK;IAOL;;;OAGG;IACH,qBAAqB;IAMrB;;;;OAIG;IACH,OAAO,CAAC,uBAAuB;IAoB/B;;;OAGG;IACH,MAAM;IAON;;;OAGG;IACH,OAAO,CAAC,4BAA4B;IAkCpC;;OAEG;IACH,eAAe,CACX,KAAK,EAAE,YAAY,EACnB,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,EACrB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM;CAsBtB"}
|
package/build/WebRecorder.web.js
CHANGED
|
@@ -1,28 +1,17 @@
|
|
|
1
|
-
// src/WebRecorder.ts
|
|
2
|
-
import { convertPCMToFloat32 } from './utils/convertPCMToFloat32';
|
|
1
|
+
// packages/expo-audio-stream/src/WebRecorder.web.ts
|
|
3
2
|
import { encodingToBitDepth } from './utils/encodingToBitDepth';
|
|
4
|
-
import { writeWavHeader } from './utils/writeWavHeader';
|
|
5
3
|
import { InlineFeaturesExtractor } from './workers/InlineFeaturesExtractor.web';
|
|
6
4
|
import { InlineAudioWebWorker } from './workers/inlineAudioWebWorker.web';
|
|
7
5
|
const DEFAULT_WEB_BITDEPTH = 32;
|
|
8
|
-
const
|
|
6
|
+
const DEFAULT_SEGMENT_DURATION_MS = 100;
|
|
9
7
|
const DEFAULT_WEB_INTERVAL = 500;
|
|
10
8
|
const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1;
|
|
11
|
-
const DEFAULT_ALGORITHM = 'rms';
|
|
12
9
|
const TAG = 'WebRecorder';
|
|
13
|
-
const STOP_PERFORMANCE_MARKS = {
|
|
14
|
-
STOP_INITIATED: 'stopInitiated',
|
|
15
|
-
COMPRESSED_RECORDING_STOP: 'compressedRecordingStop',
|
|
16
|
-
AUDIO_WORKLET_STOP: 'audioWorkletStop',
|
|
17
|
-
CLEANUP: 'cleanup',
|
|
18
|
-
TOTAL_STOP_TIME: 'totalStopTime',
|
|
19
|
-
};
|
|
20
10
|
export class WebRecorder {
|
|
21
11
|
audioContext;
|
|
22
12
|
audioWorkletNode;
|
|
23
13
|
featureExtractorWorker;
|
|
24
14
|
source;
|
|
25
|
-
audioWorkletUrl;
|
|
26
15
|
emitAudioEventCallback;
|
|
27
16
|
emitAudioAnalysisCallback;
|
|
28
17
|
config;
|
|
@@ -38,10 +27,19 @@ export class WebRecorder {
|
|
|
38
27
|
compressedSize = 0;
|
|
39
28
|
pendingCompressedChunk = null;
|
|
40
29
|
wavMimeType = 'audio/wav';
|
|
41
|
-
|
|
30
|
+
dataPointIdCounter = 0; // Add this property to track the counter
|
|
31
|
+
/**
|
|
32
|
+
* Initializes a new WebRecorder instance for audio recording and processing
|
|
33
|
+
* @param audioContext - The AudioContext to use for recording
|
|
34
|
+
* @param source - The MediaStreamAudioSourceNode providing the audio input
|
|
35
|
+
* @param recordingConfig - Configuration options for the recording
|
|
36
|
+
* @param emitAudioEventCallback - Callback function for audio data events
|
|
37
|
+
* @param emitAudioAnalysisCallback - Callback function for audio analysis events
|
|
38
|
+
* @param logger - Optional logger for debugging information
|
|
39
|
+
*/
|
|
40
|
+
constructor({ audioContext, source, recordingConfig, emitAudioEventCallback, emitAudioAnalysisCallback, logger, }) {
|
|
42
41
|
this.audioContext = audioContext;
|
|
43
42
|
this.source = source;
|
|
44
|
-
this.audioWorkletUrl = audioWorkletUrl;
|
|
45
43
|
this.emitAudioEventCallback = emitAudioEventCallback;
|
|
46
44
|
this.emitAudioAnalysisCallback = emitAudioAnalysisCallback;
|
|
47
45
|
this.config = recordingConfig;
|
|
@@ -66,15 +64,14 @@ export class WebRecorder {
|
|
|
66
64
|
DEFAULT_WEB_BITDEPTH;
|
|
67
65
|
this.audioAnalysisData = {
|
|
68
66
|
amplitudeRange: { min: 0, max: 0 },
|
|
67
|
+
rmsRange: { min: 0, max: 0 },
|
|
69
68
|
dataPoints: [],
|
|
70
69
|
durationMs: 0,
|
|
71
70
|
samples: 0,
|
|
72
|
-
amplitudeAlgorithm: recordingConfig.algorithm || DEFAULT_ALGORITHM,
|
|
73
71
|
bitDepth: this.bitDepth,
|
|
74
72
|
numberOfChannels: this.numberOfChannels,
|
|
75
73
|
sampleRate: this.config.sampleRate || this.audioContext.sampleRate,
|
|
76
|
-
|
|
77
|
-
speakerChanges: [],
|
|
74
|
+
segmentDurationMs: this.config.segmentDurationMs ?? DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms segments
|
|
78
75
|
};
|
|
79
76
|
if (recordingConfig.enableProcessing) {
|
|
80
77
|
this.initFeatureExtractorWorker();
|
|
@@ -84,18 +81,18 @@ export class WebRecorder {
|
|
|
84
81
|
this.initializeCompressedRecorder();
|
|
85
82
|
}
|
|
86
83
|
}
|
|
84
|
+
/**
|
|
85
|
+
* Initializes the audio worklet using an inline script
|
|
86
|
+
* Creates and connects the audio processing pipeline
|
|
87
|
+
*/
|
|
87
88
|
async init() {
|
|
88
89
|
try {
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
}
|
|
96
|
-
else {
|
|
97
|
-
await this.audioContext.audioWorklet.addModule(this.audioWorkletUrl);
|
|
98
|
-
}
|
|
90
|
+
// Create and use inline audio worklet
|
|
91
|
+
const blob = new Blob([InlineAudioWebWorker], {
|
|
92
|
+
type: 'application/javascript',
|
|
93
|
+
});
|
|
94
|
+
const url = URL.createObjectURL(blob);
|
|
95
|
+
await this.audioContext.audioWorklet.addModule(url);
|
|
99
96
|
this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'recorder-processor');
|
|
100
97
|
this.audioWorkletNode.port.onmessage = async (event) => {
|
|
101
98
|
const command = event.data.command;
|
|
@@ -110,10 +107,16 @@ export class WebRecorder {
|
|
|
110
107
|
const chunkSize = this.audioContext.sampleRate * 2; // Reduce to 2 seconds chunks
|
|
111
108
|
const sampleRate = event.data.sampleRate ?? this.audioContext.sampleRate;
|
|
112
109
|
const duration = pcmBufferFloat.length / sampleRate;
|
|
110
|
+
// Calculate bytes per sample based on bit depth
|
|
111
|
+
const bytesPerSample = this.bitDepth / 8;
|
|
113
112
|
// Emit chunks without storing them
|
|
114
113
|
for (let i = 0; i < pcmBufferFloat.length; i += chunkSize) {
|
|
115
114
|
const chunk = pcmBufferFloat.slice(i, i + chunkSize);
|
|
116
115
|
const chunkPosition = this.position + i / sampleRate;
|
|
116
|
+
// Calculate byte positions and samples
|
|
117
|
+
const startPosition = Math.floor(i * bytesPerSample);
|
|
118
|
+
const endPosition = Math.floor((i + chunk.length) * bytesPerSample);
|
|
119
|
+
const samples = chunk.length; // Number of samples in this chunk
|
|
117
120
|
// Process features if enabled
|
|
118
121
|
if (this.config.enableProcessing &&
|
|
119
122
|
this.featureExtractorWorker) {
|
|
@@ -121,14 +124,17 @@ export class WebRecorder {
|
|
|
121
124
|
command: 'process',
|
|
122
125
|
channelData: chunk,
|
|
123
126
|
sampleRate,
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
algorithm: this.config.algorithm || 'rms',
|
|
127
|
+
segmentDurationMs: this.config.segmentDurationMs ??
|
|
128
|
+
DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms
|
|
127
129
|
bitDepth: this.bitDepth,
|
|
128
|
-
fullAudioDurationMs:
|
|
130
|
+
fullAudioDurationMs: chunkPosition * 1000,
|
|
129
131
|
numberOfChannels: this.numberOfChannels,
|
|
130
132
|
features: this.config.features,
|
|
131
|
-
|
|
133
|
+
intervalAnalysis: this.config.intervalAnalysis,
|
|
134
|
+
startPosition,
|
|
135
|
+
endPosition,
|
|
136
|
+
samples,
|
|
137
|
+
});
|
|
132
138
|
}
|
|
133
139
|
// Emit chunk immediately
|
|
134
140
|
this.emitAudioEventCallback({
|
|
@@ -159,6 +165,7 @@ export class WebRecorder {
|
|
|
159
165
|
exportBitDepth: this.exportBitDepth,
|
|
160
166
|
channels: this.numberOfChannels,
|
|
161
167
|
interval: this.config.interval ?? DEFAULT_WEB_INTERVAL,
|
|
168
|
+
// enableLogging: !!this.logger,
|
|
162
169
|
});
|
|
163
170
|
// Connect the source to the AudioWorkletNode and start recording
|
|
164
171
|
this.source.connect(this.audioWorkletNode);
|
|
@@ -168,29 +175,11 @@ export class WebRecorder {
|
|
|
168
175
|
console.error(`[${TAG}] Failed to initialize WebRecorder`, error);
|
|
169
176
|
}
|
|
170
177
|
}
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
// We keep the url during dev and use the blob in production.
|
|
177
|
-
this.featureExtractorWorker = new Worker(new URL(featuresExtratorUrl, window.location.href));
|
|
178
|
-
this.featureExtractorWorker.onmessage =
|
|
179
|
-
this.handleFeatureExtractorMessage.bind(this);
|
|
180
|
-
this.featureExtractorWorker.onerror =
|
|
181
|
-
this.handleWorkerError.bind(this);
|
|
182
|
-
}
|
|
183
|
-
else {
|
|
184
|
-
// Fallback to the inline worker if the URL is not provided
|
|
185
|
-
this.initFallbackWorker();
|
|
186
|
-
}
|
|
187
|
-
}
|
|
188
|
-
catch (error) {
|
|
189
|
-
console.error(`[${TAG}] Failed to initialize feature extractor worker`, error);
|
|
190
|
-
this.initFallbackWorker();
|
|
191
|
-
}
|
|
192
|
-
}
|
|
193
|
-
initFallbackWorker() {
|
|
178
|
+
/**
|
|
179
|
+
* Initializes the feature extractor worker for audio analysis
|
|
180
|
+
* Creates an inline worker from a blob for audio feature extraction
|
|
181
|
+
*/
|
|
182
|
+
initFeatureExtractorWorker() {
|
|
194
183
|
try {
|
|
195
184
|
const blob = new Blob([InlineFeaturesExtractor], {
|
|
196
185
|
type: 'application/javascript',
|
|
@@ -200,44 +189,115 @@ export class WebRecorder {
|
|
|
200
189
|
this.featureExtractorWorker.onmessage =
|
|
201
190
|
this.handleFeatureExtractorMessage.bind(this);
|
|
202
191
|
this.featureExtractorWorker.onerror = (error) => {
|
|
203
|
-
console.error(`[${TAG}]
|
|
192
|
+
console.error(`[${TAG}] Feature extractor worker error:`, error);
|
|
204
193
|
};
|
|
205
|
-
this.logger?.log('
|
|
194
|
+
this.logger?.log('Feature extractor worker initialized successfully');
|
|
206
195
|
}
|
|
207
196
|
catch (error) {
|
|
208
|
-
console.error(`[${TAG}] Failed to initialize
|
|
197
|
+
console.error(`[${TAG}] Failed to initialize feature extractor worker`, error);
|
|
209
198
|
}
|
|
210
199
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
200
|
+
/**
|
|
201
|
+
* Processes audio analysis results from the feature extractor worker
|
|
202
|
+
* Updates the audio analysis data and emits events
|
|
203
|
+
* @param event - The event containing audio analysis results
|
|
204
|
+
*/
|
|
214
205
|
handleFeatureExtractorMessage(event) {
|
|
215
206
|
if (event.data.command === 'features') {
|
|
216
207
|
const segmentResult = event.data.result;
|
|
217
|
-
//
|
|
208
|
+
// Update the dataPointIdCounter based on the last ID received
|
|
209
|
+
if (segmentResult.dataPoints &&
|
|
210
|
+
segmentResult.dataPoints.length > 0) {
|
|
211
|
+
const lastDataPoint = segmentResult.dataPoints[segmentResult.dataPoints.length - 1];
|
|
212
|
+
if (lastDataPoint && typeof lastDataPoint.id === 'number') {
|
|
213
|
+
this.dataPointIdCounter = Math.max(this.dataPointIdCounter, lastDataPoint.id + 1);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
console.debug('[WebRecorder] Raw segment result:', {
|
|
217
|
+
dataPointsLength: segmentResult.dataPoints.length,
|
|
218
|
+
durationMs: segmentResult.durationMs,
|
|
219
|
+
sampleRate: segmentResult.sampleRate,
|
|
220
|
+
amplitudeRange: segmentResult.amplitudeRange,
|
|
221
|
+
});
|
|
222
|
+
// Ensure consistent sample rate in the result
|
|
223
|
+
segmentResult.sampleRate =
|
|
224
|
+
this.config.sampleRate || this.audioContext.sampleRate;
|
|
225
|
+
// Update the full audio analysis data with proper range merging
|
|
218
226
|
this.audioAnalysisData.dataPoints.push(...segmentResult.dataPoints);
|
|
219
|
-
this.audioAnalysisData.
|
|
220
|
-
|
|
227
|
+
this.audioAnalysisData.durationMs += segmentResult.durationMs;
|
|
228
|
+
// Make sure the sample rate is consistent
|
|
229
|
+
this.audioAnalysisData.sampleRate = segmentResult.sampleRate;
|
|
230
|
+
// Properly merge amplitude ranges
|
|
221
231
|
if (segmentResult.amplitudeRange) {
|
|
222
|
-
this.audioAnalysisData.amplitudeRange
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
232
|
+
if (!this.audioAnalysisData.amplitudeRange) {
|
|
233
|
+
this.audioAnalysisData.amplitudeRange = {
|
|
234
|
+
...segmentResult.amplitudeRange,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
this.audioAnalysisData.amplitudeRange = {
|
|
239
|
+
min: Math.min(this.audioAnalysisData.amplitudeRange.min, segmentResult.amplitudeRange.min),
|
|
240
|
+
max: Math.max(this.audioAnalysisData.amplitudeRange.max, segmentResult.amplitudeRange.max),
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
// Properly merge RMS ranges
|
|
245
|
+
if (segmentResult.rmsRange) {
|
|
246
|
+
if (!this.audioAnalysisData.rmsRange) {
|
|
247
|
+
this.audioAnalysisData.rmsRange = {
|
|
248
|
+
...segmentResult.rmsRange,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
else {
|
|
252
|
+
this.audioAnalysisData.rmsRange = {
|
|
253
|
+
min: Math.min(this.audioAnalysisData.rmsRange.min, segmentResult.rmsRange.min),
|
|
254
|
+
max: Math.max(this.audioAnalysisData.rmsRange.max, segmentResult.rmsRange.max),
|
|
255
|
+
};
|
|
256
|
+
}
|
|
226
257
|
}
|
|
227
|
-
// Handle the extracted features (e.g., emit an event or log them)
|
|
228
258
|
this.logger?.debug('features event segmentResult', segmentResult);
|
|
229
259
|
this.logger?.debug(`features event audioAnalysisData duration=${this.audioAnalysisData.durationMs}`, this.audioAnalysisData);
|
|
230
260
|
this.emitAudioAnalysisCallback(segmentResult);
|
|
261
|
+
console.debug('[WebRecorder] Updated audioAnalysisData:', {
|
|
262
|
+
dataPointsLength: this.audioAnalysisData.dataPoints.length,
|
|
263
|
+
durationMs: this.audioAnalysisData.durationMs,
|
|
264
|
+
sampleRate: this.audioAnalysisData.sampleRate,
|
|
265
|
+
amplitudeRange: this.audioAnalysisData.amplitudeRange,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Resets the data point ID counter
|
|
271
|
+
* Used when starting a new recording
|
|
272
|
+
*/
|
|
273
|
+
resetDataPointCounter() {
|
|
274
|
+
this.dataPointIdCounter = 0;
|
|
275
|
+
// Reset the counter in the worker
|
|
276
|
+
if (this.featureExtractorWorker) {
|
|
277
|
+
this.featureExtractorWorker.postMessage({
|
|
278
|
+
command: 'resetCounter',
|
|
279
|
+
startCounterFrom: 0,
|
|
280
|
+
});
|
|
231
281
|
}
|
|
232
282
|
}
|
|
283
|
+
/**
|
|
284
|
+
* Starts the audio recording process
|
|
285
|
+
* Connects the audio nodes and begins capturing audio data
|
|
286
|
+
*/
|
|
233
287
|
start() {
|
|
234
288
|
this.source.connect(this.audioWorkletNode);
|
|
235
289
|
this.audioWorkletNode.connect(this.audioContext.destination);
|
|
236
290
|
this.packetCount = 0;
|
|
291
|
+
// Reset the counter when starting a new recording
|
|
292
|
+
this.resetDataPointCounter();
|
|
237
293
|
if (this.compressedMediaRecorder) {
|
|
238
294
|
this.compressedMediaRecorder.start(this.config.interval ?? 1000);
|
|
239
295
|
}
|
|
240
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* Stops the audio recording process and returns the recorded data
|
|
299
|
+
* @returns Promise resolving to an object containing PCM data and optional compressed blob
|
|
300
|
+
*/
|
|
241
301
|
async stop() {
|
|
242
302
|
try {
|
|
243
303
|
if (this.compressedMediaRecorder) {
|
|
@@ -259,6 +319,10 @@ export class WebRecorder {
|
|
|
259
319
|
this.pendingCompressedChunk = null;
|
|
260
320
|
}
|
|
261
321
|
}
|
|
322
|
+
/**
|
|
323
|
+
* Cleans up resources when recording is stopped
|
|
324
|
+
* Closes audio context and disconnects nodes
|
|
325
|
+
*/
|
|
262
326
|
cleanup() {
|
|
263
327
|
if (this.audioContext) {
|
|
264
328
|
this.audioContext.close();
|
|
@@ -271,120 +335,30 @@ export class WebRecorder {
|
|
|
271
335
|
}
|
|
272
336
|
this.stopMediaStreamTracks();
|
|
273
337
|
}
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const [compressedData, workletData] = await Promise.all([
|
|
279
|
-
this.stopCompressedRecording(),
|
|
280
|
-
this.stopAudioWorklet(),
|
|
281
|
-
]);
|
|
282
|
-
this.logger?.debug(`[Performance] Recording stop process completed in ${performance.now() - processStartTime}ms`);
|
|
283
|
-
return {
|
|
284
|
-
pcmData: workletData ??
|
|
285
|
-
new Float32Array(this.audioAnalysisData.dataPoints.length),
|
|
286
|
-
compressedBlob: compressedData,
|
|
287
|
-
};
|
|
288
|
-
}
|
|
289
|
-
// Helper method to stop compressed recording
|
|
290
|
-
stopCompressedRecording() {
|
|
291
|
-
const startTime = performance.now();
|
|
292
|
-
this.logger?.debug(`[Performance][${STOP_PERFORMANCE_MARKS.COMPRESSED_RECORDING_STOP}] Starting compressed recording stop`);
|
|
293
|
-
if (!this.compressedMediaRecorder) {
|
|
294
|
-
this.logger?.debug('[Performance] No compressed recorder to stop');
|
|
295
|
-
return Promise.resolve(undefined);
|
|
296
|
-
}
|
|
297
|
-
return new Promise((resolve) => {
|
|
298
|
-
this.compressedMediaRecorder.onstop = () => {
|
|
299
|
-
const blob = new Blob(this.compressedChunks, {
|
|
300
|
-
type: 'audio/webm;codecs=opus',
|
|
301
|
-
});
|
|
302
|
-
this.logger?.debug(`[Performance][${STOP_PERFORMANCE_MARKS.COMPRESSED_RECORDING_STOP}] Compressed recording stopped in ${performance.now() - startTime}ms, size: ${blob.size}`);
|
|
303
|
-
resolve(blob);
|
|
304
|
-
};
|
|
305
|
-
this.compressedMediaRecorder.stop();
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
// Helper method to stop audio worklet
|
|
309
|
-
stopAudioWorklet() {
|
|
310
|
-
const startTime = performance.now();
|
|
311
|
-
this.logger?.debug(`[Performance][${STOP_PERFORMANCE_MARKS.AUDIO_WORKLET_STOP}] Starting audio worklet stop`);
|
|
312
|
-
if (!this.audioWorkletNode) {
|
|
313
|
-
this.logger?.debug('[Performance] No audio worklet to stop');
|
|
314
|
-
return Promise.resolve(undefined);
|
|
315
|
-
}
|
|
316
|
-
return new Promise((resolve) => {
|
|
317
|
-
const onMessage = (event) => {
|
|
318
|
-
if (event.data.command === 'recordedData') {
|
|
319
|
-
this.audioWorkletNode?.port.removeEventListener('message', onMessage);
|
|
320
|
-
const rawPCMDataFull = event.data.recordedData?.slice(0);
|
|
321
|
-
if (!rawPCMDataFull) {
|
|
322
|
-
this.logger?.debug('[Performance] No PCM data received');
|
|
323
|
-
resolve(undefined);
|
|
324
|
-
return;
|
|
325
|
-
}
|
|
326
|
-
if (this.exportBitDepth !== this.bitDepth) {
|
|
327
|
-
const conversionStart = performance.now();
|
|
328
|
-
convertPCMToFloat32({
|
|
329
|
-
buffer: rawPCMDataFull.buffer,
|
|
330
|
-
bitDepth: this.exportBitDepth,
|
|
331
|
-
skipWavHeader: true,
|
|
332
|
-
logger: this.logger,
|
|
333
|
-
}).then(({ pcmValues }) => {
|
|
334
|
-
this.logger?.debug(`[Performance] PCM conversion completed in ${performance.now() - conversionStart}ms`);
|
|
335
|
-
this.logger?.debug(`[Performance][${STOP_PERFORMANCE_MARKS.AUDIO_WORKLET_STOP}] Audio worklet stopped in ${performance.now() - startTime}ms`);
|
|
336
|
-
resolve(pcmValues);
|
|
337
|
-
});
|
|
338
|
-
}
|
|
339
|
-
else {
|
|
340
|
-
this.logger?.debug(`[Performance][${STOP_PERFORMANCE_MARKS.AUDIO_WORKLET_STOP}] Audio worklet stopped in ${performance.now() - startTime}ms`);
|
|
341
|
-
resolve(rawPCMDataFull);
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
};
|
|
345
|
-
this.audioWorkletNode.port.addEventListener('message', onMessage);
|
|
346
|
-
this.audioWorkletNode.port.postMessage({ command: 'stop' });
|
|
347
|
-
});
|
|
348
|
-
}
|
|
338
|
+
/**
|
|
339
|
+
* Pauses the audio recording process
|
|
340
|
+
* Disconnects audio nodes and pauses the media recorder
|
|
341
|
+
*/
|
|
349
342
|
pause() {
|
|
350
343
|
this.source.disconnect(this.audioWorkletNode); // Disconnect the source from the AudioWorkletNode
|
|
351
344
|
this.audioWorkletNode.disconnect(this.audioContext.destination); // Disconnect the AudioWorkletNode from the destination
|
|
352
345
|
this.audioWorkletNode.port.postMessage({ command: 'pause' });
|
|
353
346
|
this.compressedMediaRecorder?.pause();
|
|
354
347
|
}
|
|
348
|
+
/**
|
|
349
|
+
* Stops all media stream tracks to release hardware resources
|
|
350
|
+
* Ensures recording indicators (like microphone icon) are turned off
|
|
351
|
+
*/
|
|
355
352
|
stopMediaStreamTracks() {
|
|
356
353
|
// Stop all audio tracks to stop the recording icon
|
|
357
354
|
const tracks = this.source.mediaStream.getTracks();
|
|
358
355
|
tracks.forEach((track) => track.stop());
|
|
359
356
|
}
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
sampleRate: this.audioContext.sampleRate,
|
|
366
|
-
numChannels: this.numberOfChannels,
|
|
367
|
-
bitDepth: this.exportBitDepth,
|
|
368
|
-
});
|
|
369
|
-
const blob = new Blob([wavHeaderBuffer], { type: 'audio/wav' });
|
|
370
|
-
const url = URL.createObjectURL(blob);
|
|
371
|
-
const response = await fetch(url);
|
|
372
|
-
const arrayBuffer = await response.arrayBuffer();
|
|
373
|
-
// Decode the audio data
|
|
374
|
-
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
375
|
-
// Create a buffer source node and play the audio
|
|
376
|
-
const bufferSource = this.audioContext.createBufferSource();
|
|
377
|
-
bufferSource.buffer = audioBuffer;
|
|
378
|
-
bufferSource.connect(this.audioContext.destination);
|
|
379
|
-
bufferSource.start();
|
|
380
|
-
this.logger?.debug('Playing recorded data', recordedData);
|
|
381
|
-
// Clean up
|
|
382
|
-
URL.revokeObjectURL(url);
|
|
383
|
-
}
|
|
384
|
-
catch (error) {
|
|
385
|
-
console.error(`[${TAG}] Failed to play recorded data:`, error);
|
|
386
|
-
}
|
|
387
|
-
}
|
|
357
|
+
/**
|
|
358
|
+
* Determines the audio format capabilities of the current audio context
|
|
359
|
+
* @param sampleRate - The sample rate to check
|
|
360
|
+
* @returns Object containing format information (sample rate, bit depth, channels)
|
|
361
|
+
*/
|
|
388
362
|
checkAudioContextFormat({ sampleRate }) {
|
|
389
363
|
// Create a silent AudioBuffer
|
|
390
364
|
const frameCount = sampleRate * 1.0; // 1 second buffer
|
|
@@ -398,12 +372,20 @@ export class WebRecorder {
|
|
|
398
372
|
numberOfChannels: audioBuffer.numberOfChannels,
|
|
399
373
|
};
|
|
400
374
|
}
|
|
375
|
+
/**
|
|
376
|
+
* Resumes a paused recording
|
|
377
|
+
* Reconnects audio nodes and resumes the media recorder
|
|
378
|
+
*/
|
|
401
379
|
resume() {
|
|
402
380
|
this.source.connect(this.audioWorkletNode);
|
|
403
381
|
this.audioWorkletNode.connect(this.audioContext.destination);
|
|
404
382
|
this.audioWorkletNode.port.postMessage({ command: 'resume' });
|
|
405
383
|
this.compressedMediaRecorder?.resume();
|
|
406
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* Initializes the compressed media recorder if compression is enabled
|
|
387
|
+
* Sets up event handlers for compressed audio data
|
|
388
|
+
*/
|
|
407
389
|
initializeCompressedRecorder() {
|
|
408
390
|
try {
|
|
409
391
|
const mimeType = 'audio/webm;codecs=opus';
|
|
@@ -427,5 +409,28 @@ export class WebRecorder {
|
|
|
427
409
|
this.logger?.error('Failed to initialize compressed recorder:', error);
|
|
428
410
|
}
|
|
429
411
|
}
|
|
412
|
+
/**
|
|
413
|
+
* Processes features if enabled
|
|
414
|
+
*/
|
|
415
|
+
processFeatures(chunk, sampleRate, chunkPosition, startPosition, endPosition, samples) {
|
|
416
|
+
if (this.config.enableProcessing && this.featureExtractorWorker) {
|
|
417
|
+
this.featureExtractorWorker.postMessage({
|
|
418
|
+
command: 'process',
|
|
419
|
+
channelData: chunk,
|
|
420
|
+
sampleRate,
|
|
421
|
+
segmentDurationMs: this.config.segmentDurationMs ??
|
|
422
|
+
DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms
|
|
423
|
+
bitDepth: this.bitDepth,
|
|
424
|
+
fullAudioDurationMs: chunkPosition * 1000,
|
|
425
|
+
numberOfChannels: this.numberOfChannels,
|
|
426
|
+
features: this.config.features,
|
|
427
|
+
intervalAnalysis: this.config.intervalAnalysis,
|
|
428
|
+
startPosition,
|
|
429
|
+
endPosition,
|
|
430
|
+
samples,
|
|
431
|
+
startCounterFrom: this.dataPointIdCounter, // Pass the current counter value
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
}
|
|
430
435
|
}
|
|
431
436
|
//# sourceMappingURL=WebRecorder.web.js.map
|