@siteed/expo-audio-stream 1.0.3 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -175
- package/android/src/main/java/net/siteed/audiostream/AudioProcessor.kt +47 -7
- package/android/src/main/java/net/siteed/audiostream/AudioRecorderManager.kt +1 -0
- package/android/src/main/java/net/siteed/audiostream/Constants.kt +5 -0
- package/android/src/main/java/net/siteed/audiostream/ExpoAudioStreamModule.kt +12 -3
- package/build/index.js +8 -7
- package/ios/AudioProcessor.swift +7 -5
- package/ios/AudioStreamManager.swift +1 -0
- package/ios/ExpoAudioStream.podspec +1 -1
- package/ios/ExpoAudioStreamModule.swift +36 -0
- package/ios/RecordingResult.swift +1 -0
- package/package.json +95 -65
- package/src/AudioAnalysis/AudioAnalysis.types.ts +59 -60
- package/src/AudioAnalysis/extractAudioAnalysis.ts +132 -121
- package/src/AudioAnalysis/extractWaveform.ts +18 -18
- package/src/AudioRecorder.provider.tsx +53 -53
- package/src/ExpoAudioStream.native.ts +2 -2
- package/src/ExpoAudioStream.types.ts +59 -53
- package/src/ExpoAudioStream.web.ts +231 -205
- package/src/ExpoAudioStreamModule.ts +22 -15
- package/src/WebRecorder.web.ts +407 -390
- package/src/constants.ts +11 -11
- package/src/events.ts +27 -13
- package/src/index.ts +17 -15
- package/src/logger.ts +15 -19
- package/src/useAudioRecorder.tsx +394 -389
- package/src/utils/BlobFix.ts +550 -0
- package/src/utils/concatenateBuffers.ts +24 -0
- package/src/utils/convertPCMToFloat32.ts +72 -45
- package/src/utils/encodingToBitDepth.ts +14 -14
- package/src/utils/getWavFileInfo.ts +106 -99
- package/src/utils/writeWavHeader.ts +50 -45
- package/src/workers/InlineFeaturesExtractor.web.tsx +296 -286
- package/src/workers/inlineAudioWebWorker.web.tsx +230 -222
- package/.eslintrc.js +0 -2
- package/.size-limit.json +0 -6
- package/android/.gradle/8.1.1/checksums/checksums.lock +0 -0
- package/android/.gradle/8.1.1/dependencies-accessors/dependencies-accessors.lock +0 -0
- package/android/.gradle/8.1.1/dependencies-accessors/gc.properties +0 -0
- package/android/.gradle/8.1.1/fileChanges/last-build.bin +0 -0
- package/android/.gradle/8.1.1/fileHashes/fileHashes.lock +0 -0
- package/android/.gradle/8.1.1/gc.properties +0 -0
- package/android/.gradle/buildOutputCleanup/buildOutputCleanup.lock +0 -0
- package/android/.gradle/buildOutputCleanup/cache.properties +0 -2
- package/android/.gradle/vcs-1/gc.properties +0 -0
- package/app.plugin.js +0 -1
- package/build/AudioAnalysis/AudioAnalysis.types.d.ts +0 -76
- package/build/AudioAnalysis/AudioAnalysis.types.d.ts.map +0 -1
- package/build/AudioAnalysis/AudioAnalysis.types.js +0 -3
- package/build/AudioAnalysis/AudioAnalysis.types.js.map +0 -1
- package/build/AudioAnalysis/extractAudioAnalysis.d.ts +0 -4
- package/build/AudioAnalysis/extractAudioAnalysis.d.ts.map +0 -1
- package/build/AudioAnalysis/extractAudioAnalysis.js +0 -101
- package/build/AudioAnalysis/extractAudioAnalysis.js.map +0 -1
- package/build/AudioAnalysis/extractWaveform.d.ts +0 -8
- package/build/AudioAnalysis/extractWaveform.d.ts.map +0 -1
- package/build/AudioAnalysis/extractWaveform.js +0 -14
- package/build/AudioAnalysis/extractWaveform.js.map +0 -1
- package/build/AudioRecorder.provider.d.ts +0 -23
- package/build/AudioRecorder.provider.d.ts.map +0 -1
- package/build/AudioRecorder.provider.js +0 -36
- package/build/AudioRecorder.provider.js.map +0 -1
- package/build/ExpoAudioStream.native.d.ts +0 -3
- package/build/ExpoAudioStream.native.d.ts.map +0 -1
- package/build/ExpoAudioStream.native.js +0 -6
- package/build/ExpoAudioStream.native.js.map +0 -1
- package/build/ExpoAudioStream.types.d.ts +0 -60
- package/build/ExpoAudioStream.types.d.ts.map +0 -1
- package/build/ExpoAudioStream.types.js +0 -2
- package/build/ExpoAudioStream.types.js.map +0 -1
- package/build/ExpoAudioStream.web.d.ts +0 -42
- package/build/ExpoAudioStream.web.d.ts.map +0 -1
- package/build/ExpoAudioStream.web.js +0 -185
- package/build/ExpoAudioStream.web.js.map +0 -1
- package/build/ExpoAudioStreamModule.d.ts +0 -3
- package/build/ExpoAudioStreamModule.d.ts.map +0 -1
- package/build/ExpoAudioStreamModule.js +0 -18
- package/build/ExpoAudioStreamModule.js.map +0 -1
- package/build/WebRecorder.web.d.ts +0 -51
- package/build/WebRecorder.web.d.ts.map +0 -1
- package/build/WebRecorder.web.js +0 -288
- package/build/WebRecorder.web.js.map +0 -1
- package/build/constants.d.ts +0 -11
- package/build/constants.d.ts.map +0 -1
- package/build/constants.js +0 -14
- package/build/constants.js.map +0 -1
- package/build/events.d.ts +0 -6
- package/build/events.d.ts.map +0 -1
- package/build/events.js +0 -15
- package/build/events.js.map +0 -1
- package/build/index.d.ts +0 -10
- package/build/index.d.ts.map +0 -1
- package/build/index.js.map +0 -1
- package/build/logger.d.ts +0 -9
- package/build/logger.d.ts.map +0 -1
- package/build/logger.js +0 -17
- package/build/logger.js.map +0 -1
- package/build/useAudioRecorder.d.ts +0 -37
- package/build/useAudioRecorder.d.ts.map +0 -1
- package/build/useAudioRecorder.js +0 -271
- package/build/useAudioRecorder.js.map +0 -1
- package/build/utils/convertPCMToFloat32.d.ts +0 -11
- package/build/utils/convertPCMToFloat32.d.ts.map +0 -1
- package/build/utils/convertPCMToFloat32.js +0 -41
- package/build/utils/convertPCMToFloat32.js.map +0 -1
- package/build/utils/encodingToBitDepth.d.ts +0 -5
- package/build/utils/encodingToBitDepth.d.ts.map +0 -1
- package/build/utils/encodingToBitDepth.js +0 -13
- package/build/utils/encodingToBitDepth.js.map +0 -1
- package/build/utils/getWavFileInfo.d.ts +0 -25
- package/build/utils/getWavFileInfo.d.ts.map +0 -1
- package/build/utils/getWavFileInfo.js +0 -89
- package/build/utils/getWavFileInfo.js.map +0 -1
- package/build/utils/writeWavHeader.d.ts +0 -9
- package/build/utils/writeWavHeader.d.ts.map +0 -1
- package/build/utils/writeWavHeader.js +0 -41
- package/build/utils/writeWavHeader.js.map +0 -1
- package/build/workers/InlineFeaturesExtractor.web.d.ts +0 -2
- package/build/workers/InlineFeaturesExtractor.web.d.ts.map +0 -1
- package/build/workers/InlineFeaturesExtractor.web.js +0 -303
- package/build/workers/InlineFeaturesExtractor.web.js.map +0 -1
- package/build/workers/inlineAudioWebWorker.web.d.ts +0 -2
- package/build/workers/inlineAudioWebWorker.web.d.ts.map +0 -1
- package/build/workers/inlineAudioWebWorker.web.js +0 -243
- package/build/workers/inlineAudioWebWorker.web.js.map +0 -1
- package/expo-module.config.json +0 -18
- package/plugin/build/index.d.ts +0 -5
- package/plugin/build/index.js +0 -28
- package/plugin/src/index.ts +0 -53
- package/plugin/tsconfig.json +0 -14
- package/publish.sh +0 -8
- package/tsconfig.json +0 -9
package/src/WebRecorder.web.ts
CHANGED
|
@@ -1,416 +1,433 @@
|
|
|
1
1
|
// src/WebRecorder.ts
|
|
2
|
-
import {
|
|
3
|
-
import { RecordingConfig } from
|
|
2
|
+
import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
|
|
3
|
+
import { RecordingConfig } from './ExpoAudioStream.types'
|
|
4
4
|
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} from
|
|
8
|
-
import { getLogger } from
|
|
9
|
-
import { encodingToBitDepth } from
|
|
10
|
-
import { InlineFeaturesExtractor } from
|
|
11
|
-
import { InlineAudioWebWorker } from
|
|
5
|
+
EmitAudioAnalysisFunction,
|
|
6
|
+
EmitAudioEventFunction,
|
|
7
|
+
} from './ExpoAudioStream.web'
|
|
8
|
+
import { getLogger } from './logger'
|
|
9
|
+
import { encodingToBitDepth } from './utils/encodingToBitDepth'
|
|
10
|
+
import { InlineFeaturesExtractor } from './workers/InlineFeaturesExtractor.web'
|
|
11
|
+
import { InlineAudioWebWorker } from './workers/inlineAudioWebWorker.web'
|
|
12
12
|
|
|
13
13
|
interface AudioWorkletEvent {
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
data: {
|
|
15
|
+
command: string
|
|
16
|
+
recordedData?: Float32Array
|
|
17
|
+
sampleRate?: number
|
|
18
|
+
}
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
interface AudioFeaturesEvent {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
22
|
+
data: {
|
|
23
|
+
command: string
|
|
24
|
+
result: AudioAnalysis
|
|
25
|
+
}
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
const DEFAULT_WEB_BITDEPTH = 32
|
|
29
|
-
const DEFAULT_WEB_POINTS_PER_SECOND = 10
|
|
30
|
-
const DEFAULT_WEB_INTERVAL = 500
|
|
31
|
-
const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1
|
|
28
|
+
const DEFAULT_WEB_BITDEPTH = 32
|
|
29
|
+
const DEFAULT_WEB_POINTS_PER_SECOND = 10
|
|
30
|
+
const DEFAULT_WEB_INTERVAL = 500
|
|
31
|
+
const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1
|
|
32
|
+
const DEFAULT_ALGORITHM = 'rms'
|
|
32
33
|
|
|
33
|
-
const TAG =
|
|
34
|
-
const logger = getLogger(TAG)
|
|
34
|
+
const TAG = 'WebRecorder'
|
|
35
|
+
const logger = getLogger(TAG)
|
|
35
36
|
|
|
36
37
|
export class WebRecorder {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
38
|
+
private audioContext: AudioContext
|
|
39
|
+
private audioWorkletNode!: AudioWorkletNode
|
|
40
|
+
private featureExtractorWorker?: Worker
|
|
41
|
+
private source: MediaStreamAudioSourceNode
|
|
42
|
+
private audioWorkletUrl: string
|
|
43
|
+
private emitAudioEventCallback: EmitAudioEventFunction
|
|
44
|
+
private emitAudioAnalysisCallback: EmitAudioAnalysisFunction
|
|
45
|
+
private config: RecordingConfig
|
|
46
|
+
private position: number // Track the cumulative position
|
|
47
|
+
private numberOfChannels: number // Number of audio channels
|
|
48
|
+
private bitDepth: number // Bit depth of the audio
|
|
49
|
+
private exportBitDepth: number // Bit depth of the audio
|
|
50
|
+
private buffer: Float32Array // Single buffer to store the audio data
|
|
51
|
+
private bufferSize: number // Keep track of the buffer size
|
|
52
|
+
private audioAnalysisData: AudioAnalysis // Keep updating the full audio analysis data with latest events
|
|
53
|
+
|
|
54
|
+
constructor({
|
|
55
|
+
audioContext,
|
|
56
|
+
source,
|
|
57
|
+
recordingConfig,
|
|
58
|
+
audioWorkletUrl,
|
|
59
|
+
emitAudioEventCallback,
|
|
60
|
+
emitAudioAnalysisCallback,
|
|
61
|
+
}: {
|
|
62
|
+
audioContext: AudioContext
|
|
63
|
+
source: MediaStreamAudioSourceNode
|
|
64
|
+
recordingConfig: RecordingConfig
|
|
65
|
+
audioWorkletUrl: string
|
|
66
|
+
emitAudioEventCallback: EmitAudioEventFunction
|
|
67
|
+
emitAudioAnalysisCallback: EmitAudioAnalysisFunction
|
|
68
|
+
}) {
|
|
69
|
+
this.audioContext = audioContext
|
|
70
|
+
this.source = source
|
|
71
|
+
this.audioWorkletUrl = audioWorkletUrl
|
|
72
|
+
this.emitAudioEventCallback = emitAudioEventCallback
|
|
73
|
+
this.emitAudioAnalysisCallback = emitAudioAnalysisCallback
|
|
74
|
+
this.config = recordingConfig
|
|
75
|
+
this.position = 0
|
|
76
|
+
this.bufferSize = 0
|
|
77
|
+
this.buffer = new Float32Array(0) // Initialize the buffer
|
|
78
|
+
|
|
79
|
+
const audioContextFormat = this.checkAudioContextFormat({
|
|
80
|
+
sampleRate: this.audioContext.sampleRate,
|
|
81
|
+
})
|
|
82
|
+
logger.debug('Initialized WebRecorder with config:', {
|
|
83
|
+
sampleRate: audioContextFormat.sampleRate,
|
|
84
|
+
bitDepth: audioContextFormat.bitDepth,
|
|
85
|
+
numberOfChannels: audioContextFormat.numberOfChannels,
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
this.bitDepth = audioContextFormat.bitDepth
|
|
89
|
+
this.numberOfChannels =
|
|
90
|
+
audioContextFormat.numberOfChannels ||
|
|
91
|
+
DEFAULT_WEB_NUMBER_OF_CHANNELS // Default to 1 if not available
|
|
92
|
+
this.exportBitDepth =
|
|
93
|
+
encodingToBitDepth({
|
|
94
|
+
encoding: recordingConfig.encoding ?? 'pcm_32bit',
|
|
95
|
+
}) ||
|
|
96
|
+
audioContextFormat.bitDepth ||
|
|
97
|
+
DEFAULT_WEB_BITDEPTH
|
|
98
|
+
|
|
99
|
+
this.audioAnalysisData = {
|
|
100
|
+
amplitudeRange: { min: 0, max: 0 },
|
|
101
|
+
dataPoints: [],
|
|
102
|
+
durationMs: 0,
|
|
103
|
+
samples: 0,
|
|
104
|
+
amplitudeAlgorithm: recordingConfig.algorithm || DEFAULT_ALGORITHM,
|
|
105
|
+
bitDepth: this.bitDepth,
|
|
106
|
+
numberOfChannels: this.numberOfChannels,
|
|
107
|
+
sampleRate: this.config.sampleRate || this.audioContext.sampleRate,
|
|
108
|
+
pointsPerSecond:
|
|
109
|
+
this.config.pointsPerSecond || DEFAULT_WEB_POINTS_PER_SECOND,
|
|
110
|
+
speakerChanges: [],
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (recordingConfig.enableProcessing) {
|
|
114
|
+
this.initFeatureExtractorWorker()
|
|
115
|
+
}
|
|
112
116
|
}
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
117
|
+
|
|
118
|
+
async init() {
|
|
119
|
+
try {
|
|
120
|
+
if (!this.audioWorkletUrl) {
|
|
121
|
+
const blob = new Blob([InlineAudioWebWorker], {
|
|
122
|
+
type: 'application/javascript',
|
|
123
|
+
})
|
|
124
|
+
const url = URL.createObjectURL(blob)
|
|
125
|
+
await this.audioContext.audioWorklet.addModule(url)
|
|
126
|
+
} else {
|
|
127
|
+
await this.audioContext.audioWorklet.addModule(
|
|
128
|
+
this.audioWorkletUrl
|
|
129
|
+
)
|
|
130
|
+
}
|
|
131
|
+
this.audioWorkletNode = new AudioWorkletNode(
|
|
132
|
+
this.audioContext,
|
|
133
|
+
'recorder-processor'
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
this.audioWorkletNode.port.onmessage = async (
|
|
137
|
+
event: AudioWorkletEvent
|
|
138
|
+
) => {
|
|
139
|
+
const command = event.data.command
|
|
140
|
+
if (command !== 'newData') {
|
|
141
|
+
return
|
|
142
|
+
}
|
|
143
|
+
// Handle the audio blob (e.g., send it to the server or process it further)
|
|
144
|
+
logger.debug('Received audio blob from processor', event)
|
|
145
|
+
const pcmBufferFloat = event.data.recordedData
|
|
146
|
+
|
|
147
|
+
if (!pcmBufferFloat) {
|
|
148
|
+
return
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// Concatenate the incoming Float32Array to the existing buffer
|
|
152
|
+
const newBuffer = new Float32Array(
|
|
153
|
+
this.bufferSize + pcmBufferFloat.length
|
|
154
|
+
)
|
|
155
|
+
newBuffer.set(this.buffer, 0)
|
|
156
|
+
newBuffer.set(pcmBufferFloat, this.bufferSize)
|
|
157
|
+
this.buffer = newBuffer
|
|
158
|
+
this.bufferSize += pcmBufferFloat.length
|
|
159
|
+
|
|
160
|
+
const sampleRate =
|
|
161
|
+
event.data.sampleRate ?? this.audioContext.sampleRate
|
|
162
|
+
const duration = pcmBufferFloat.length / sampleRate // Calculate duration of the current buffer
|
|
163
|
+
|
|
164
|
+
this.emitAudioEventCallback({
|
|
165
|
+
data: pcmBufferFloat,
|
|
166
|
+
position: this.position,
|
|
167
|
+
})
|
|
168
|
+
this.position += duration // Update position
|
|
169
|
+
|
|
170
|
+
this.featureExtractorWorker?.postMessage(
|
|
171
|
+
{
|
|
172
|
+
command: 'process',
|
|
173
|
+
channelData: pcmBufferFloat,
|
|
174
|
+
sampleRate,
|
|
175
|
+
pointsPerSecond:
|
|
176
|
+
this.config.pointsPerSecond ||
|
|
177
|
+
DEFAULT_WEB_POINTS_PER_SECOND,
|
|
178
|
+
algorithm: this.config.algorithm || 'rms',
|
|
179
|
+
bitDepth: this.bitDepth,
|
|
180
|
+
fullAudioDurationMs: this.position * 1000,
|
|
181
|
+
numberOfChannels: this.numberOfChannels,
|
|
182
|
+
features: this.config.features,
|
|
183
|
+
},
|
|
184
|
+
[]
|
|
185
|
+
)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
logger.debug(
|
|
189
|
+
`WebRecorder initialized -- recordSampleRate=${this.audioContext.sampleRate}`,
|
|
190
|
+
this.config
|
|
191
|
+
)
|
|
192
|
+
this.audioWorkletNode.port.postMessage({
|
|
193
|
+
command: 'init',
|
|
194
|
+
recordSampleRate: this.audioContext.sampleRate, // Pass the original sample rate
|
|
195
|
+
exportSampleRate:
|
|
196
|
+
this.config.sampleRate ?? this.audioContext.sampleRate,
|
|
197
|
+
bitDepth: this.bitDepth,
|
|
198
|
+
exportBitDepth: this.exportBitDepth,
|
|
199
|
+
channels: this.numberOfChannels,
|
|
200
|
+
interval: this.config.interval ?? DEFAULT_WEB_INTERVAL,
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
// Connect the source to the AudioWorkletNode and start recording
|
|
204
|
+
this.source.connect(this.audioWorkletNode)
|
|
205
|
+
this.audioWorkletNode.connect(this.audioContext.destination)
|
|
206
|
+
} catch (error) {
|
|
207
|
+
console.error(`[${TAG}] Failed to initialize WebRecorder`, error)
|
|
137
208
|
}
|
|
138
|
-
|
|
139
|
-
logger.debug("Received audio blob from processor", event);
|
|
140
|
-
const pcmBuffer = event.data.recordedData;
|
|
209
|
+
}
|
|
141
210
|
|
|
142
|
-
|
|
143
|
-
|
|
211
|
+
initFeatureExtractorWorker(featuresExtratorUrl?: string) {
|
|
212
|
+
try {
|
|
213
|
+
if (featuresExtratorUrl) {
|
|
214
|
+
// Initialize the feature extractor worker
|
|
215
|
+
//TODO: create audio feature extractor from a Blob instead of url since we cannot include the url directly in the library
|
|
216
|
+
// We keep the url during dev and use the blob in production.
|
|
217
|
+
this.featureExtractorWorker = new Worker(
|
|
218
|
+
new URL(featuresExtratorUrl, window.location.href)
|
|
219
|
+
)
|
|
220
|
+
this.featureExtractorWorker.onmessage =
|
|
221
|
+
this.handleFeatureExtractorMessage.bind(this)
|
|
222
|
+
this.featureExtractorWorker.onerror =
|
|
223
|
+
this.handleWorkerError.bind(this)
|
|
224
|
+
} else {
|
|
225
|
+
// Fallback to the inline worker if the URL is not provided
|
|
226
|
+
this.initFallbackWorker()
|
|
227
|
+
}
|
|
228
|
+
} catch (error) {
|
|
229
|
+
console.error(
|
|
230
|
+
`[${TAG}] Failed to initialize feature extractor worker`,
|
|
231
|
+
error
|
|
232
|
+
)
|
|
233
|
+
this.initFallbackWorker()
|
|
144
234
|
}
|
|
235
|
+
}
|
|
145
236
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
position: this.position,
|
|
166
|
-
});
|
|
167
|
-
this.position += duration; // Update position
|
|
168
|
-
|
|
169
|
-
this.featureExtractorWorker?.postMessage(
|
|
170
|
-
{
|
|
171
|
-
command: "process",
|
|
172
|
-
channelData,
|
|
173
|
-
sampleRate: this.audioContext.sampleRate,
|
|
174
|
-
pointsPerSecond:
|
|
175
|
-
this.config.pointsPerSecond || DEFAULT_WEB_POINTS_PER_SECOND,
|
|
176
|
-
algorithm: this.config.algorithm || "rms",
|
|
177
|
-
bitDepth: this.bitDepth,
|
|
178
|
-
fullAudioDurationMs: this.position * 1000,
|
|
179
|
-
numberOfChannels: this.numberOfChannels,
|
|
180
|
-
features: this.config.features,
|
|
181
|
-
},
|
|
182
|
-
[],
|
|
183
|
-
);
|
|
184
|
-
};
|
|
185
|
-
|
|
186
|
-
logger.debug(
|
|
187
|
-
`WebRecorder initialized -- recordSampleRate=${this.audioContext.sampleRate}`,
|
|
188
|
-
this.config,
|
|
189
|
-
);
|
|
190
|
-
this.audioWorkletNode.port.postMessage({
|
|
191
|
-
command: "init",
|
|
192
|
-
recordSampleRate: this.audioContext.sampleRate, // Pass the original sample rate
|
|
193
|
-
exportSampleRate:
|
|
194
|
-
this.config.sampleRate ?? this.audioContext.sampleRate,
|
|
195
|
-
bitDepth: this.bitDepth,
|
|
196
|
-
exportBitDepth: this.exportBitDepth,
|
|
197
|
-
channels: this.numberOfChannels,
|
|
198
|
-
interval: this.config.interval ?? DEFAULT_WEB_INTERVAL,
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
// Connect the source to the AudioWorkletNode and start recording
|
|
202
|
-
this.source.connect(this.audioWorkletNode);
|
|
203
|
-
this.audioWorkletNode.connect(this.audioContext.destination);
|
|
204
|
-
} catch (error) {
|
|
205
|
-
console.error(`[${TAG}] Failed to initialize WebRecorder`, error);
|
|
237
|
+
initFallbackWorker() {
|
|
238
|
+
try {
|
|
239
|
+
const blob = new Blob([InlineFeaturesExtractor], {
|
|
240
|
+
type: 'application/javascript',
|
|
241
|
+
})
|
|
242
|
+
const url = URL.createObjectURL(blob)
|
|
243
|
+
this.featureExtractorWorker = new Worker(url)
|
|
244
|
+
this.featureExtractorWorker.onmessage =
|
|
245
|
+
this.handleFeatureExtractorMessage.bind(this)
|
|
246
|
+
this.featureExtractorWorker.onerror = (error) => {
|
|
247
|
+
console.error(`[${TAG}] Default Inline worker failed`, error)
|
|
248
|
+
}
|
|
249
|
+
logger.log('Inline worker initialized successfully')
|
|
250
|
+
} catch (error) {
|
|
251
|
+
console.error(
|
|
252
|
+
`[${TAG}] Failed to initialize Inline Feature Extractor worker`,
|
|
253
|
+
error
|
|
254
|
+
)
|
|
255
|
+
}
|
|
206
256
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
try {
|
|
211
|
-
if (featuresExtratorUrl) {
|
|
212
|
-
// Initialize the feature extractor worker
|
|
213
|
-
//TODO: create audio feature extractor from a Blob instead of url since we cannot include the url directly in the library
|
|
214
|
-
// We keep the url during dev and use the blob in production.
|
|
215
|
-
this.featureExtractorWorker = new Worker(
|
|
216
|
-
new URL(featuresExtratorUrl, window.location.href),
|
|
217
|
-
);
|
|
218
|
-
this.featureExtractorWorker.onmessage =
|
|
219
|
-
this.handleFeatureExtractorMessage.bind(this);
|
|
220
|
-
this.featureExtractorWorker.onerror = this.handleWorkerError.bind(this);
|
|
221
|
-
} else {
|
|
222
|
-
// Fallback to the inline worker if the URL is not provided
|
|
223
|
-
this.initFallbackWorker();
|
|
224
|
-
}
|
|
225
|
-
} catch (error) {
|
|
226
|
-
console.error(
|
|
227
|
-
`[${TAG}] Failed to initialize feature extractor worker`,
|
|
228
|
-
error,
|
|
229
|
-
);
|
|
230
|
-
this.initFallbackWorker();
|
|
257
|
+
|
|
258
|
+
handleWorkerError(error: ErrorEvent) {
|
|
259
|
+
console.error(`[${TAG}] Feature extractor worker error:`, error)
|
|
231
260
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
261
|
+
|
|
262
|
+
handleFeatureExtractorMessage(event: AudioFeaturesEvent) {
|
|
263
|
+
if (event.data.command === 'features') {
|
|
264
|
+
const segmentResult = event.data.result
|
|
265
|
+
|
|
266
|
+
// Merge the segment result with the full audio analysis data
|
|
267
|
+
this.audioAnalysisData.dataPoints.push(...segmentResult.dataPoints)
|
|
268
|
+
this.audioAnalysisData.speakerChanges?.push(
|
|
269
|
+
...(segmentResult.speakerChanges ?? [])
|
|
270
|
+
)
|
|
271
|
+
this.audioAnalysisData.durationMs = segmentResult.durationMs
|
|
272
|
+
if (segmentResult.amplitudeRange) {
|
|
273
|
+
this.audioAnalysisData.amplitudeRange = {
|
|
274
|
+
min: Math.min(
|
|
275
|
+
this.audioAnalysisData.amplitudeRange.min,
|
|
276
|
+
segmentResult.amplitudeRange.min
|
|
277
|
+
),
|
|
278
|
+
max: Math.max(
|
|
279
|
+
this.audioAnalysisData.amplitudeRange.max,
|
|
280
|
+
segmentResult.amplitudeRange.max
|
|
281
|
+
),
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
// Handle the extracted features (e.g., emit an event or log them)
|
|
285
|
+
logger.debug('features event segmentResult', segmentResult)
|
|
286
|
+
logger.debug(
|
|
287
|
+
'features event audioAnalysisData',
|
|
288
|
+
this.audioAnalysisData
|
|
289
|
+
)
|
|
290
|
+
this.emitAudioAnalysisCallback(segmentResult)
|
|
291
|
+
}
|
|
252
292
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
handleFeatureExtractorMessage(event: AudioFeaturesEvent) {
|
|
260
|
-
if (event.data.command === "features") {
|
|
261
|
-
const segmentResult = event.data.result;
|
|
262
|
-
|
|
263
|
-
// Merge the segment result with the full audio analysis data
|
|
264
|
-
this.audioAnalysisData.dataPoints.push(...segmentResult.dataPoints);
|
|
265
|
-
this.audioAnalysisData.speakerChanges?.push(
|
|
266
|
-
...(segmentResult.speakerChanges ?? []),
|
|
267
|
-
);
|
|
268
|
-
this.audioAnalysisData.durationMs = segmentResult.durationMs;
|
|
269
|
-
if (segmentResult.amplitudeRange) {
|
|
270
|
-
this.audioAnalysisData.amplitudeRange = {
|
|
271
|
-
min: Math.min(
|
|
272
|
-
this.audioAnalysisData.amplitudeRange.min,
|
|
273
|
-
segmentResult.amplitudeRange.min,
|
|
274
|
-
),
|
|
275
|
-
max: Math.max(
|
|
276
|
-
this.audioAnalysisData.amplitudeRange.max,
|
|
277
|
-
segmentResult.amplitudeRange.max,
|
|
278
|
-
),
|
|
279
|
-
};
|
|
280
|
-
}
|
|
281
|
-
// Handle the extracted features (e.g., emit an event or log them)
|
|
282
|
-
logger.debug("features event segmentResult", segmentResult);
|
|
283
|
-
logger.debug("features event audioAnalysisData", this.audioAnalysisData);
|
|
284
|
-
this.emitAudioAnalysisCallback(segmentResult);
|
|
293
|
+
|
|
294
|
+
start() {
|
|
295
|
+
this.source.connect(this.audioWorkletNode)
|
|
296
|
+
this.audioWorkletNode.connect(this.audioContext.destination)
|
|
285
297
|
}
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
298
|
+
|
|
299
|
+
stop(): Promise<Float32Array> {
|
|
300
|
+
return new Promise((resolve, reject) => {
|
|
301
|
+
try {
|
|
302
|
+
if (this.audioWorkletNode) {
|
|
303
|
+
// this.source.disconnect(this.audioWorkletNode);
|
|
304
|
+
// this.audioWorkletNode.disconnect(this.audioContext.destination);
|
|
305
|
+
this.audioWorkletNode.port.postMessage({ command: 'stop' })
|
|
306
|
+
|
|
307
|
+
// Set a timeout to reject the promise if no message is received within 5 seconds
|
|
308
|
+
const timeout = setTimeout(() => {
|
|
309
|
+
this.audioWorkletNode.port.removeEventListener(
|
|
310
|
+
'message',
|
|
311
|
+
onMessage
|
|
312
|
+
)
|
|
313
|
+
reject(
|
|
314
|
+
new Error(
|
|
315
|
+
"Timeout error, audioWorkletNode didn't complete."
|
|
316
|
+
)
|
|
317
|
+
)
|
|
318
|
+
}, 5000)
|
|
319
|
+
|
|
320
|
+
// Listen for the recordedData message to confirm stopping
|
|
321
|
+
const onMessage = async (event: AudioWorkletEvent) => {
|
|
322
|
+
const command = event.data.command
|
|
323
|
+
if (command === 'recordedData') {
|
|
324
|
+
clearTimeout(timeout) // Clear the timeout
|
|
325
|
+
|
|
326
|
+
const rawPCMDataFull =
|
|
327
|
+
event.data.recordedData?.slice(0)
|
|
328
|
+
|
|
329
|
+
if (!rawPCMDataFull) {
|
|
330
|
+
reject(new Error('Failed to get recorded data'))
|
|
331
|
+
return
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
// Compute duration of the recorded data
|
|
335
|
+
const duration =
|
|
336
|
+
rawPCMDataFull.byteLength /
|
|
337
|
+
(this.audioContext.sampleRate *
|
|
338
|
+
(this.exportBitDepth /
|
|
339
|
+
this.numberOfChannels))
|
|
340
|
+
logger.debug(
|
|
341
|
+
`Received recorded data -- Duration: ${duration} vs ${rawPCMDataFull.byteLength / this.audioContext.sampleRate} seconds`
|
|
342
|
+
)
|
|
343
|
+
logger.debug(
|
|
344
|
+
`recordedData.length=${rawPCMDataFull.byteLength} vs transmittedData.length=${this.bufferSize}`
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
// Remove the event listener after receiving the final data
|
|
348
|
+
this.audioWorkletNode.port.removeEventListener(
|
|
349
|
+
'message',
|
|
350
|
+
onMessage
|
|
351
|
+
)
|
|
352
|
+
resolve(this.buffer) // Resolve the promise with the collected buffers
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
this.audioWorkletNode.port.addEventListener(
|
|
356
|
+
'message',
|
|
357
|
+
onMessage
|
|
358
|
+
)
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
// Stop all media stream tracks to stop the browser recording
|
|
362
|
+
this.stopMediaStreamTracks()
|
|
363
|
+
} catch (error) {
|
|
364
|
+
reject(error)
|
|
340
365
|
}
|
|
341
|
-
|
|
342
|
-
|
|
366
|
+
})
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
pause() {
|
|
370
|
+
this.source.disconnect(this.audioWorkletNode) // Disconnect the source from the AudioWorkletNode
|
|
371
|
+
this.audioWorkletNode.disconnect(this.audioContext.destination) // Disconnect the AudioWorkletNode from the destination
|
|
372
|
+
this.audioWorkletNode.port.postMessage({ command: 'pause' })
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
stopMediaStreamTracks() {
|
|
376
|
+
// Stop all audio tracks to stop the recording icon
|
|
377
|
+
const tracks = this.source.mediaStream.getTracks()
|
|
378
|
+
tracks.forEach((track) => track.stop())
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async playRecordedData({
|
|
382
|
+
recordedData,
|
|
383
|
+
}: {
|
|
384
|
+
recordedData: ArrayBuffer
|
|
385
|
+
mimeType?: string
|
|
386
|
+
}) {
|
|
387
|
+
try {
|
|
388
|
+
const blob = new Blob([recordedData])
|
|
389
|
+
const url = URL.createObjectURL(blob)
|
|
390
|
+
const response = await fetch(url)
|
|
391
|
+
const arrayBuffer = await response.arrayBuffer()
|
|
392
|
+
|
|
393
|
+
// Decode the audio data
|
|
394
|
+
const audioBuffer =
|
|
395
|
+
await this.audioContext.decodeAudioData(arrayBuffer)
|
|
396
|
+
|
|
397
|
+
// Create a buffer source node and play the audio
|
|
398
|
+
const bufferSource = this.audioContext.createBufferSource()
|
|
399
|
+
bufferSource.buffer = audioBuffer
|
|
400
|
+
bufferSource.connect(this.audioContext.destination)
|
|
401
|
+
bufferSource.start()
|
|
402
|
+
logger.debug('Playing recorded data', recordedData)
|
|
403
|
+
} catch (error) {
|
|
404
|
+
console.error(`[${TAG}] Failed to play recorded data:`, error)
|
|
343
405
|
}
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
private checkAudioContextFormat({ sampleRate }: { sampleRate: number }) {
|
|
409
|
+
// Create a silent AudioBuffer
|
|
410
|
+
const frameCount = sampleRate * 1.0 // 1 second buffer
|
|
411
|
+
const audioBuffer = this.audioContext.createBuffer(
|
|
412
|
+
1,
|
|
413
|
+
frameCount,
|
|
414
|
+
sampleRate
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
// Check the format
|
|
418
|
+
const channelData = audioBuffer.getChannelData(0)
|
|
419
|
+
const bitDepth = channelData.BYTES_PER_ELEMENT * 8 // 4 bytes per element means 32-bit
|
|
420
|
+
|
|
421
|
+
return {
|
|
422
|
+
sampleRate: audioBuffer.sampleRate,
|
|
423
|
+
bitDepth,
|
|
424
|
+
numberOfChannels: audioBuffer.numberOfChannels,
|
|
425
|
+
}
|
|
426
|
+
}
|
|
344
427
|
|
|
345
|
-
|
|
346
|
-
this.
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
}
|
|
350
|
-
});
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
pause() {
|
|
354
|
-
this.source.disconnect(this.audioWorkletNode); // Disconnect the source from the AudioWorkletNode
|
|
355
|
-
this.audioWorkletNode.disconnect(this.audioContext.destination); // Disconnect the AudioWorkletNode from the destination
|
|
356
|
-
this.audioWorkletNode.port.postMessage({ command: "pause" });
|
|
357
|
-
}
|
|
358
|
-
|
|
359
|
-
stopMediaStreamTracks() {
|
|
360
|
-
// Stop all audio tracks to stop the recording icon
|
|
361
|
-
const tracks = this.source.mediaStream.getTracks();
|
|
362
|
-
tracks.forEach((track) => track.stop());
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
async playRecordedData({
|
|
366
|
-
recordedData,
|
|
367
|
-
}: {
|
|
368
|
-
recordedData: ArrayBuffer;
|
|
369
|
-
mimeType?: string;
|
|
370
|
-
}) {
|
|
371
|
-
try {
|
|
372
|
-
const blob = new Blob([recordedData]);
|
|
373
|
-
const url = URL.createObjectURL(blob);
|
|
374
|
-
const response = await fetch(url);
|
|
375
|
-
const arrayBuffer = await response.arrayBuffer();
|
|
376
|
-
|
|
377
|
-
// Decode the audio data
|
|
378
|
-
const audioBuffer = await this.audioContext.decodeAudioData(arrayBuffer);
|
|
379
|
-
|
|
380
|
-
// Create a buffer source node and play the audio
|
|
381
|
-
const bufferSource = this.audioContext.createBufferSource();
|
|
382
|
-
bufferSource.buffer = audioBuffer;
|
|
383
|
-
bufferSource.connect(this.audioContext.destination);
|
|
384
|
-
bufferSource.start();
|
|
385
|
-
logger.debug("Playing recorded data", recordedData);
|
|
386
|
-
} catch (error) {
|
|
387
|
-
console.error(`[${TAG}] Failed to play recorded data:`, error);
|
|
428
|
+
resume() {
|
|
429
|
+
this.source.connect(this.audioWorkletNode)
|
|
430
|
+
this.audioWorkletNode.connect(this.audioContext.destination)
|
|
431
|
+
this.audioWorkletNode.port.postMessage({ command: 'resume' })
|
|
388
432
|
}
|
|
389
|
-
}
|
|
390
|
-
|
|
391
|
-
private checkAudioContextFormat({ sampleRate }: { sampleRate: number }) {
|
|
392
|
-
// Create a silent AudioBuffer
|
|
393
|
-
const frameCount = sampleRate * 1.0; // 1 second buffer
|
|
394
|
-
const audioBuffer = this.audioContext.createBuffer(
|
|
395
|
-
1,
|
|
396
|
-
frameCount,
|
|
397
|
-
sampleRate,
|
|
398
|
-
);
|
|
399
|
-
|
|
400
|
-
// Check the format
|
|
401
|
-
const channelData = audioBuffer.getChannelData(0);
|
|
402
|
-
const bitDepth = channelData.BYTES_PER_ELEMENT * 8; // 4 bytes per element means 32-bit
|
|
403
|
-
|
|
404
|
-
return {
|
|
405
|
-
sampleRate: audioBuffer.sampleRate,
|
|
406
|
-
bitDepth,
|
|
407
|
-
numberOfChannels: audioBuffer.numberOfChannels,
|
|
408
|
-
};
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
resume() {
|
|
412
|
-
this.source.connect(this.audioWorkletNode);
|
|
413
|
-
this.audioWorkletNode.connect(this.audioContext.destination);
|
|
414
|
-
this.audioWorkletNode.port.postMessage({ command: "resume" });
|
|
415
|
-
}
|
|
416
433
|
}
|