@geastack/targets 0.1.99 → 0.1.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build-config.json +10 -2
- package/package.json +7 -7
- package/targets/cmake/geatsc-generated-archive-build.mjs +32 -1
- package/targets/esp32/app_project.cmake +2 -2
- package/targets/esp32/chip_bindings/audio/echo_cancellation.cpp +392 -28
- package/targets/esp32/chip_bindings/audio/echo_cancellation.h +18 -2
- package/targets/esp32/chip_bindings/audio/es8311.cpp +117 -13
- package/targets/esp32/chip_bindings/displays/co5300.cpp +68 -28
- package/targets/esp32/components/gea_framework/CMakeLists.txt +17 -1
- package/targets/esp32/components/gea_framework/idf_component.yml +4 -0
- package/targets/esp32/components/gea_vpx/CMakeLists.txt +1 -1
- package/targets/esp32/connectivity/wifi.cpp +800 -543
- package/targets/esp32/display.cpp +121 -26
- package/targets/esp32/gea_framework.cmake +22 -28
- package/targets/esp32/host_sources.cmake +1 -0
- package/targets/esp32/services/comparison_benchmark.h +137 -0
- package/targets/esp32/services/comparison_gesture.h +133 -0
- package/targets/esp32/services/comparison_input_trace.h +64 -0
- package/targets/esp32/services/comparison_upload_capture.h +125 -0
- package/targets/esp32/services/debugger.h +634 -0
- package/targets/esp32/services/device_control.cpp +965 -20
- package/targets/esp32/services/frame_scheduler.cpp +128 -4
- package/targets/esp32/services/heap.cpp +125 -0
- package/targets/esp32/services/opus_benchmark.h +1 -1
- package/targets/esp32/services/storage_service.cpp +119 -104
- package/targets/esp32-p4-m5stack-tab5/dependencies.lock +27 -16
- package/targets/esp32-s3-m5stack-stopwatch/README.md +14 -9
- package/targets/esp32-s3-m5stack-stopwatch/dependencies.lock +54 -1
- package/targets/esp32-s3-m5stack-stopwatch/main/CMakeLists.txt +10 -7
- package/targets/esp32-s3-m5stack-stopwatch/main/buttons.cpp +80 -0
- package/targets/esp32-s3-m5stack-stopwatch/main/idf_component.yml +18 -0
- package/targets/esp32-s3-m5stack-stopwatch/main/include/board.h +38 -17
- package/targets/esp32-s3-m5stack-stopwatch/main/panel_reset.cpp +132 -129
- package/targets/esp32-s3-m5stack-stopwatch/main/power.cpp +253 -0
- package/targets/esp32-s3-m5stack-stopwatch/main/touch.cpp +158 -88
- package/targets/esp32-s3-m5stack-stopwatch/sdkconfig.defaults +13 -0
- package/targets/esp32-s3-touch-amoled-1.8/main/CMakeLists.txt +30 -3
- package/targets/esp32-s3-touch-amoled-2.06/dependencies.lock +10 -148
- package/targets/esp32-s31-espressif-mosaico/README.md +34 -0
- package/targets/esp32-s31-espressif-mosaico/dependencies.lock +49 -1
- package/targets/esp32-s31-espressif-mosaico/main/CMakeLists.txt +23 -5
- package/targets/esp32-s31-espressif-mosaico/main/board_power.cpp +117 -0
- package/targets/esp32-s31-espressif-mosaico/main/camera.cpp +326 -0
- package/targets/esp32-s31-espressif-mosaico/main/idf_component.yml +11 -0
- package/targets/esp32-s31-espressif-mosaico/main/include/board.h +23 -0
- package/targets/esp32-s31-espressif-mosaico/main/nand_mount.cpp +172 -0
- package/targets/esp32-s31-espressif-mosaico/partitions.csv +7 -3
- package/targets/esp32-s31-espressif-mosaico/sdkconfig.defaults +24 -0
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
// Official Espressif ESP-SR AEC; adapter only, no custom cancellation DSP.
|
|
2
2
|
#include "echo_cancellation.h"
|
|
3
3
|
#include "esp_aec.h"
|
|
4
|
+
#include "esp_agc.h"
|
|
5
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
6
|
+
#include "esp_ns.h"
|
|
7
|
+
#endif
|
|
4
8
|
#include "esp_heap_caps.h"
|
|
5
9
|
#include "esp_log.h"
|
|
6
10
|
#include "esp_timer.h"
|
|
@@ -16,31 +20,149 @@
|
|
|
16
20
|
namespace {
|
|
17
21
|
constexpr uint32_t kMaxFrame = 512, kDmaFrames = 240, kCaptureRing = 1024;
|
|
18
22
|
portMUX_TYPE dmaMux = portMUX_INITIALIZER_UNLOCKED;
|
|
19
|
-
int16_t dmaMic[kCaptureRing]{}, dmaRef[kCaptureRing]{};
|
|
20
|
-
int16_t previousMic[kDmaFrames]{}, latestTx[kDmaFrames]{}, captureRef[320]{};
|
|
23
|
+
int16_t dmaMic[kCaptureRing]{}, dmaMic2[kCaptureRing]{}, dmaRef[kCaptureRing]{};
|
|
24
|
+
int16_t previousMic[kDmaFrames]{}, latestTx[kDmaFrames]{}, captureRef[320]{}, captureMic2[320]{};
|
|
21
25
|
uint32_t dmaWritten = 0, dmaRead = 0, dmaDrops = 0;
|
|
22
26
|
bool dmaActive = false, previousMicReady = false;
|
|
23
27
|
bool hardwareReference = false;
|
|
24
28
|
float outputGain = 1.0f;
|
|
29
|
+
unsigned microphoneCount = 1;
|
|
30
|
+
bool releaseQuietReference = false;
|
|
31
|
+
size_t quietReferenceSamples = 0;
|
|
25
32
|
std::mutex aecMutex;
|
|
26
33
|
aec_handle_t *aec = nullptr;
|
|
27
|
-
|
|
28
|
-
|
|
34
|
+
void *agc = nullptr;
|
|
35
|
+
constexpr size_t kAgcFrame = 160; // ESP-SR AGC requires 10 ms at 16 kHz.
|
|
36
|
+
constexpr int kOutputPeakLimit = 28000; // 1.37 dB below the PCM16 ceiling.
|
|
37
|
+
int16_t *agcInput = nullptr, *agcOutput = nullptr;
|
|
38
|
+
size_t agcFill = 0;
|
|
39
|
+
bool gainFailed = false;
|
|
40
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
41
|
+
bool experimentAec = true, experimentNlp = true, experimentAgc = true;
|
|
42
|
+
bool experimentInitialized = false;
|
|
43
|
+
int experimentMic = 0, experimentNlpLevel = 0, experimentNsLevel = -1;
|
|
44
|
+
int experimentCompression = 12, experimentTarget = 6, experimentQuietMs = 128;
|
|
45
|
+
aec_config_t experimentAecConfig{};
|
|
46
|
+
ns_handle_t experimentNs = nullptr;
|
|
47
|
+
int16_t *experimentCapture = nullptr;
|
|
48
|
+
size_t captureCapacity = 0, captureFrames = 0;
|
|
49
|
+
bool captureRunning = false;
|
|
50
|
+
int64_t captureStartedUs = 0;
|
|
51
|
+
uint32_t captureDrops = 0, captureEndDrops = 0;
|
|
52
|
+
#endif
|
|
53
|
+
// Only the ISR capture ring needs internal RAM. Keep the aligned DSP scratch
|
|
54
|
+
// in PSRAM, leaving DMA-capable RAM available for I2S and TLS encryption.
|
|
55
|
+
int16_t *scratch = nullptr, *mic = nullptr, *secondMicFrame = nullptr, *ref = nullptr, *clean = nullptr,
|
|
56
|
+
*linear = nullptr, *output = nullptr;
|
|
29
57
|
size_t fill = 0, frameSize = 0;
|
|
30
58
|
|
|
31
59
|
uint64_t rawEnergy = 0, cleanEnergy = 0, linearEnergy = 0, refEnergy = 0, totalSamples = 0;
|
|
32
60
|
int64_t logAt = 0, processUs = 0;
|
|
33
61
|
uint32_t processedFrames = 0, clippedSamples = 0, clippedReferenceSamples = 0;
|
|
34
62
|
int rawPeak = 0, cleanPeak = 0, referencePeak = 0;
|
|
63
|
+
uint32_t outputClipped = 0;
|
|
64
|
+
uint64_t secondMicEnergy = 0;
|
|
65
|
+
int secondMicPeak = 0;
|
|
66
|
+
uint64_t outputEnergy = 0, outputSamples = 0;
|
|
67
|
+
int outputPeak = 0;
|
|
68
|
+
uint32_t peakLimitedFrames = 0, agcFullScaleSamples = 0;
|
|
69
|
+
|
|
70
|
+
// AGC consumes 10 ms blocks while AEC produces 32 ms blocks. Retain the tail
|
|
71
|
+
// across calls, preserving sample order without padding or an extra audio task.
|
|
72
|
+
size_t applyOutputGain(const int16_t *samples, size_t count, int16_t *destination) {
|
|
73
|
+
bool useAgc = agc != nullptr;
|
|
74
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
75
|
+
useAgc = useAgc && experimentAgc;
|
|
76
|
+
if (!useAgc && !experimentNs) {
|
|
77
|
+
int peak = 0;
|
|
78
|
+
for (size_t j = 0; j < count; ++j) peak = std::max(peak, std::abs(int(samples[j])));
|
|
79
|
+
const float gain = peak ? std::min(outputGain, float(kOutputPeakLimit) / peak) : outputGain;
|
|
80
|
+
peakLimitedFrames += gain < outputGain;
|
|
81
|
+
for (size_t j = 0; j < count; ++j) destination[j] = int16_t(std::lround(samples[j] * gain));
|
|
82
|
+
return count;
|
|
83
|
+
}
|
|
84
|
+
#else
|
|
85
|
+
if (!useAgc) {
|
|
86
|
+
for (size_t j = 0; j < count; ++j) {
|
|
87
|
+
const float amplified = samples[j] * outputGain;
|
|
88
|
+
outputClipped += amplified > 32767.0f || amplified < -32768.0f;
|
|
89
|
+
destination[j] = int16_t(std::lround(std::clamp(amplified, -32768.0f, 32767.0f)));
|
|
90
|
+
}
|
|
91
|
+
return count;
|
|
92
|
+
}
|
|
93
|
+
#endif
|
|
94
|
+
int peak = 0;
|
|
95
|
+
for (size_t j = 0; j < count; ++j) peak = std::max(peak, std::abs(int(samples[j])));
|
|
96
|
+
// Protect the AGC input BEFORE converting amplified samples to PCM16.
|
|
97
|
+
// Clamping first would irreversibly distort peaks before the limiter saw them.
|
|
98
|
+
const float gain = peak ? std::min(outputGain, float(kOutputPeakLimit) / peak) : outputGain;
|
|
99
|
+
peakLimitedFrames += gain < outputGain;
|
|
100
|
+
size_t produced = 0;
|
|
101
|
+
for (size_t j = 0; j < count; ++j) {
|
|
102
|
+
agcInput[agcFill++] = int16_t(std::lround(samples[j] * gain));
|
|
103
|
+
if (agcFill != kAgcFrame) continue;
|
|
104
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
105
|
+
if (experimentNs) {
|
|
106
|
+
ns_process(experimentNs, agcInput, agcOutput);
|
|
107
|
+
std::memcpy(agcInput, agcOutput, kAgcFrame * sizeof(int16_t));
|
|
108
|
+
}
|
|
109
|
+
#endif
|
|
110
|
+
int status = 0;
|
|
111
|
+
if (useAgc) status = esp_agc_process(agc, agcInput, agcOutput, kAgcFrame, 16000);
|
|
112
|
+
else std::memcpy(agcOutput, agcInput, kAgcFrame * sizeof(int16_t));
|
|
113
|
+
agcFill = 0;
|
|
114
|
+
if (status < 0) {
|
|
115
|
+
ESP_LOGE("gea_aec", "Post-AEC AGC failed: %d", status);
|
|
116
|
+
gainFailed = true;
|
|
117
|
+
return 0;
|
|
118
|
+
}
|
|
119
|
+
int processedPeak = 0;
|
|
120
|
+
for (size_t k = 0; k < kAgcFrame; ++k) {
|
|
121
|
+
processedPeak = std::max(processedPeak, std::abs(int(agcOutput[k])));
|
|
122
|
+
agcFullScaleSamples += agcOutput[k] == 32767 || agcOutput[k] == -32768;
|
|
123
|
+
}
|
|
124
|
+
// A final uniform block attenuation preserves waveform shape and leaves
|
|
125
|
+
// headroom for transport resampling, including negative full-scale samples.
|
|
126
|
+
const float protection = processedPeak > kOutputPeakLimit
|
|
127
|
+
? float(kOutputPeakLimit) / processedPeak : 1.0f;
|
|
128
|
+
peakLimitedFrames += protection < 1.0f;
|
|
129
|
+
for (size_t k = 0; k < kAgcFrame; ++k)
|
|
130
|
+
destination[produced + k] = int16_t(std::lround(agcOutput[k] * protection));
|
|
131
|
+
produced += kAgcFrame;
|
|
132
|
+
}
|
|
133
|
+
return produced;
|
|
134
|
+
}
|
|
35
135
|
} // namespace
|
|
36
136
|
|
|
37
|
-
bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp) {
|
|
137
|
+
bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp, unsigned inputMicrophones, bool adaptiveGain) {
|
|
38
138
|
std::lock_guard<std::mutex> lock(aecMutex);
|
|
39
139
|
if (aec)
|
|
40
140
|
return true;
|
|
41
|
-
if (!std::isfinite(gain) || gain <= 0
|
|
141
|
+
if (!std::isfinite(gain) || gain <= 0 || inputMicrophones < 1 || inputMicrophones > 2 ||
|
|
142
|
+
(inputMicrophones == 2 && !useHardwareReference))
|
|
143
|
+
return false;
|
|
144
|
+
scratch = static_cast<int16_t *>(heap_caps_aligned_alloc(
|
|
145
|
+
16, (7 * kMaxFrame + 2 * kAgcFrame) * sizeof(int16_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
|
|
146
|
+
if (!scratch) {
|
|
147
|
+
ESP_LOGE("gea_aec", "AEC processing buffer allocation failed");
|
|
42
148
|
return false;
|
|
149
|
+
}
|
|
150
|
+
mic = scratch;
|
|
151
|
+
secondMicFrame = mic + kMaxFrame;
|
|
152
|
+
ref = secondMicFrame + kMaxFrame;
|
|
153
|
+
clean = ref + kMaxFrame;
|
|
154
|
+
linear = clean + kMaxFrame;
|
|
155
|
+
output = linear + kMaxFrame;
|
|
156
|
+
#if !GEA_AUDIO_EXPERIMENT
|
|
43
157
|
outputGain = gain;
|
|
158
|
+
#endif
|
|
159
|
+
microphoneCount = inputMicrophones;
|
|
160
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
161
|
+
releaseQuietReference = experimentNlpLevel == 0;
|
|
162
|
+
#else
|
|
163
|
+
releaseQuietReference = !aggressiveNlp;
|
|
164
|
+
#endif
|
|
165
|
+
quietReferenceSamples = 0;
|
|
44
166
|
aec_config_t config{};
|
|
45
167
|
config.mic_num = 1;
|
|
46
168
|
config.ref_num = 1;
|
|
@@ -51,9 +173,15 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
|
|
|
51
173
|
config.mode = AEC_MODE_FD_HIGH_PERF;
|
|
52
174
|
// Select residual echo suppression independently of the output gain.
|
|
53
175
|
config.nlp_level = aggressiveNlp ? AEC_NLP_LEVEL_AGGR : AEC_NLP_LEVEL_NORMAL;
|
|
176
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
177
|
+
config.nlp_level = static_cast<aec_nlp_level_t>(experimentNlpLevel);
|
|
178
|
+
experimentAecConfig = config;
|
|
179
|
+
#endif
|
|
54
180
|
aec = aec_create_from_config(&config);
|
|
55
181
|
if (!aec) {
|
|
56
182
|
ESP_LOGE("gea_aec", "ESP-SR AEC allocation failed");
|
|
183
|
+
heap_caps_free(scratch);
|
|
184
|
+
scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
|
|
57
185
|
return false;
|
|
58
186
|
}
|
|
59
187
|
frameSize = aec_get_chunksize(aec);
|
|
@@ -61,8 +189,46 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
|
|
|
61
189
|
ESP_LOGE("gea_aec", "Unsupported AEC frame size %u", unsigned(frameSize));
|
|
62
190
|
aec_destroy(aec);
|
|
63
191
|
aec = nullptr;
|
|
192
|
+
heap_caps_free(scratch);
|
|
193
|
+
scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
|
|
64
194
|
return false;
|
|
65
195
|
}
|
|
196
|
+
agcInput = output + 2 * kMaxFrame;
|
|
197
|
+
agcOutput = agcInput + kAgcFrame;
|
|
198
|
+
if (adaptiveGain) {
|
|
199
|
+
agc = esp_agc_open(AGC_MODE_2, 16000);
|
|
200
|
+
if (!agc) {
|
|
201
|
+
ESP_LOGE("gea_aec", "Post-AEC AGC allocation failed");
|
|
202
|
+
aec_destroy(aec);
|
|
203
|
+
aec = nullptr;
|
|
204
|
+
heap_caps_free(scratch);
|
|
205
|
+
scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
|
|
206
|
+
agcInput = agcOutput = nullptr;
|
|
207
|
+
return false;
|
|
208
|
+
}
|
|
209
|
+
// 12 dB compression gain, built-in limiter, -6 dBFS target.
|
|
210
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
211
|
+
set_agc_config(agc, experimentCompression, 1, experimentTarget);
|
|
212
|
+
#else
|
|
213
|
+
set_agc_config(agc, 12, 1, 6);
|
|
214
|
+
#endif
|
|
215
|
+
}
|
|
216
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
217
|
+
if (!experimentNs && experimentNsLevel >= 0) {
|
|
218
|
+
experimentNs = ns_pro_create(10, experimentNsLevel, 16000);
|
|
219
|
+
if (!experimentNs) {
|
|
220
|
+
if (agc) esp_agc_close(agc);
|
|
221
|
+
agc = nullptr;
|
|
222
|
+
aec_destroy(aec); aec = nullptr;
|
|
223
|
+
heap_caps_free(scratch);
|
|
224
|
+
scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
|
|
225
|
+
agcInput = agcOutput = nullptr;
|
|
226
|
+
return false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
#endif
|
|
230
|
+
agcFill = 0;
|
|
231
|
+
gainFailed = false;
|
|
66
232
|
fill = 0;
|
|
67
233
|
portENTER_CRITICAL(&dmaMux);
|
|
68
234
|
dmaWritten = dmaRead = dmaDrops = 0;
|
|
@@ -74,18 +240,32 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
|
|
|
74
240
|
rawEnergy = cleanEnergy = linearEnergy = refEnergy = totalSamples = 0;
|
|
75
241
|
processUs = processedFrames = clippedSamples = clippedReferenceSamples = 0;
|
|
76
242
|
rawPeak = cleanPeak = referencePeak = 0;
|
|
243
|
+
outputClipped = 0;
|
|
244
|
+
secondMicEnergy = 0;
|
|
245
|
+
secondMicPeak = 0;
|
|
246
|
+
outputEnergy = outputSamples = 0;
|
|
247
|
+
outputPeak = 0;
|
|
248
|
+
peakLimitedFrames = agcFullScaleSamples = 0;
|
|
77
249
|
logAt = esp_timer_get_time();
|
|
78
250
|
ESP_LOGI("gea_aec",
|
|
79
|
-
"ESP-SR
|
|
80
|
-
"reference=%s internal_free=%u",
|
|
251
|
+
"ESP-SR AEC ready: rate=16000 frame=%u NLP=%s "
|
|
252
|
+
"physical_mics=%u aec_channels=1 reference=%s adaptive_gain=%s internal_free=%u",
|
|
81
253
|
unsigned(frameSize), aggressiveNlp ? "aggressive" : "normal",
|
|
82
|
-
hardwareReference ? "ADC" : "TX-DMA",
|
|
254
|
+
microphoneCount, hardwareReference ? "ADC" : "TX-DMA", agc ? "digital+limiter" : "off",
|
|
83
255
|
unsigned(heap_caps_get_free_size(MALLOC_CAP_INTERNAL)));
|
|
84
256
|
return true;
|
|
85
257
|
}
|
|
86
258
|
|
|
87
259
|
void geaAudioAecStop() {
|
|
88
260
|
std::lock_guard<std::mutex> lock(aecMutex);
|
|
261
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
262
|
+
if (captureRunning) {
|
|
263
|
+
portENTER_CRITICAL(&dmaMux);
|
|
264
|
+
captureEndDrops = dmaDrops;
|
|
265
|
+
portEXIT_CRITICAL(&dmaMux);
|
|
266
|
+
}
|
|
267
|
+
captureRunning = false; // Retain the bounded recording for silent USB export.
|
|
268
|
+
#endif
|
|
89
269
|
portENTER_CRITICAL(&dmaMux);
|
|
90
270
|
dmaActive = false;
|
|
91
271
|
portEXIT_CRITICAL(&dmaMux);
|
|
@@ -93,6 +273,18 @@ void geaAudioAecStop() {
|
|
|
93
273
|
aec_destroy(aec);
|
|
94
274
|
aec = nullptr;
|
|
95
275
|
}
|
|
276
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
277
|
+
if (experimentNs) ns_destroy(experimentNs);
|
|
278
|
+
experimentNs = nullptr;
|
|
279
|
+
|
|
280
|
+
#endif
|
|
281
|
+
if (agc) esp_agc_close(agc);
|
|
282
|
+
agc = nullptr;
|
|
283
|
+
agcInput = agcOutput = nullptr;
|
|
284
|
+
agcFill = 0;
|
|
285
|
+
gainFailed = false;
|
|
286
|
+
heap_caps_free(scratch);
|
|
287
|
+
scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
|
|
96
288
|
fill = 0;
|
|
97
289
|
}
|
|
98
290
|
|
|
@@ -110,7 +302,8 @@ void geaAudioAecReference(const int16_t *pcm, size_t count) {
|
|
|
110
302
|
}
|
|
111
303
|
void geaAudioAecReceive(const int16_t *stereo, size_t count) {
|
|
112
304
|
portENTER_CRITICAL_ISR(&dmaMux);
|
|
113
|
-
|
|
305
|
+
const size_t stride = hardwareReference ? microphoneCount + 1 : 2;
|
|
306
|
+
if (dmaActive && count == kDmaFrames * stride) {
|
|
114
307
|
if (hardwareReference) {
|
|
115
308
|
// ES7210 analog loopback or ES8311 ADCL + DACR arrive in the same RX
|
|
116
309
|
// frame. Keep them paired across callback/task scheduling jitter.
|
|
@@ -119,8 +312,11 @@ void geaAudioAecReceive(const int16_t *stereo, size_t count) {
|
|
|
119
312
|
++dmaDrops;
|
|
120
313
|
}
|
|
121
314
|
for (size_t i = 0; i < kDmaFrames; ++i) {
|
|
122
|
-
|
|
123
|
-
|
|
315
|
+
// ES7210 slots are MIC1, speaker reference, MIC2.
|
|
316
|
+
dmaMic[(dmaWritten + i) % kCaptureRing] = stereo[i * stride];
|
|
317
|
+
dmaRef[(dmaWritten + i) % kCaptureRing] = stereo[i * stride + 1];
|
|
318
|
+
if (microphoneCount == 2)
|
|
319
|
+
dmaMic2[(dmaWritten + i) % kCaptureRing] = stereo[i * stride + 2];
|
|
124
320
|
}
|
|
125
321
|
dmaWritten += kDmaFrames;
|
|
126
322
|
portEXIT_CRITICAL_ISR(&dmaMux);
|
|
@@ -152,6 +348,8 @@ bool geaAudioAecRead(int16_t *pcm, size_t count) {
|
|
|
152
348
|
for (size_t i = 0; i < count; ++i) {
|
|
153
349
|
pcm[i] = dmaMic[(dmaRead + i) % kCaptureRing];
|
|
154
350
|
captureRef[i] = dmaRef[(dmaRead + i) % kCaptureRing];
|
|
351
|
+
if (microphoneCount == 2)
|
|
352
|
+
captureMic2[i] = dmaMic2[(dmaRead + i) % kCaptureRing];
|
|
155
353
|
}
|
|
156
354
|
dmaRead += count;
|
|
157
355
|
}
|
|
@@ -163,13 +361,37 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
|
|
|
163
361
|
const int16_t **result) {
|
|
164
362
|
std::lock_guard<std::mutex> lock(aecMutex);
|
|
165
363
|
*result = output;
|
|
166
|
-
if (!aec)
|
|
364
|
+
if (!aec || gainFailed)
|
|
167
365
|
return 0; // Never silently forward uncancelled audio after an AEC failure.
|
|
168
366
|
size_t produced = 0;
|
|
169
367
|
for (size_t i = 0; i < count; ++i) {
|
|
170
|
-
|
|
368
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
369
|
+
if (captureRunning && captureFrames < captureCapacity) {
|
|
370
|
+
if (!captureFrames) captureStartedUs = esp_timer_get_time();
|
|
371
|
+
experimentCapture[captureFrames * 3] = pcm[i];
|
|
372
|
+
experimentCapture[captureFrames * 3 + 1] = microphoneCount == 2 ? captureMic2[i] : 0;
|
|
373
|
+
experimentCapture[captureFrames * 3 + 2] = captureRef[i];
|
|
374
|
+
if (++captureFrames == captureCapacity) {
|
|
375
|
+
captureRunning = false;
|
|
376
|
+
portENTER_CRITICAL(&dmaMux);
|
|
377
|
+
captureEndDrops = dmaDrops;
|
|
378
|
+
portEXIT_CRITICAL(&dmaMux);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
#endif
|
|
382
|
+
// Both microphones are synchronized by the same ADC/I2S frame. Mix
|
|
383
|
+
// before cancellation so one AEC models their combined speaker echo path.
|
|
384
|
+
// Averaging avoids doubling amplitude and retains the single-channel CPU cost.
|
|
385
|
+
mic[fill] = microphoneCount == 2
|
|
386
|
+
? int16_t((int32_t(pcm[i]) + captureMic2[i]) / 2) : pcm[i];
|
|
387
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
388
|
+
if (microphoneCount == 2 && experimentMic == 1) mic[fill] = pcm[i];
|
|
389
|
+
if (microphoneCount == 2 && experimentMic == 2) mic[fill] = captureMic2[i];
|
|
390
|
+
#endif
|
|
391
|
+
if (microphoneCount == 2) secondMicFrame[fill] = captureMic2[i];
|
|
171
392
|
ref[fill] = captureRef[i];
|
|
172
393
|
if (++fill == frameSize) {
|
|
394
|
+
uint64_t frameReferenceEnergy = 0;
|
|
173
395
|
for (size_t j = 0; j < frameSize; ++j) {
|
|
174
396
|
rawPeak = std::max(rawPeak, std::abs(int(mic[j])));
|
|
175
397
|
referencePeak = std::max(referencePeak, std::abs(int(ref[j])));
|
|
@@ -177,15 +399,29 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
|
|
|
177
399
|
clippedReferenceSamples += std::abs(int(ref[j])) >= 32760;
|
|
178
400
|
rawEnergy += int64_t(mic[j]) * mic[j];
|
|
179
401
|
refEnergy += int64_t(ref[j]) * ref[j];
|
|
402
|
+
frameReferenceEnergy += int64_t(ref[j]) * ref[j];
|
|
403
|
+
if (microphoneCount == 2) {
|
|
404
|
+
const int16_t second = secondMicFrame[j];
|
|
405
|
+
secondMicEnergy += int64_t(second) * second;
|
|
406
|
+
secondMicPeak = std::max(secondMicPeak, std::abs(int(second)));
|
|
407
|
+
}
|
|
180
408
|
}
|
|
181
409
|
auto before = esp_timer_get_time();
|
|
182
410
|
// ESP-SR's documented two-stage API preserves both cancellation stages.
|
|
183
411
|
// Snapshot the linear result so telemetry can identify suppression of
|
|
184
412
|
// near-end speech separately from cancellation of the speaker echo.
|
|
413
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
414
|
+
if (experimentAec) aec_linear_process(aec, mic, ref, clean);
|
|
415
|
+
else std::memcpy(clean, mic, frameSize * sizeof(int16_t));
|
|
416
|
+
#else
|
|
185
417
|
aec_linear_process(aec, mic, ref, clean);
|
|
418
|
+
#endif
|
|
186
419
|
std::memcpy(linear, clean, frameSize * sizeof(int16_t));
|
|
420
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
421
|
+
if (experimentAec && experimentNlp) aec_nlp_process(aec, clean);
|
|
422
|
+
#else
|
|
187
423
|
aec_nlp_process(aec, clean);
|
|
188
|
-
|
|
424
|
+
#endif
|
|
189
425
|
++processedFrames;
|
|
190
426
|
for (size_t j = 0; j < frameSize; ++j) {
|
|
191
427
|
linearEnergy += int64_t(linear[j]) * linear[j];
|
|
@@ -193,25 +429,37 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
|
|
|
193
429
|
cleanPeak = std::max(cleanPeak, std::abs(int(clean[j])));
|
|
194
430
|
}
|
|
195
431
|
totalSamples += frameSize;
|
|
196
|
-
//
|
|
197
|
-
//
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
432
|
+
// Keep NLP trained, but release its hangover once playback has been
|
|
433
|
+
// quiet for 128 ms. A whole quiet interval covers the linear filter's
|
|
434
|
+
// output delay and acoustic tail; one quiet frame inside a word does not.
|
|
435
|
+
// Linear AEC remains active and removes residual room/DMA echo.
|
|
436
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
437
|
+
const size_t kQuietReferenceSamples = size_t(experimentQuietMs) * 16;
|
|
438
|
+
#else
|
|
439
|
+
constexpr size_t kQuietReferenceSamples = 2048;
|
|
440
|
+
#endif
|
|
441
|
+
quietReferenceSamples = frameReferenceEnergy < uint64_t(128 * 128) * frameSize
|
|
442
|
+
? std::min(kQuietReferenceSamples, quietReferenceSamples + frameSize) : 0;
|
|
443
|
+
const bool released = releaseQuietReference && kQuietReferenceSamples > 0 && quietReferenceSamples >= kQuietReferenceSamples;
|
|
444
|
+
const size_t gained = applyOutputGain(released ? linear : clean,
|
|
445
|
+
frameSize, output + produced);
|
|
207
446
|
fill = 0;
|
|
447
|
+
if (gainFailed) return 0;
|
|
448
|
+
for (size_t j = 0; j < gained; ++j) {
|
|
449
|
+
const int16_t value = output[produced + j];
|
|
450
|
+
outputEnergy += int64_t(value) * value;
|
|
451
|
+
outputPeak = std::max(outputPeak, std::abs(int(value)));
|
|
452
|
+
}
|
|
453
|
+
outputSamples += gained;
|
|
454
|
+
produced += gained;
|
|
455
|
+
processUs += esp_timer_get_time() - before;
|
|
208
456
|
}
|
|
209
457
|
}
|
|
210
458
|
const auto now = esp_timer_get_time();
|
|
211
459
|
if (now - logAt >= 1000000 && totalSamples) {
|
|
212
460
|
ESP_LOGI("gea_aec",
|
|
213
461
|
"raw_rms=%.0f clean_rms=%.0f ref_rms=%.0f linear_rms=%.0f reduction_db=%.1f "
|
|
214
|
-
"process_us=%lld dma_drops=%lu clipped=%lu ref_clipped=%lu raw_peak=%d clean_peak=%d ref_peak=%d",
|
|
462
|
+
"process_us=%lld dma_drops=%lu clipped=%lu ref_clipped=%lu raw_peak=%d clean_peak=%d ref_peak=%d output_clipped=%lu output_clipped_pct=%.4f output_gain=%.3f output_rms=%.0f output_peak=%d peak_limited_frames=%lu agc_full_scale_samples=%lu mic2_rms=%.0f mic2_peak=%d",
|
|
215
463
|
std::sqrt(double(rawEnergy) / totalSamples),
|
|
216
464
|
std::sqrt(double(cleanEnergy) / totalSamples),
|
|
217
465
|
std::sqrt(double(refEnergy) / totalSamples),
|
|
@@ -219,11 +467,127 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
|
|
|
219
467
|
10 * std::log10(double(rawEnergy + 1) / double(cleanEnergy + 1)),
|
|
220
468
|
(long long)(processUs / std::max<uint32_t>(1, processedFrames)),
|
|
221
469
|
(unsigned long)dmaDrops, (unsigned long)clippedSamples,
|
|
222
|
-
(unsigned long)clippedReferenceSamples, rawPeak, cleanPeak, referencePeak
|
|
470
|
+
(unsigned long)clippedReferenceSamples, rawPeak, cleanPeak, referencePeak,
|
|
471
|
+
(unsigned long)outputClipped, 100.0 * outputClipped / totalSamples, outputGain,
|
|
472
|
+
std::sqrt(double(outputEnergy) / std::max<uint64_t>(1, outputSamples)), outputPeak,
|
|
473
|
+
(unsigned long)peakLimitedFrames, (unsigned long)agcFullScaleSamples,
|
|
474
|
+
std::sqrt(double(secondMicEnergy) / totalSamples), secondMicPeak);
|
|
223
475
|
rawEnergy = cleanEnergy = linearEnergy = refEnergy = totalSamples = 0;
|
|
224
476
|
rawPeak = cleanPeak = referencePeak = 0;
|
|
477
|
+
outputClipped = 0;
|
|
478
|
+
secondMicEnergy = 0;
|
|
479
|
+
secondMicPeak = 0;
|
|
480
|
+
outputEnergy = outputSamples = 0;
|
|
481
|
+
outputPeak = 0;
|
|
482
|
+
peakLimitedFrames = agcFullScaleSamples = 0;
|
|
225
483
|
processUs = processedFrames = clippedSamples = clippedReferenceSamples = 0;
|
|
226
484
|
logAt = now;
|
|
227
485
|
}
|
|
228
486
|
return produced;
|
|
229
487
|
}
|
|
488
|
+
|
|
489
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
490
|
+
void geaAudioAecExperimentDefaults(float gain, bool aggressive, unsigned microphones, bool adaptive) {
|
|
491
|
+
std::lock_guard<std::mutex> lock(aecMutex);
|
|
492
|
+
if (experimentInitialized) return;
|
|
493
|
+
experimentInitialized = true;
|
|
494
|
+
outputGain = gain;
|
|
495
|
+
experimentNlpLevel = aggressive ? 1 : 0;
|
|
496
|
+
experimentAgc = adaptive;
|
|
497
|
+
microphoneCount = microphones;
|
|
498
|
+
}
|
|
499
|
+
bool geaAudioAecConfigure(const char *key, float value) {
|
|
500
|
+
if (!key || !std::isfinite(value) || value < -1 || value > 1000) return false;
|
|
501
|
+
std::lock_guard<std::mutex> lock(aecMutex);
|
|
502
|
+
const int number = int(value);
|
|
503
|
+
const bool integer = value == float(number);
|
|
504
|
+
if (!std::strcmp(key, "capture") && integer && number >= 0 && number <= 8) {
|
|
505
|
+
if (!number) {
|
|
506
|
+
if (captureRunning) {
|
|
507
|
+
portENTER_CRITICAL(&dmaMux);
|
|
508
|
+
captureEndDrops = dmaDrops;
|
|
509
|
+
portEXIT_CRITICAL(&dmaMux);
|
|
510
|
+
}
|
|
511
|
+
captureRunning = false;
|
|
512
|
+
return true;
|
|
513
|
+
}
|
|
514
|
+
if (!aec || captureRunning) return false;
|
|
515
|
+
const size_t capacity = size_t(number) * 16000;
|
|
516
|
+
auto *replacement = static_cast<int16_t *>(heap_caps_malloc(capacity * 3 * sizeof(int16_t),
|
|
517
|
+
MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
|
|
518
|
+
if (!replacement) return false;
|
|
519
|
+
heap_caps_free(experimentCapture);
|
|
520
|
+
experimentCapture = replacement;
|
|
521
|
+
captureCapacity = capacity;
|
|
522
|
+
captureFrames = 0;
|
|
523
|
+
captureStartedUs = 0;
|
|
524
|
+
portENTER_CRITICAL(&dmaMux);
|
|
525
|
+
captureDrops = captureEndDrops = dmaDrops;
|
|
526
|
+
portEXIT_CRITICAL(&dmaMux);
|
|
527
|
+
captureRunning = true;
|
|
528
|
+
return true;
|
|
529
|
+
}
|
|
530
|
+
if (!std::strcmp(key, "gain") && value >= .01f && value <= 256) outputGain = value;
|
|
531
|
+
else if (!std::strcmp(key, "aec") && integer && number >= 0 && number <= 1) experimentAec = number;
|
|
532
|
+
else if (!std::strcmp(key, "nlp") && integer && number >= 0 && number <= 1) experimentNlp = number;
|
|
533
|
+
else if (!std::strcmp(key, "mic") && integer && number >= 0 && number <= 2) {
|
|
534
|
+
if (number == 2 && microphoneCount != 2) return false;
|
|
535
|
+
experimentMic = number;
|
|
536
|
+
} else if (!std::strcmp(key, "agc") && integer && number >= 0 && number <= 1) {
|
|
537
|
+
if (number && !agc && aec) return false;
|
|
538
|
+
experimentAgc = number;
|
|
539
|
+
} else if (!std::strcmp(key, "agc_db") && integer && number >= 0 && number <= 30) {
|
|
540
|
+
experimentCompression = number;
|
|
541
|
+
if (agc) set_agc_config(agc, experimentCompression, 1, experimentTarget);
|
|
542
|
+
} else if (!std::strcmp(key, "agc_target") && integer && number >= 1 && number <= 20) {
|
|
543
|
+
experimentTarget = number;
|
|
544
|
+
if (agc) set_agc_config(agc, experimentCompression, 1, experimentTarget);
|
|
545
|
+
} else if (!std::strcmp(key, "quiet_ms") && integer && number >= 0 && number <= 1000) experimentQuietMs = number;
|
|
546
|
+
else if (!std::strcmp(key, "nlp_level") && integer && number >= 0 && number <= 2) {
|
|
547
|
+
if (number == experimentNlpLevel) return true;
|
|
548
|
+
if (aec) {
|
|
549
|
+
auto config = experimentAecConfig;
|
|
550
|
+
config.nlp_level = static_cast<aec_nlp_level_t>(number);
|
|
551
|
+
auto *replacement = aec_create_from_config(&config);
|
|
552
|
+
if (!replacement) return false;
|
|
553
|
+
if (size_t(aec_get_chunksize(replacement)) != frameSize) { aec_destroy(replacement); return false; }
|
|
554
|
+
aec_destroy(aec);
|
|
555
|
+
aec = replacement;
|
|
556
|
+
experimentAecConfig = config;
|
|
557
|
+
fill = 0;
|
|
558
|
+
}
|
|
559
|
+
experimentNlpLevel = number;
|
|
560
|
+
releaseQuietReference = number == 0;
|
|
561
|
+
} else if (!std::strcmp(key, "ns") && integer && number >= -1 && number <= 2) {
|
|
562
|
+
if (number == experimentNsLevel) return true;
|
|
563
|
+
ns_handle_t replacement = number < 0 ? nullptr : ns_pro_create(10, number, 16000);
|
|
564
|
+
if (number >= 0 && !replacement) return false;
|
|
565
|
+
if (experimentNs) ns_destroy(experimentNs);
|
|
566
|
+
experimentNs = replacement;
|
|
567
|
+
experimentNsLevel = number;
|
|
568
|
+
} else return false;
|
|
569
|
+
agcFill = 0;
|
|
570
|
+
quietReferenceSamples = 0;
|
|
571
|
+
return true;
|
|
572
|
+
}
|
|
573
|
+
bool geaAudioAecCaptureRead(size_t offset, int16_t *samples, size_t capacity,
|
|
574
|
+
size_t *frames, size_t *total, int64_t *startedUs,
|
|
575
|
+
uint32_t *drops) {
|
|
576
|
+
std::lock_guard<std::mutex> lock(aecMutex);
|
|
577
|
+
if (captureRunning || !experimentCapture || offset > captureFrames) return false;
|
|
578
|
+
*frames = std::min(capacity, captureFrames - offset);
|
|
579
|
+
*total = captureFrames;
|
|
580
|
+
*startedUs = captureStartedUs;
|
|
581
|
+
*drops = captureEndDrops - captureDrops;
|
|
582
|
+
std::memcpy(samples, experimentCapture + offset * 3, *frames * 3 * sizeof(int16_t));
|
|
583
|
+
return true;
|
|
584
|
+
}
|
|
585
|
+
void geaAudioAecDescribe(char *buffer, size_t capacity) {
|
|
586
|
+
std::lock_guard<std::mutex> lock(aecMutex);
|
|
587
|
+
std::snprintf(buffer, capacity,
|
|
588
|
+
"aec=%d nlp=%d nlp_level=%d gain=%.3f agc=%d agc_db=%d agc_target=%d ns=%d mic=%d physical_mics=%u quiet_ms=%d ready=%d",
|
|
589
|
+
experimentAec, experimentNlp, experimentNlpLevel, outputGain, experimentAgc,
|
|
590
|
+
experimentCompression, experimentTarget, experimentNsLevel, experimentMic,
|
|
591
|
+
microphoneCount, experimentQuietMs, aec != nullptr);
|
|
592
|
+
}
|
|
593
|
+
#endif
|
|
@@ -3,11 +3,27 @@
|
|
|
3
3
|
#include <cstddef>
|
|
4
4
|
#include <cstdint>
|
|
5
5
|
|
|
6
|
+
// Two physical inputs require paired ADC slots [mic1, reference, mic2].
|
|
7
|
+
// They are averaged before one AEC channel; published output remains mono.
|
|
6
8
|
bool geaAudioAecStart(bool hardwareReference = false, float outputGain = 1.0f,
|
|
7
|
-
bool aggressiveNlp = true);
|
|
9
|
+
bool aggressiveNlp = true, unsigned microphoneCount = 1, bool adaptiveGain = false);
|
|
8
10
|
void geaAudioAecStop();
|
|
9
11
|
void geaAudioAecReference(const int16_t *pcm, size_t count);
|
|
10
|
-
void geaAudioAecReceive(const int16_t *
|
|
12
|
+
void geaAudioAecReceive(const int16_t *interleaved, size_t count);
|
|
11
13
|
bool geaAudioAecRead(int16_t *pcm, size_t count);
|
|
12
14
|
size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
|
|
13
15
|
const int16_t **result);
|
|
16
|
+
|
|
17
|
+
#if GEA_AUDIO_EXPERIMENT
|
|
18
|
+
// Lab-only runtime controls. Updates are synchronized with the DSP frame.
|
|
19
|
+
void geaAudioAecExperimentDefaults(float gain, bool aggressive, unsigned microphones, bool agc);
|
|
20
|
+
bool geaAudioAecConfigure(const char *key, float value);
|
|
21
|
+
void geaAudioAecDescribe(char *buffer, size_t capacity);
|
|
22
|
+
// Bounded diagnostic capture: interleaved physical mic1, mic2, speaker reference.
|
|
23
|
+
// Read only after capture has finished/stopped; never changes published audio.
|
|
24
|
+
bool geaAudioAecCaptureRead(size_t offset, int16_t *samples, size_t capacity,
|
|
25
|
+
size_t *frames, size_t *total, int64_t *startedUs,
|
|
26
|
+
uint32_t *drops);
|
|
27
|
+
bool geaAudioExperimentConfigure(const char *key, float value);
|
|
28
|
+
void geaAudioExperimentDescribe(char *buffer, size_t capacity);
|
|
29
|
+
#endif
|