@geastack/targets 0.1.100 → 0.1.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/package.json +7 -7
  2. package/targets/cmake/geatsc-generated-archive-build.mjs +32 -1
  3. package/targets/esp32/app_project.cmake +2 -2
  4. package/targets/esp32/chip_bindings/audio/echo_cancellation.cpp +392 -28
  5. package/targets/esp32/chip_bindings/audio/echo_cancellation.h +18 -2
  6. package/targets/esp32/chip_bindings/audio/es8311.cpp +117 -13
  7. package/targets/esp32/chip_bindings/displays/co5300.cpp +68 -28
  8. package/targets/esp32/components/gea_framework/CMakeLists.txt +17 -1
  9. package/targets/esp32/components/gea_framework/idf_component.yml +4 -0
  10. package/targets/esp32/components/gea_vpx/CMakeLists.txt +1 -1
  11. package/targets/esp32/connectivity/wifi.cpp +800 -543
  12. package/targets/esp32/display.cpp +121 -26
  13. package/targets/esp32/gea_framework.cmake +22 -28
  14. package/targets/esp32/host_sources.cmake +1 -0
  15. package/targets/esp32/services/comparison_benchmark.h +137 -0
  16. package/targets/esp32/services/comparison_gesture.h +133 -0
  17. package/targets/esp32/services/comparison_input_trace.h +64 -0
  18. package/targets/esp32/services/comparison_upload_capture.h +125 -0
  19. package/targets/esp32/services/debugger.h +634 -0
  20. package/targets/esp32/services/device_control.cpp +965 -20
  21. package/targets/esp32/services/frame_scheduler.cpp +128 -4
  22. package/targets/esp32/services/heap.cpp +125 -0
  23. package/targets/esp32/services/opus_benchmark.h +1 -1
  24. package/targets/esp32/services/storage_service.cpp +119 -104
  25. package/targets/esp32-p4-m5stack-tab5/dependencies.lock +27 -16
  26. package/targets/esp32-s3-m5stack-stopwatch/.clang-format +10 -0
  27. package/targets/esp32-s3-m5stack-stopwatch/README.md +14 -9
  28. package/targets/esp32-s3-m5stack-stopwatch/dependencies.lock +54 -1
  29. package/targets/esp32-s3-m5stack-stopwatch/main/CMakeLists.txt +10 -7
  30. package/targets/esp32-s3-m5stack-stopwatch/main/buttons.cpp +80 -0
  31. package/targets/esp32-s3-m5stack-stopwatch/main/idf_component.yml +18 -0
  32. package/targets/esp32-s3-m5stack-stopwatch/main/include/board.h +38 -17
  33. package/targets/esp32-s3-m5stack-stopwatch/main/panel_reset.cpp +132 -129
  34. package/targets/esp32-s3-m5stack-stopwatch/main/power.cpp +253 -0
  35. package/targets/esp32-s3-m5stack-stopwatch/main/touch.cpp +158 -88
  36. package/targets/esp32-s3-m5stack-stopwatch/sdkconfig.defaults +13 -0
  37. package/targets/esp32-s3-touch-amoled-1.8/main/CMakeLists.txt +30 -3
  38. package/targets/esp32-s3-touch-amoled-2.06/dependencies.lock +10 -148
  39. package/targets/esp32-s31-espressif-mosaico/CMakeLists.txt +8 -0
  40. package/targets/esp32-s31-espressif-mosaico/README.md +54 -2
  41. package/targets/esp32-s31-espressif-mosaico/dependencies.lock +405 -1
  42. package/targets/esp32-s31-espressif-mosaico/main/CMakeLists.txt +31 -1
  43. package/targets/esp32-s31-espressif-mosaico/main/board_power.cpp +117 -0
  44. package/targets/esp32-s31-espressif-mosaico/main/camera.cpp +389 -0
  45. package/targets/esp32-s31-espressif-mosaico/main/idf_component.yml +17 -3
  46. package/targets/esp32-s31-espressif-mosaico/main/include/board.h +23 -0
  47. package/targets/esp32-s31-espressif-mosaico/main/nand_mount.cpp +172 -0
  48. package/targets/esp32-s31-espressif-mosaico/partitions.csv +7 -3
  49. package/targets/esp32-s31-espressif-mosaico/sdkconfig.defaults +24 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@geastack/targets",
3
- "version": "0.1.100",
3
+ "version": "0.1.102",
4
4
  "description": "Native embedded build targets (ESP32-IDF, RP2350) for geastack apps: board sources, target catalog, and CMake glue driven by the gea CLI.",
5
5
  "license": "GPL-3.0-only",
6
6
  "repository": {
@@ -37,13 +37,13 @@
37
37
  },
38
38
  "dependencies": {
39
39
  "@geastack/chips": "^0.1.6",
40
- "@geastack/compiler": "^1.0.25",
41
- "@geastack/core": "^0.1.39",
42
- "@geastack/elements": "^0.1.5",
43
- "@geastack/engine": "^0.1.15",
40
+ "@geastack/compiler": "^1.0.33",
41
+ "@geastack/core": "^0.1.40",
42
+ "@geastack/elements": "^0.1.6",
43
+ "@geastack/engine": "^0.1.17",
44
44
  "@geastack/geaos": "^0.1.3",
45
- "@geastack/geatsc-plugin-gea": "^0.1.21",
46
- "@geastack/host": "^0.1.14"
45
+ "@geastack/geatsc-plugin-gea": "^0.1.22",
46
+ "@geastack/host": "^0.1.15"
47
47
  },
48
48
  "scripts": {
49
49
  "test": "for f in $(git ls-files | grep -E '(^|/)(test[-_][^/]*|[^/]*\\.test)\\.mjs$'); do echo \"==> $f\"; node \"$f\" || exit 1; done"
@@ -244,6 +244,28 @@ function runCommand(command, args) {
244
244
  })
245
245
  }
246
246
 
247
+ // GEA_GEATSC_OPTIMIZE_SIZE_EXCEPT=<regex>: compile every generated unit whose
248
+ // path does NOT match the regex with -Os (the board's flags are -O2). An app
249
+ // whose image outgrows its flash slot keeps its hot units (the matches) fast
250
+ // and trades speed for size everywhere else.
251
+ //
252
+ // GEA_GEATSC_OPTIMIZE_SPEED=<regex>: the converse for a board built -Os: the
253
+ // generated units whose path matches are compiled -O2, so a size-bound image
254
+ // pays the larger code only for its hot units.
255
+ function sizeOptimizedFlags(sourcePath) {
256
+ const speed = process.env.GEA_GEATSC_OPTIMIZE_SPEED
257
+ // -O2's deeper inlining makes GCC's flow-based bounds warnings misfire on
258
+ // the runtime's header-prefixed allocations (subscript -2 of a payload).
259
+ if (speed && new RegExp(speed).test(sourcePath)) return ['-O2', ...(process.env.GEA_GEATSC_OPTIMIZE_SPEED_FLAGS ?? '').split(/\s+/).filter(Boolean), '-Wno-array-bounds', '-Wno-stringop-overflow', '-Wno-stringop-overread']
260
+ // GEA_GEATSC_OPTIMIZE_LEVEL=<flag>: every other generated unit at this level
261
+ // instead of the board's (e.g. -Oz where -Os does not fit the app slot).
262
+ const level = process.env.GEA_GEATSC_OPTIMIZE_LEVEL
263
+ if (level) return [level]
264
+ const except = process.env.GEA_GEATSC_OPTIMIZE_SIZE_EXCEPT
265
+ if (!except) return []
266
+ return new RegExp(except).test(sourcePath) ? [] : ['-Os']
267
+ }
268
+
247
269
  async function compileObject(record, options, compileSignature) {
248
270
  const suffix = `${process.pid}.${Math.random().toString(16).slice(2)}.tmp`
249
271
  const temporaryObject = `${record.objectPath}.${suffix}`
@@ -253,6 +275,7 @@ async function compileObject(record, options, compileSignature) {
253
275
  const compilerArgs = [
254
276
  `@${options.flagsFile}`,
255
277
  ...options.generatedCompileFlags,
278
+ ...sizeOptimizedFlags(record.sourcePath),
256
279
  '-MMD',
257
280
  '-MP',
258
281
  '-MF',
@@ -491,6 +514,11 @@ export async function buildGeneratedArchive(rawOptions) {
491
514
  generatedCompileFlags: [
492
515
  ...GENERATED_WARNING_TOLERANCE_FLAGS,
493
516
  ...(rawOptions.debugInfo === true || process.env.GEA_GEATSC_DEBUG_INFO === '1' ? [] : NO_GENERATED_DEBUG_INFO_FLAGS),
517
+ // GEA_GEATSC_EXTRA_FLAGS: extra flags for the generated units only (part
518
+ // of the compile signature through this list), e.g. an app whose image
519
+ // must fit a slot dropping -fthreadsafe-statics when its generated code
520
+ // only ever runs on one task.
521
+ ...(process.env.GEA_GEATSC_EXTRA_FLAGS ?? '').split(/\s+/).filter(Boolean),
494
522
  ],
495
523
  }
496
524
  for (const [name, filePath] of Object.entries({
@@ -551,6 +579,9 @@ export async function buildGeneratedArchive(rawOptions) {
551
579
  '-MMD',
552
580
  '-MP',
553
581
  ...options.generatedCompileFlags,
582
+ `size-except:${process.env.GEA_GEATSC_OPTIMIZE_SIZE_EXCEPT ?? ''}`,
583
+ `speed:${process.env.GEA_GEATSC_OPTIMIZE_SPEED ?? ''}:${process.env.GEA_GEATSC_OPTIMIZE_SPEED_FLAGS ?? ''}`,
584
+ `level:${process.env.GEA_GEATSC_OPTIMIZE_LEVEL ?? ''}`,
554
585
  // A new .gch means every object that baked the old one in must rebuild;
555
586
  // stat identity is stable across builds that reuse the cached PCH.
556
587
  pch ? toolIdentity(pch.gchPath) : 'no-pch',
@@ -580,7 +611,7 @@ export async function buildGeneratedArchive(rawOptions) {
580
611
  const compiled = staleRecords.length
581
612
  const reused = records.length - compiled
582
613
  process.stdout.write(
583
- `geatsc-generated-archive: compiled=${compiled} reused=${reused} jobs=${Math.min(jobs, Math.max(1, compiled))} archived=${archiveChanged ? 1 : 0} pch=${pch ? (pch.rebuilt ? 'built' : 'reused') : 'off'}\n`,
614
+ `geatsc-generated-archive: compiled=${compiled} reused=${reused} jobs=${Math.min(jobs, Math.max(1, compiled))} archived=${archiveChanged ? 1 : 0} pch=${pch ? (pch.rebuilt ? 'built' : 'reused') : 'off'} os=${records.filter((record) => sizeOptimizedFlags(record.sourcePath).length > 0).length}\n`,
584
615
  )
585
616
  return { compiled, reused, jobs, archived: archiveChanged, records, pch }
586
617
  }
@@ -30,10 +30,10 @@ endif()
30
30
  # components that belong to one capability -- the codec, the WebSocket client,
31
31
  # the WebRTC peer -- on these variables, and the component manager treats an
32
32
  # unset environment variable in an `if:` rule as a hard error rather than a
33
- # false. The gea CLI always exports all three; a direct `idf.py` build has no
33
+ # false. The gea CLI always exports these values; a direct `idf.py` build has no
34
34
  # app analysis to go on, so it defaults to keeping everything, which is what it
35
35
  # built before the rules existed.
36
- foreach(_GEA_CAPABILITY NETWORK BLE AUDIO)
36
+ foreach(_GEA_CAPABILITY NETWORK BLE AUDIO RTC H264)
37
37
  if(NOT DEFINED ENV{GEA_EMBEDDED_CAPABILITY_${_GEA_CAPABILITY}}
38
38
  OR "$ENV{GEA_EMBEDDED_CAPABILITY_${_GEA_CAPABILITY}}" STREQUAL "")
39
39
  # Incremental Ninja/CMake reconfiguration does not inherit the CLI's
@@ -1,6 +1,10 @@
1
1
  // Official Espressif ESP-SR AEC; adapter only, no custom cancellation DSP.
2
2
  #include "echo_cancellation.h"
3
3
  #include "esp_aec.h"
4
+ #include "esp_agc.h"
5
+ #if GEA_AUDIO_EXPERIMENT
6
+ #include "esp_ns.h"
7
+ #endif
4
8
  #include "esp_heap_caps.h"
5
9
  #include "esp_log.h"
6
10
  #include "esp_timer.h"
@@ -16,31 +20,149 @@
16
20
  namespace {
17
21
  constexpr uint32_t kMaxFrame = 512, kDmaFrames = 240, kCaptureRing = 1024;
18
22
  portMUX_TYPE dmaMux = portMUX_INITIALIZER_UNLOCKED;
19
- int16_t dmaMic[kCaptureRing]{}, dmaRef[kCaptureRing]{};
20
- int16_t previousMic[kDmaFrames]{}, latestTx[kDmaFrames]{}, captureRef[320]{};
23
+ int16_t dmaMic[kCaptureRing]{}, dmaMic2[kCaptureRing]{}, dmaRef[kCaptureRing]{};
24
+ int16_t previousMic[kDmaFrames]{}, latestTx[kDmaFrames]{}, captureRef[320]{}, captureMic2[320]{};
21
25
  uint32_t dmaWritten = 0, dmaRead = 0, dmaDrops = 0;
22
26
  bool dmaActive = false, previousMicReady = false;
23
27
  bool hardwareReference = false;
24
28
  float outputGain = 1.0f;
29
+ unsigned microphoneCount = 1;
30
+ bool releaseQuietReference = false;
31
+ size_t quietReferenceSamples = 0;
25
32
  std::mutex aecMutex;
26
33
  aec_handle_t *aec = nullptr;
27
- alignas(16) int16_t mic[kMaxFrame], ref[kMaxFrame], clean[kMaxFrame], linear[kMaxFrame],
28
- output[kMaxFrame * 2];
34
+ void *agc = nullptr;
35
+ constexpr size_t kAgcFrame = 160; // ESP-SR AGC requires 10 ms at 16 kHz.
36
+ constexpr int kOutputPeakLimit = 28000; // 1.37 dB below the PCM16 ceiling.
37
+ int16_t *agcInput = nullptr, *agcOutput = nullptr;
38
+ size_t agcFill = 0;
39
+ bool gainFailed = false;
40
+ #if GEA_AUDIO_EXPERIMENT
41
+ bool experimentAec = true, experimentNlp = true, experimentAgc = true;
42
+ bool experimentInitialized = false;
43
+ int experimentMic = 0, experimentNlpLevel = 0, experimentNsLevel = -1;
44
+ int experimentCompression = 12, experimentTarget = 6, experimentQuietMs = 128;
45
+ aec_config_t experimentAecConfig{};
46
+ ns_handle_t experimentNs = nullptr;
47
+ int16_t *experimentCapture = nullptr;
48
+ size_t captureCapacity = 0, captureFrames = 0;
49
+ bool captureRunning = false;
50
+ int64_t captureStartedUs = 0;
51
+ uint32_t captureDrops = 0, captureEndDrops = 0;
52
+ #endif
53
+ // Only the ISR capture ring needs internal RAM. Keep the aligned DSP scratch
54
+ // in PSRAM, leaving DMA-capable RAM available for I2S and TLS encryption.
55
+ int16_t *scratch = nullptr, *mic = nullptr, *secondMicFrame = nullptr, *ref = nullptr, *clean = nullptr,
56
+ *linear = nullptr, *output = nullptr;
29
57
  size_t fill = 0, frameSize = 0;
30
58
 
31
59
  uint64_t rawEnergy = 0, cleanEnergy = 0, linearEnergy = 0, refEnergy = 0, totalSamples = 0;
32
60
  int64_t logAt = 0, processUs = 0;
33
61
  uint32_t processedFrames = 0, clippedSamples = 0, clippedReferenceSamples = 0;
34
62
  int rawPeak = 0, cleanPeak = 0, referencePeak = 0;
63
+ uint32_t outputClipped = 0;
64
+ uint64_t secondMicEnergy = 0;
65
+ int secondMicPeak = 0;
66
+ uint64_t outputEnergy = 0, outputSamples = 0;
67
+ int outputPeak = 0;
68
+ uint32_t peakLimitedFrames = 0, agcFullScaleSamples = 0;
69
+
70
+ // AGC consumes 10 ms blocks while AEC produces 32 ms blocks. Retain the tail
71
+ // across calls, preserving sample order without padding or an extra audio task.
72
+ size_t applyOutputGain(const int16_t *samples, size_t count, int16_t *destination) {
73
+ bool useAgc = agc != nullptr;
74
+ #if GEA_AUDIO_EXPERIMENT
75
+ useAgc = useAgc && experimentAgc;
76
+ if (!useAgc && !experimentNs) {
77
+ int peak = 0;
78
+ for (size_t j = 0; j < count; ++j) peak = std::max(peak, std::abs(int(samples[j])));
79
+ const float gain = peak ? std::min(outputGain, float(kOutputPeakLimit) / peak) : outputGain;
80
+ peakLimitedFrames += gain < outputGain;
81
+ for (size_t j = 0; j < count; ++j) destination[j] = int16_t(std::lround(samples[j] * gain));
82
+ return count;
83
+ }
84
+ #else
85
+ if (!useAgc) {
86
+ for (size_t j = 0; j < count; ++j) {
87
+ const float amplified = samples[j] * outputGain;
88
+ outputClipped += amplified > 32767.0f || amplified < -32768.0f;
89
+ destination[j] = int16_t(std::lround(std::clamp(amplified, -32768.0f, 32767.0f)));
90
+ }
91
+ return count;
92
+ }
93
+ #endif
94
+ int peak = 0;
95
+ for (size_t j = 0; j < count; ++j) peak = std::max(peak, std::abs(int(samples[j])));
96
+ // Protect the AGC input BEFORE converting amplified samples to PCM16.
97
+ // Clamping first would irreversibly distort peaks before the limiter saw them.
98
+ const float gain = peak ? std::min(outputGain, float(kOutputPeakLimit) / peak) : outputGain;
99
+ peakLimitedFrames += gain < outputGain;
100
+ size_t produced = 0;
101
+ for (size_t j = 0; j < count; ++j) {
102
+ agcInput[agcFill++] = int16_t(std::lround(samples[j] * gain));
103
+ if (agcFill != kAgcFrame) continue;
104
+ #if GEA_AUDIO_EXPERIMENT
105
+ if (experimentNs) {
106
+ ns_process(experimentNs, agcInput, agcOutput);
107
+ std::memcpy(agcInput, agcOutput, kAgcFrame * sizeof(int16_t));
108
+ }
109
+ #endif
110
+ int status = 0;
111
+ if (useAgc) status = esp_agc_process(agc, agcInput, agcOutput, kAgcFrame, 16000);
112
+ else std::memcpy(agcOutput, agcInput, kAgcFrame * sizeof(int16_t));
113
+ agcFill = 0;
114
+ if (status < 0) {
115
+ ESP_LOGE("gea_aec", "Post-AEC AGC failed: %d", status);
116
+ gainFailed = true;
117
+ return 0;
118
+ }
119
+ int processedPeak = 0;
120
+ for (size_t k = 0; k < kAgcFrame; ++k) {
121
+ processedPeak = std::max(processedPeak, std::abs(int(agcOutput[k])));
122
+ agcFullScaleSamples += agcOutput[k] == 32767 || agcOutput[k] == -32768;
123
+ }
124
+ // A final uniform block attenuation preserves waveform shape and leaves
125
+ // headroom for transport resampling, including negative full-scale samples.
126
+ const float protection = processedPeak > kOutputPeakLimit
127
+ ? float(kOutputPeakLimit) / processedPeak : 1.0f;
128
+ peakLimitedFrames += protection < 1.0f;
129
+ for (size_t k = 0; k < kAgcFrame; ++k)
130
+ destination[produced + k] = int16_t(std::lround(agcOutput[k] * protection));
131
+ produced += kAgcFrame;
132
+ }
133
+ return produced;
134
+ }
35
135
  } // namespace
36
136
 
37
- bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp) {
137
+ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp, unsigned inputMicrophones, bool adaptiveGain) {
38
138
  std::lock_guard<std::mutex> lock(aecMutex);
39
139
  if (aec)
40
140
  return true;
41
- if (!std::isfinite(gain) || gain <= 0)
141
+ if (!std::isfinite(gain) || gain <= 0 || inputMicrophones < 1 || inputMicrophones > 2 ||
142
+ (inputMicrophones == 2 && !useHardwareReference))
143
+ return false;
144
+ scratch = static_cast<int16_t *>(heap_caps_aligned_alloc(
145
+ 16, (7 * kMaxFrame + 2 * kAgcFrame) * sizeof(int16_t), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
146
+ if (!scratch) {
147
+ ESP_LOGE("gea_aec", "AEC processing buffer allocation failed");
42
148
  return false;
149
+ }
150
+ mic = scratch;
151
+ secondMicFrame = mic + kMaxFrame;
152
+ ref = secondMicFrame + kMaxFrame;
153
+ clean = ref + kMaxFrame;
154
+ linear = clean + kMaxFrame;
155
+ output = linear + kMaxFrame;
156
+ #if !GEA_AUDIO_EXPERIMENT
43
157
  outputGain = gain;
158
+ #endif
159
+ microphoneCount = inputMicrophones;
160
+ #if GEA_AUDIO_EXPERIMENT
161
+ releaseQuietReference = experimentNlpLevel == 0;
162
+ #else
163
+ releaseQuietReference = !aggressiveNlp;
164
+ #endif
165
+ quietReferenceSamples = 0;
44
166
  aec_config_t config{};
45
167
  config.mic_num = 1;
46
168
  config.ref_num = 1;
@@ -51,9 +173,15 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
51
173
  config.mode = AEC_MODE_FD_HIGH_PERF;
52
174
  // Select residual echo suppression independently of the output gain.
53
175
  config.nlp_level = aggressiveNlp ? AEC_NLP_LEVEL_AGGR : AEC_NLP_LEVEL_NORMAL;
176
+ #if GEA_AUDIO_EXPERIMENT
177
+ config.nlp_level = static_cast<aec_nlp_level_t>(experimentNlpLevel);
178
+ experimentAecConfig = config;
179
+ #endif
54
180
  aec = aec_create_from_config(&config);
55
181
  if (!aec) {
56
182
  ESP_LOGE("gea_aec", "ESP-SR AEC allocation failed");
183
+ heap_caps_free(scratch);
184
+ scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
57
185
  return false;
58
186
  }
59
187
  frameSize = aec_get_chunksize(aec);
@@ -61,8 +189,46 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
61
189
  ESP_LOGE("gea_aec", "Unsupported AEC frame size %u", unsigned(frameSize));
62
190
  aec_destroy(aec);
63
191
  aec = nullptr;
192
+ heap_caps_free(scratch);
193
+ scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
64
194
  return false;
65
195
  }
196
+ agcInput = output + 2 * kMaxFrame;
197
+ agcOutput = agcInput + kAgcFrame;
198
+ if (adaptiveGain) {
199
+ agc = esp_agc_open(AGC_MODE_2, 16000);
200
+ if (!agc) {
201
+ ESP_LOGE("gea_aec", "Post-AEC AGC allocation failed");
202
+ aec_destroy(aec);
203
+ aec = nullptr;
204
+ heap_caps_free(scratch);
205
+ scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
206
+ agcInput = agcOutput = nullptr;
207
+ return false;
208
+ }
209
+ // 12 dB compression gain, built-in limiter, -6 dBFS target.
210
+ #if GEA_AUDIO_EXPERIMENT
211
+ set_agc_config(agc, experimentCompression, 1, experimentTarget);
212
+ #else
213
+ set_agc_config(agc, 12, 1, 6);
214
+ #endif
215
+ }
216
+ #if GEA_AUDIO_EXPERIMENT
217
+ if (!experimentNs && experimentNsLevel >= 0) {
218
+ experimentNs = ns_pro_create(10, experimentNsLevel, 16000);
219
+ if (!experimentNs) {
220
+ if (agc) esp_agc_close(agc);
221
+ agc = nullptr;
222
+ aec_destroy(aec); aec = nullptr;
223
+ heap_caps_free(scratch);
224
+ scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
225
+ agcInput = agcOutput = nullptr;
226
+ return false;
227
+ }
228
+ }
229
+ #endif
230
+ agcFill = 0;
231
+ gainFailed = false;
66
232
  fill = 0;
67
233
  portENTER_CRITICAL(&dmaMux);
68
234
  dmaWritten = dmaRead = dmaDrops = 0;
@@ -74,18 +240,32 @@ bool geaAudioAecStart(bool useHardwareReference, float gain, bool aggressiveNlp)
74
240
  rawEnergy = cleanEnergy = linearEnergy = refEnergy = totalSamples = 0;
75
241
  processUs = processedFrames = clippedSamples = clippedReferenceSamples = 0;
76
242
  rawPeak = cleanPeak = referencePeak = 0;
243
+ outputClipped = 0;
244
+ secondMicEnergy = 0;
245
+ secondMicPeak = 0;
246
+ outputEnergy = outputSamples = 0;
247
+ outputPeak = 0;
248
+ peakLimitedFrames = agcFullScaleSamples = 0;
77
249
  logAt = esp_timer_get_time();
78
250
  ESP_LOGI("gea_aec",
79
- "ESP-SR full-duplex high-performance AEC ready: rate=16000 frame=%u NLP=%s "
80
- "reference=%s internal_free=%u",
251
+ "ESP-SR AEC ready: rate=16000 frame=%u NLP=%s "
252
+ "physical_mics=%u aec_channels=1 reference=%s adaptive_gain=%s internal_free=%u",
81
253
  unsigned(frameSize), aggressiveNlp ? "aggressive" : "normal",
82
- hardwareReference ? "ADC" : "TX-DMA",
254
+ microphoneCount, hardwareReference ? "ADC" : "TX-DMA", agc ? "digital+limiter" : "off",
83
255
  unsigned(heap_caps_get_free_size(MALLOC_CAP_INTERNAL)));
84
256
  return true;
85
257
  }
86
258
 
87
259
  void geaAudioAecStop() {
88
260
  std::lock_guard<std::mutex> lock(aecMutex);
261
+ #if GEA_AUDIO_EXPERIMENT
262
+ if (captureRunning) {
263
+ portENTER_CRITICAL(&dmaMux);
264
+ captureEndDrops = dmaDrops;
265
+ portEXIT_CRITICAL(&dmaMux);
266
+ }
267
+ captureRunning = false; // Retain the bounded recording for silent USB export.
268
+ #endif
89
269
  portENTER_CRITICAL(&dmaMux);
90
270
  dmaActive = false;
91
271
  portEXIT_CRITICAL(&dmaMux);
@@ -93,6 +273,18 @@ void geaAudioAecStop() {
93
273
  aec_destroy(aec);
94
274
  aec = nullptr;
95
275
  }
276
+ #if GEA_AUDIO_EXPERIMENT
277
+ if (experimentNs) ns_destroy(experimentNs);
278
+ experimentNs = nullptr;
279
+
280
+ #endif
281
+ if (agc) esp_agc_close(agc);
282
+ agc = nullptr;
283
+ agcInput = agcOutput = nullptr;
284
+ agcFill = 0;
285
+ gainFailed = false;
286
+ heap_caps_free(scratch);
287
+ scratch = mic = secondMicFrame = ref = clean = linear = output = nullptr;
96
288
  fill = 0;
97
289
  }
98
290
 
@@ -110,7 +302,8 @@ void geaAudioAecReference(const int16_t *pcm, size_t count) {
110
302
  }
111
303
  void geaAudioAecReceive(const int16_t *stereo, size_t count) {
112
304
  portENTER_CRITICAL_ISR(&dmaMux);
113
- if (dmaActive && count == kDmaFrames * 2) {
305
+ const size_t stride = hardwareReference ? microphoneCount + 1 : 2;
306
+ if (dmaActive && count == kDmaFrames * stride) {
114
307
  if (hardwareReference) {
115
308
  // ES7210 analog loopback or ES8311 ADCL + DACR arrive in the same RX
116
309
  // frame. Keep them paired across callback/task scheduling jitter.
@@ -119,8 +312,11 @@ void geaAudioAecReceive(const int16_t *stereo, size_t count) {
119
312
  ++dmaDrops;
120
313
  }
121
314
  for (size_t i = 0; i < kDmaFrames; ++i) {
122
- dmaMic[(dmaWritten + i) % kCaptureRing] = stereo[i * 2];
123
- dmaRef[(dmaWritten + i) % kCaptureRing] = stereo[i * 2 + 1];
315
+ // ES7210 slots are MIC1, speaker reference, MIC2.
316
+ dmaMic[(dmaWritten + i) % kCaptureRing] = stereo[i * stride];
317
+ dmaRef[(dmaWritten + i) % kCaptureRing] = stereo[i * stride + 1];
318
+ if (microphoneCount == 2)
319
+ dmaMic2[(dmaWritten + i) % kCaptureRing] = stereo[i * stride + 2];
124
320
  }
125
321
  dmaWritten += kDmaFrames;
126
322
  portEXIT_CRITICAL_ISR(&dmaMux);
@@ -152,6 +348,8 @@ bool geaAudioAecRead(int16_t *pcm, size_t count) {
152
348
  for (size_t i = 0; i < count; ++i) {
153
349
  pcm[i] = dmaMic[(dmaRead + i) % kCaptureRing];
154
350
  captureRef[i] = dmaRef[(dmaRead + i) % kCaptureRing];
351
+ if (microphoneCount == 2)
352
+ captureMic2[i] = dmaMic2[(dmaRead + i) % kCaptureRing];
155
353
  }
156
354
  dmaRead += count;
157
355
  }
@@ -163,13 +361,37 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
163
361
  const int16_t **result) {
164
362
  std::lock_guard<std::mutex> lock(aecMutex);
165
363
  *result = output;
166
- if (!aec)
364
+ if (!aec || gainFailed)
167
365
  return 0; // Never silently forward uncancelled audio after an AEC failure.
168
366
  size_t produced = 0;
169
367
  for (size_t i = 0; i < count; ++i) {
170
- mic[fill] = pcm[i];
368
+ #if GEA_AUDIO_EXPERIMENT
369
+ if (captureRunning && captureFrames < captureCapacity) {
370
+ if (!captureFrames) captureStartedUs = esp_timer_get_time();
371
+ experimentCapture[captureFrames * 3] = pcm[i];
372
+ experimentCapture[captureFrames * 3 + 1] = microphoneCount == 2 ? captureMic2[i] : 0;
373
+ experimentCapture[captureFrames * 3 + 2] = captureRef[i];
374
+ if (++captureFrames == captureCapacity) {
375
+ captureRunning = false;
376
+ portENTER_CRITICAL(&dmaMux);
377
+ captureEndDrops = dmaDrops;
378
+ portEXIT_CRITICAL(&dmaMux);
379
+ }
380
+ }
381
+ #endif
382
+ // Both microphones are synchronized by the same ADC/I2S frame. Mix
383
+ // before cancellation so one AEC models their combined speaker echo path.
384
+ // Averaging avoids doubling amplitude and retains the single-channel CPU cost.
385
+ mic[fill] = microphoneCount == 2
386
+ ? int16_t((int32_t(pcm[i]) + captureMic2[i]) / 2) : pcm[i];
387
+ #if GEA_AUDIO_EXPERIMENT
388
+ if (microphoneCount == 2 && experimentMic == 1) mic[fill] = pcm[i];
389
+ if (microphoneCount == 2 && experimentMic == 2) mic[fill] = captureMic2[i];
390
+ #endif
391
+ if (microphoneCount == 2) secondMicFrame[fill] = captureMic2[i];
171
392
  ref[fill] = captureRef[i];
172
393
  if (++fill == frameSize) {
394
+ uint64_t frameReferenceEnergy = 0;
173
395
  for (size_t j = 0; j < frameSize; ++j) {
174
396
  rawPeak = std::max(rawPeak, std::abs(int(mic[j])));
175
397
  referencePeak = std::max(referencePeak, std::abs(int(ref[j])));
@@ -177,15 +399,29 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
177
399
  clippedReferenceSamples += std::abs(int(ref[j])) >= 32760;
178
400
  rawEnergy += int64_t(mic[j]) * mic[j];
179
401
  refEnergy += int64_t(ref[j]) * ref[j];
402
+ frameReferenceEnergy += int64_t(ref[j]) * ref[j];
403
+ if (microphoneCount == 2) {
404
+ const int16_t second = secondMicFrame[j];
405
+ secondMicEnergy += int64_t(second) * second;
406
+ secondMicPeak = std::max(secondMicPeak, std::abs(int(second)));
407
+ }
180
408
  }
181
409
  auto before = esp_timer_get_time();
182
410
  // ESP-SR's documented two-stage API preserves both cancellation stages.
183
411
  // Snapshot the linear result so telemetry can identify suppression of
184
412
  // near-end speech separately from cancellation of the speaker echo.
413
+ #if GEA_AUDIO_EXPERIMENT
414
+ if (experimentAec) aec_linear_process(aec, mic, ref, clean);
415
+ else std::memcpy(clean, mic, frameSize * sizeof(int16_t));
416
+ #else
185
417
  aec_linear_process(aec, mic, ref, clean);
418
+ #endif
186
419
  std::memcpy(linear, clean, frameSize * sizeof(int16_t));
420
+ #if GEA_AUDIO_EXPERIMENT
421
+ if (experimentAec && experimentNlp) aec_nlp_process(aec, clean);
422
+ #else
187
423
  aec_nlp_process(aec, clean);
188
- processUs += esp_timer_get_time() - before;
424
+ #endif
189
425
  ++processedFrames;
190
426
  for (size_t j = 0; j < frameSize; ++j) {
191
427
  linearEnergy += int64_t(linear[j]) * linear[j];
@@ -193,25 +429,37 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
193
429
  cleanPeak = std::max(cleanPeak, std::abs(int(clean[j])));
194
430
  }
195
431
  totalSamples += frameSize;
196
- // Apply the board's calibrated voice gain after AEC, keeping the ADC
197
- // unclipped. Unity-gain boards retain the direct copy. Never gate speech.
198
- if (outputGain == 1.0f) {
199
- std::memcpy(output + produced, clean, frameSize * sizeof(int16_t));
200
- } else {
201
- for (size_t j = 0; j < frameSize; ++j) {
202
- const float amplified = clean[j] * outputGain;
203
- output[produced + j] = int16_t(std::lround(std::clamp(amplified, -32768.0f, 32767.0f)));
204
- }
205
- }
206
- produced += frameSize;
432
+ // Keep NLP trained, but release its hangover once playback has been
433
+ // quiet for 128 ms. A whole quiet interval covers the linear filter's
434
+ // output delay and acoustic tail; one quiet frame inside a word does not.
435
+ // Linear AEC remains active and removes residual room/DMA echo.
436
+ #if GEA_AUDIO_EXPERIMENT
437
+ const size_t kQuietReferenceSamples = size_t(experimentQuietMs) * 16;
438
+ #else
439
+ constexpr size_t kQuietReferenceSamples = 2048;
440
+ #endif
441
+ quietReferenceSamples = frameReferenceEnergy < uint64_t(128 * 128) * frameSize
442
+ ? std::min(kQuietReferenceSamples, quietReferenceSamples + frameSize) : 0;
443
+ const bool released = releaseQuietReference && kQuietReferenceSamples > 0 && quietReferenceSamples >= kQuietReferenceSamples;
444
+ const size_t gained = applyOutputGain(released ? linear : clean,
445
+ frameSize, output + produced);
207
446
  fill = 0;
447
+ if (gainFailed) return 0;
448
+ for (size_t j = 0; j < gained; ++j) {
449
+ const int16_t value = output[produced + j];
450
+ outputEnergy += int64_t(value) * value;
451
+ outputPeak = std::max(outputPeak, std::abs(int(value)));
452
+ }
453
+ outputSamples += gained;
454
+ produced += gained;
455
+ processUs += esp_timer_get_time() - before;
208
456
  }
209
457
  }
210
458
  const auto now = esp_timer_get_time();
211
459
  if (now - logAt >= 1000000 && totalSamples) {
212
460
  ESP_LOGI("gea_aec",
213
461
  "raw_rms=%.0f clean_rms=%.0f ref_rms=%.0f linear_rms=%.0f reduction_db=%.1f "
214
- "process_us=%lld dma_drops=%lu clipped=%lu ref_clipped=%lu raw_peak=%d clean_peak=%d ref_peak=%d",
462
+ "process_us=%lld dma_drops=%lu clipped=%lu ref_clipped=%lu raw_peak=%d clean_peak=%d ref_peak=%d output_clipped=%lu output_clipped_pct=%.4f output_gain=%.3f output_rms=%.0f output_peak=%d peak_limited_frames=%lu agc_full_scale_samples=%lu mic2_rms=%.0f mic2_peak=%d",
215
463
  std::sqrt(double(rawEnergy) / totalSamples),
216
464
  std::sqrt(double(cleanEnergy) / totalSamples),
217
465
  std::sqrt(double(refEnergy) / totalSamples),
@@ -219,11 +467,127 @@ size_t geaAudioAecProcess(const int16_t *pcm, size_t count,
219
467
  10 * std::log10(double(rawEnergy + 1) / double(cleanEnergy + 1)),
220
468
  (long long)(processUs / std::max<uint32_t>(1, processedFrames)),
221
469
  (unsigned long)dmaDrops, (unsigned long)clippedSamples,
222
- (unsigned long)clippedReferenceSamples, rawPeak, cleanPeak, referencePeak);
470
+ (unsigned long)clippedReferenceSamples, rawPeak, cleanPeak, referencePeak,
471
+ (unsigned long)outputClipped, 100.0 * outputClipped / totalSamples, outputGain,
472
+ std::sqrt(double(outputEnergy) / std::max<uint64_t>(1, outputSamples)), outputPeak,
473
+ (unsigned long)peakLimitedFrames, (unsigned long)agcFullScaleSamples,
474
+ std::sqrt(double(secondMicEnergy) / totalSamples), secondMicPeak);
223
475
  rawEnergy = cleanEnergy = linearEnergy = refEnergy = totalSamples = 0;
224
476
  rawPeak = cleanPeak = referencePeak = 0;
477
+ outputClipped = 0;
478
+ secondMicEnergy = 0;
479
+ secondMicPeak = 0;
480
+ outputEnergy = outputSamples = 0;
481
+ outputPeak = 0;
482
+ peakLimitedFrames = agcFullScaleSamples = 0;
225
483
  processUs = processedFrames = clippedSamples = clippedReferenceSamples = 0;
226
484
  logAt = now;
227
485
  }
228
486
  return produced;
229
487
  }
488
+
489
+ #if GEA_AUDIO_EXPERIMENT
490
+ void geaAudioAecExperimentDefaults(float gain, bool aggressive, unsigned microphones, bool adaptive) {
491
+ std::lock_guard<std::mutex> lock(aecMutex);
492
+ if (experimentInitialized) return;
493
+ experimentInitialized = true;
494
+ outputGain = gain;
495
+ experimentNlpLevel = aggressive ? 1 : 0;
496
+ experimentAgc = adaptive;
497
+ microphoneCount = microphones;
498
+ }
499
+ bool geaAudioAecConfigure(const char *key, float value) {
500
+ if (!key || !std::isfinite(value) || value < -1 || value > 1000) return false;
501
+ std::lock_guard<std::mutex> lock(aecMutex);
502
+ const int number = int(value);
503
+ const bool integer = value == float(number);
504
+ if (!std::strcmp(key, "capture") && integer && number >= 0 && number <= 8) {
505
+ if (!number) {
506
+ if (captureRunning) {
507
+ portENTER_CRITICAL(&dmaMux);
508
+ captureEndDrops = dmaDrops;
509
+ portEXIT_CRITICAL(&dmaMux);
510
+ }
511
+ captureRunning = false;
512
+ return true;
513
+ }
514
+ if (!aec || captureRunning) return false;
515
+ const size_t capacity = size_t(number) * 16000;
516
+ auto *replacement = static_cast<int16_t *>(heap_caps_malloc(capacity * 3 * sizeof(int16_t),
517
+ MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT));
518
+ if (!replacement) return false;
519
+ heap_caps_free(experimentCapture);
520
+ experimentCapture = replacement;
521
+ captureCapacity = capacity;
522
+ captureFrames = 0;
523
+ captureStartedUs = 0;
524
+ portENTER_CRITICAL(&dmaMux);
525
+ captureDrops = captureEndDrops = dmaDrops;
526
+ portEXIT_CRITICAL(&dmaMux);
527
+ captureRunning = true;
528
+ return true;
529
+ }
530
+ if (!std::strcmp(key, "gain") && value >= .01f && value <= 256) outputGain = value;
531
+ else if (!std::strcmp(key, "aec") && integer && number >= 0 && number <= 1) experimentAec = number;
532
+ else if (!std::strcmp(key, "nlp") && integer && number >= 0 && number <= 1) experimentNlp = number;
533
+ else if (!std::strcmp(key, "mic") && integer && number >= 0 && number <= 2) {
534
+ if (number == 2 && microphoneCount != 2) return false;
535
+ experimentMic = number;
536
+ } else if (!std::strcmp(key, "agc") && integer && number >= 0 && number <= 1) {
537
+ if (number && !agc && aec) return false;
538
+ experimentAgc = number;
539
+ } else if (!std::strcmp(key, "agc_db") && integer && number >= 0 && number <= 30) {
540
+ experimentCompression = number;
541
+ if (agc) set_agc_config(agc, experimentCompression, 1, experimentTarget);
542
+ } else if (!std::strcmp(key, "agc_target") && integer && number >= 1 && number <= 20) {
543
+ experimentTarget = number;
544
+ if (agc) set_agc_config(agc, experimentCompression, 1, experimentTarget);
545
+ } else if (!std::strcmp(key, "quiet_ms") && integer && number >= 0 && number <= 1000) experimentQuietMs = number;
546
+ else if (!std::strcmp(key, "nlp_level") && integer && number >= 0 && number <= 2) {
547
+ if (number == experimentNlpLevel) return true;
548
+ if (aec) {
549
+ auto config = experimentAecConfig;
550
+ config.nlp_level = static_cast<aec_nlp_level_t>(number);
551
+ auto *replacement = aec_create_from_config(&config);
552
+ if (!replacement) return false;
553
+ if (size_t(aec_get_chunksize(replacement)) != frameSize) { aec_destroy(replacement); return false; }
554
+ aec_destroy(aec);
555
+ aec = replacement;
556
+ experimentAecConfig = config;
557
+ fill = 0;
558
+ }
559
+ experimentNlpLevel = number;
560
+ releaseQuietReference = number == 0;
561
+ } else if (!std::strcmp(key, "ns") && integer && number >= -1 && number <= 2) {
562
+ if (number == experimentNsLevel) return true;
563
+ ns_handle_t replacement = number < 0 ? nullptr : ns_pro_create(10, number, 16000);
564
+ if (number >= 0 && !replacement) return false;
565
+ if (experimentNs) ns_destroy(experimentNs);
566
+ experimentNs = replacement;
567
+ experimentNsLevel = number;
568
+ } else return false;
569
+ agcFill = 0;
570
+ quietReferenceSamples = 0;
571
+ return true;
572
+ }
573
+ bool geaAudioAecCaptureRead(size_t offset, int16_t *samples, size_t capacity,
574
+ size_t *frames, size_t *total, int64_t *startedUs,
575
+ uint32_t *drops) {
576
+ std::lock_guard<std::mutex> lock(aecMutex);
577
+ if (captureRunning || !experimentCapture || offset > captureFrames) return false;
578
+ *frames = std::min(capacity, captureFrames - offset);
579
+ *total = captureFrames;
580
+ *startedUs = captureStartedUs;
581
+ *drops = captureEndDrops - captureDrops;
582
+ std::memcpy(samples, experimentCapture + offset * 3, *frames * 3 * sizeof(int16_t));
583
+ return true;
584
+ }
585
+ void geaAudioAecDescribe(char *buffer, size_t capacity) {
586
+ std::lock_guard<std::mutex> lock(aecMutex);
587
+ std::snprintf(buffer, capacity,
588
+ "aec=%d nlp=%d nlp_level=%d gain=%.3f agc=%d agc_db=%d agc_target=%d ns=%d mic=%d physical_mics=%u quiet_ms=%d ready=%d",
589
+ experimentAec, experimentNlp, experimentNlpLevel, outputGain, experimentAgc,
590
+ experimentCompression, experimentTarget, experimentNsLevel, experimentMic,
591
+ microphoneCount, experimentQuietMs, aec != nullptr);
592
+ }
593
+ #endif