@geastack/targets 0.1.86 → 0.1.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/targets/esp32/app_project.cmake +18 -1
- package/targets/esp32/chip_bindings/audio/echo_cancellation.cpp +213 -0
- package/targets/esp32/chip_bindings/audio/echo_cancellation.h +12 -0
- package/targets/esp32/chip_bindings/audio/es8311.cpp +280 -41
- package/targets/esp32/chip_bindings/displays/rgb_panel.cpp +115 -12
- package/targets/esp32/chip_bindings/expanders/io_expander.h +2 -0
- package/targets/esp32/chip_bindings/expanders/tca9554.cpp +74 -0
- package/targets/esp32/chip_bindings/touch/ft3168.cpp +23 -1
- package/targets/esp32/chip_bindings/touch/ft3168_controller.h +1 -0
- package/targets/esp32/chip_bindings/touch/gt911.cpp +4 -3
- package/targets/esp32/components/gea_framework/CMakeLists.txt +8 -1
- package/targets/esp32/components/gea_juice/CMakeLists.txt +46 -0
- package/targets/esp32/components/gea_juice/README.md +20 -0
- package/targets/esp32/components/gea_juice/juice_memory.c +19 -0
- package/targets/esp32/components/gea_juice/juice_memory.h +16 -0
- package/targets/esp32/components/gea_juice/patch_threads.py +50 -0
- package/targets/esp32/components/gea_vp8_peer/CMakeLists.txt +16 -0
- package/targets/esp32/components/gea_vp8_peer/README.md +39 -0
- package/targets/esp32/components/gea_vp8_peer/gea_vp8_peer.h +21 -0
- package/targets/esp32/components/gea_vp8_peer/vp8_peer.cpp +538 -0
- package/targets/esp32/components/gea_vpx/CMakeLists.txt +235 -0
- package/targets/esp32/components/gea_vpx/README.md +133 -0
- package/targets/esp32/components/gea_vpx/i420_rgb565_s3.S +169 -0
- package/targets/esp32/components/gea_vpx/vp8_blocks_s3.c +42 -0
- package/targets/esp32/components/gea_vpx/vp8_copy_s3.S +39 -0
- package/targets/esp32/components/gea_vpx/vp8_entropy_s3.c +76 -0
- package/targets/esp32/components/gea_vpx/vp8_filter_s3.S +58 -0
- package/targets/esp32/components/gea_vpx/vp8_idct_s3.S +269 -0
- package/targets/esp32/components/gea_vpx/vp8_intrapred_s3.S +65 -0
- package/targets/esp32/components/gea_vpx/vp8_loopfilter_s3.S +333 -0
- package/targets/esp32/components/gea_vpx/vp8_loopfilter_s3.c +62 -0
- package/targets/esp32/components/gea_vpx/vp8_profile_s3.c +22 -0
- package/targets/esp32/components/gea_vpx/vp8_profile_s3.h +20 -0
- package/targets/esp32/components/gea_vpx/vp8_s3.c +120 -0
- package/targets/esp32/components/gea_vpx/vp8_s3.h +58 -0
- package/targets/esp32/components/gea_vpx/vp8_workspace_s3.c +14 -0
- package/targets/esp32/components/gea_vpx/vpx_memory.c +41 -0
- package/targets/esp32/connectivity/diagnostics_transport.cpp +18 -0
- package/targets/esp32/connectivity/wifi.cpp +26 -13
- package/targets/esp32/crypto/srtp_hmac_psa.c +118 -0
- package/targets/esp32/display.cpp +13 -0
- package/targets/esp32/gea_framework.cmake +7 -0
- package/targets/esp32/host_sources.cmake +2 -0
- package/targets/esp32/i2c_completion_order.cmake +57 -0
- package/targets/esp32/patch_streaming_dependencies.py +233 -0
- package/targets/esp32/services/device_control.cpp +141 -5
- package/targets/esp32/services/frame_scheduler.cpp +29 -1
- package/targets/esp32/services/opus_benchmark.h +82 -0
- package/targets/esp32/services/pcm_simd_check.h +73 -0
- package/targets/esp32/services/srtp_benchmark.h +106 -0
- package/targets/esp32/services/video_simd_benchmark.h +477 -0
- package/targets/esp32/services/vp8_benchmark.h +147 -0
- package/targets/esp32/streaming_dependency_patches.cmake +103 -0
- package/targets/esp32-m5stack-m5paper/main/idf_component.yml +2 -0
- package/targets/esp32-p4-m5stack-tab5/main/idf_component.yml +2 -0
- package/targets/esp32-p4-waveshare-touch-lcd-3.5/main/idf_component.yml +2 -0
- package/targets/esp32-p4-waveshare-touch-lcd-7/main/idf_component.yml +2 -0
- package/targets/esp32-s3-elecrow-rotary-2.1/main/idf_component.yml +2 -0
- package/targets/esp32-s3-epaper-1.54/main/idf_component.yml +2 -0
- package/targets/esp32-s3-lilygo-t-display-s3-long/main/idf_component.yml +2 -0
- package/targets/esp32-s3-m5stack-papers3/main/idf_component.yml +2 -0
- package/targets/esp32-s3-m5stack-sticks3/main/idf_component.yml +2 -0
- package/targets/esp32-s3-touch-amoled-1.75/main/idf_component.yml +2 -0
- package/targets/esp32-s3-touch-amoled-1.8/dependencies.lock +110 -2
- package/targets/esp32-s3-touch-amoled-1.8/main/CMakeLists.txt +29 -3
- package/targets/esp32-s3-touch-amoled-1.8/main/app_main.cpp +6 -3
- package/targets/esp32-s3-touch-amoled-1.8/main/idf_component.yml +11 -0
- package/targets/esp32-s3-touch-amoled-1.8/sdkconfig.defaults +9 -1
- package/targets/esp32-s3-touch-amoled-2.06/CMakeLists.txt +2 -0
- package/targets/esp32-s3-touch-amoled-2.06/dependencies.lock +171 -8
- package/targets/esp32-s3-touch-amoled-2.06/main/CMakeLists.txt +9 -2
- package/targets/esp32-s3-touch-amoled-2.06/main/app_main.cpp +10 -3
- package/targets/esp32-s3-touch-amoled-2.06/main/idf_component.yml +16 -1
- package/targets/esp32-s3-touch-amoled-2.06/main/include/board.h +3 -0
- package/targets/esp32-s3-touch-amoled-2.06/main/touch.cpp +7 -0
- package/targets/esp32-s3-touch-amoled-2.41/dependencies.lock +57 -1
- package/targets/esp32-s3-touch-amoled-2.41/main/idf_component.yml +10 -0
- package/targets/esp32-s3-touch-lcd-3.5b/dependencies.lock +12 -1
- package/targets/esp32-s3-touch-lcd-3.5b/main/CMakeLists.txt +5 -1
- package/targets/esp32-s3-touch-lcd-3.5b/main/idf_component.yml +2 -1
- package/targets/esp32-s3-touch-lcd-3.5b/main/include/board.h +3 -0
- package/targets/esp32-s3-touch-lcd-3.5b/main/platform_stubs.cpp +1 -18
- package/targets/esp32-s3-touch-lcd-3.5b/main/speaker_power.cpp +27 -0
- package/targets/esp32-s3-waveshare-touch-lcd-4b.json +504 -0
- package/targets.json +12 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#pragma once
|
|
3
|
+
#include "esp_peer.h"
|
|
4
|
+
#include <atomic>
|
|
5
|
+
|
|
6
|
+
namespace gea::host::rtc::vp8 { struct EncodedFrame; }
|
|
7
|
+
|
|
8
|
+
// This value is local to this implementation, never passed to peer_default.
|
|
9
|
+
inline constexpr auto GEA_PEER_VIDEO_CODEC_VP8 = static_cast<esp_peer_video_codec_t>(3);
|
|
10
|
+
|
|
11
|
+
struct GeaVp8PeerOptions {
|
|
12
|
+
std::atomic<bool>* running = nullptr;
|
|
13
|
+
std::atomic<bool>* requestKeyframe = nullptr;
|
|
14
|
+
void (*wake)(void*) = nullptr;
|
|
15
|
+
void* wakeContext = nullptr;
|
|
16
|
+
// Optional ownership transfer for the native decoder worker. The legacy
|
|
17
|
+
// esp_peer callback borrows a pointer and therefore requires a frame copy.
|
|
18
|
+
int (*onOwnedVideo)(gea::host::rtc::vp8::EncodedFrame&&, void*) = nullptr;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const esp_peer_ops_t* gea_vp8_peer_impl();
|
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
#include "gea_vp8_peer.h"
|
|
3
|
+
#include "host/rtc_receive_sdp.h"
|
|
4
|
+
#include "host/rtc_vp8_receiver.h"
|
|
5
|
+
#include "esp_heap_caps.h"
|
|
6
|
+
#include "esp_log.h"
|
|
7
|
+
#include "esp_random.h"
|
|
8
|
+
#include "esp_timer.h"
|
|
9
|
+
#include "freertos/FreeRTOS.h"
|
|
10
|
+
#include "freertos/task.h"
|
|
11
|
+
#include "juice/juice.h"
|
|
12
|
+
#include "mbedtls/version.h"
|
|
13
|
+
#include "mbedtls/md.h"
|
|
14
|
+
#include "dtls_srtp.h"
|
|
15
|
+
#include <array>
|
|
16
|
+
#include <atomic>
|
|
17
|
+
#include <cstring>
|
|
18
|
+
#include <mutex>
|
|
19
|
+
#include <new>
|
|
20
|
+
#include <optional>
|
|
21
|
+
#include <sys/time.h>
|
|
22
|
+
|
|
23
|
+
namespace {
|
|
24
|
+
using namespace gea::host::rtc;
|
|
25
|
+
constexpr const char* tag = "gea_vp8_peer";
|
|
26
|
+
uint64_t nowMs() { return uint64_t(esp_timer_get_time()) / 1000; }
|
|
27
|
+
|
|
28
|
+
struct Datagram {
|
|
29
|
+
std::array<uint8_t, 4096> bytes;
|
|
30
|
+
size_t size = 0;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
// Embedded in the PSRAM-allocated peer. The ICE callback only copies bytes;
|
|
34
|
+
// crypto, packet assembly and media callbacks run on Gea's peer worker.
|
|
35
|
+
template<size_t Capacity>
|
|
36
|
+
struct Queue {
|
|
37
|
+
std::array<Datagram, Capacity> packets;
|
|
38
|
+
size_t read = 0, size = 0;
|
|
39
|
+
std::mutex mutex;
|
|
40
|
+
|
|
41
|
+
bool push(const char* bytes, size_t length) {
|
|
42
|
+
if (!length || length > packets[0].bytes.size()) return false;
|
|
43
|
+
std::lock_guard lock(mutex);
|
|
44
|
+
if (size == Capacity) return false;
|
|
45
|
+
auto& packet = packets[(read + size++) % Capacity];
|
|
46
|
+
packet.size = length;
|
|
47
|
+
std::memcpy(packet.bytes.data(), bytes, length);
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
bool pop(Datagram& result) {
|
|
52
|
+
std::lock_guard lock(mutex);
|
|
53
|
+
if (!size) return false;
|
|
54
|
+
auto& packet = packets[read];
|
|
55
|
+
result.size = packet.size;
|
|
56
|
+
std::memcpy(result.bytes.data(), packet.bytes.data(), result.size);
|
|
57
|
+
read = (read + 1) % Capacity;
|
|
58
|
+
--size;
|
|
59
|
+
return true;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
bool empty() {
|
|
63
|
+
std::lock_guard lock(mutex);
|
|
64
|
+
return !size;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
struct Peer {
|
|
69
|
+
esp_peer_cfg_t cfg{};
|
|
70
|
+
GeaVp8PeerOptions options;
|
|
71
|
+
juice_agent_t* ice = nullptr;
|
|
72
|
+
dtls_srtp_t* crypto = nullptr;
|
|
73
|
+
Queue<8> dtls;
|
|
74
|
+
Queue<8> media;
|
|
75
|
+
using ReceiveQueue = Queue<64>;
|
|
76
|
+
ReceiveQueue* receiveMedia = nullptr;
|
|
77
|
+
Queue<8> outgoing;
|
|
78
|
+
std::mutex signalingMutex;
|
|
79
|
+
std::optional<receive::Offer> pending, offer;
|
|
80
|
+
bool offerAccepted = false; // Guarded by signalingMutex.
|
|
81
|
+
std::optional<vp8::Receiver> video;
|
|
82
|
+
rtp::Reorder audio;
|
|
83
|
+
receive::Track audioTrack, videoTrack;
|
|
84
|
+
std::atomic<juice_state_t> iceState{JUICE_STATE_DISCONNECTED};
|
|
85
|
+
std::atomic<bool> stopped{false}, overflow{false};
|
|
86
|
+
std::atomic<bool> offerRequested{false}, sendReady{false};
|
|
87
|
+
bool sending = false;
|
|
88
|
+
uint16_t sendSequence = 0;
|
|
89
|
+
uint32_t sentPackets = 0, sentOctets = 0, lastTimestamp = 0;
|
|
90
|
+
uint64_t lastAudioMs = 0;
|
|
91
|
+
uint64_t startedMs = 0, handshakeDeadlineMs = 0, lastReportMs = 0;
|
|
92
|
+
uint64_t lastMediaLogMs = 0;
|
|
93
|
+
uint32_t videoPackets = 0, videoFrames = 0, audioPackets = 0, unmatchedPackets = 0;
|
|
94
|
+
uint32_t authenticationDrops = 0, videoSequenceGaps = 0, videoLatePackets = 0;
|
|
95
|
+
uint32_t videoStarts = 0, videoKeyframes = 0, videoMarkers = 0;
|
|
96
|
+
uint32_t videoNonReferences = 0;
|
|
97
|
+
uint16_t nextVideoSequence = 0;
|
|
98
|
+
std::atomic<uint32_t> receiveDrops{0};
|
|
99
|
+
unsigned videoWidth = 0, videoHeight = 0;
|
|
100
|
+
uint32_t localSsrc = 0;
|
|
101
|
+
bool answered = false, authenticated = false, failed = false;
|
|
102
|
+
bool certificateMatched = false;
|
|
103
|
+
|
|
104
|
+
bool running() const { return !stopped && options.running && options.running->load(); }
|
|
105
|
+
|
|
106
|
+
void wake() const { if (options.wake) options.wake(options.wakeContext); }
|
|
107
|
+
|
|
108
|
+
void state(esp_peer_state_t value) {
|
|
109
|
+
if (cfg.on_state) cfg.on_state(value, cfg.ctx);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
int fail(const char* reason) {
|
|
113
|
+
sendReady = false;
|
|
114
|
+
if (!failed) { ESP_LOGE(tag, "%s", reason); state(ESP_PEER_STATE_CONNECT_FAILED); }
|
|
115
|
+
failed = true;
|
|
116
|
+
return ESP_PEER_ERR_FAIL;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
static Peer* bioPeer(void* context) {
|
|
120
|
+
return static_cast<Peer*>(static_cast<dtls_srtp_t*>(context)->ctx);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
static int send(void* context, const unsigned char* bytes, size_t size) {
|
|
124
|
+
auto* self = bioPeer(context);
|
|
125
|
+
if (!self->running()) return MBEDTLS_ERR_SSL_INTERNAL_ERROR;
|
|
126
|
+
const int result = juice_send(self->ice, reinterpret_cast<const char*>(bytes), size);
|
|
127
|
+
return result == JUICE_ERR_SUCCESS ? int(size) : MBEDTLS_ERR_SSL_INTERNAL_ERROR;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
static int recv(void* context, unsigned char* bytes, size_t size) {
|
|
131
|
+
auto* self = bioPeer(context);
|
|
132
|
+
Datagram packet;
|
|
133
|
+
if (!self->running() || (self->handshakeDeadlineMs && nowMs() > self->handshakeDeadlineMs))
|
|
134
|
+
return MBEDTLS_ERR_SSL_INTERNAL_ERROR;
|
|
135
|
+
if (!self->dtls.pop(packet)) {
|
|
136
|
+
vTaskDelay(1);
|
|
137
|
+
return MBEDTLS_ERR_SSL_WANT_READ;
|
|
138
|
+
}
|
|
139
|
+
if (packet.size > size) return MBEDTLS_ERR_SSL_BUFFER_TOO_SMALL;
|
|
140
|
+
std::memcpy(bytes, packet.bytes.data(), packet.size);
|
|
141
|
+
return int(packet.size);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
bool initialize() {
|
|
145
|
+
localSsrc = esp_random();
|
|
146
|
+
if (!localSsrc) localSsrc = 1;
|
|
147
|
+
sendSequence = uint16_t(esp_random());
|
|
148
|
+
startedMs = nowMs();
|
|
149
|
+
dtls_srtp_cfg_t config{};
|
|
150
|
+
config.role = DTLS_SRTP_ROLE_CLIENT;
|
|
151
|
+
config.ctx = this;
|
|
152
|
+
config.udp_send = send;
|
|
153
|
+
config.udp_recv = recv;
|
|
154
|
+
crypto = dtls_srtp_init(&config);
|
|
155
|
+
if (!crypto) return false;
|
|
156
|
+
juice_config_t iceConfig{};
|
|
157
|
+
iceConfig.user_ptr = this;
|
|
158
|
+
iceConfig.cb_state_changed = [](juice_agent_t*, juice_state_t value, void* ctx) {
|
|
159
|
+
auto* self = static_cast<Peer*>(ctx);
|
|
160
|
+
self->iceState = value;
|
|
161
|
+
self->wake();
|
|
162
|
+
};
|
|
163
|
+
iceConfig.cb_recv = [](juice_agent_t*, const char* bytes, size_t size, void* ctx) {
|
|
164
|
+
auto* self = static_cast<Peer*>(ctx);
|
|
165
|
+
if (!self->running() || !size) return;
|
|
166
|
+
const auto first = uint8_t(bytes[0]);
|
|
167
|
+
if (first >= 20 && first <= 63) {
|
|
168
|
+
if (!self->dtls.push(bytes, size)) self->overflow = true;
|
|
169
|
+
} else if (first >= 128 && first <= 191) {
|
|
170
|
+
const bool queued = self->receiveMedia ? self->receiveMedia->push(bytes, size) : self->media.push(bytes, size);
|
|
171
|
+
if (!queued) { self->overflow = true; ++self->receiveDrops; }
|
|
172
|
+
}
|
|
173
|
+
self->wake();
|
|
174
|
+
};
|
|
175
|
+
// The SFU is publicly reachable ICE-lite. Connectivity checks create the
|
|
176
|
+
// NAT mapping; the server learns the peer-reflexive address. No relay-only
|
|
177
|
+
// claim or silently ignored TURN config is supported by this adapter yet.
|
|
178
|
+
ice = juice_create(&iceConfig);
|
|
179
|
+
if (!ice) return false;
|
|
180
|
+
state(ESP_PEER_STATE_CANDIDATE_GATHERING);
|
|
181
|
+
if (juice_set_ice_tcp_mode(ice, JUICE_ICE_TCP_MODE_ACTIVE) != JUICE_ERR_SUCCESS ||
|
|
182
|
+
juice_gather_candidates(ice) != JUICE_ERR_SUCCESS)
|
|
183
|
+
return false;
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
bool start(receive::Offer remote) {
|
|
188
|
+
offer = std::move(remote);
|
|
189
|
+
if (!sending) {
|
|
190
|
+
for (const auto& track : offer->tracks) {
|
|
191
|
+
if (track.kind == "audio") audioTrack = track;
|
|
192
|
+
else videoTrack = track;
|
|
193
|
+
}
|
|
194
|
+
if (videoTrack.ssrc) video.emplace(videoTrack.payloadType, videoTrack.ssrc);
|
|
195
|
+
if (!initialize()) return false;
|
|
196
|
+
}
|
|
197
|
+
if (!ice || juice_set_remote_description(ice, offer->iceDescription.c_str()) != JUICE_ERR_SUCCESS)
|
|
198
|
+
return false;
|
|
199
|
+
juice_set_remote_gathering_done(ice);
|
|
200
|
+
state(ESP_PEER_STATE_PAIRING);
|
|
201
|
+
return true;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
bool makeDescription() {
|
|
205
|
+
std::array<char, JUICE_MAX_SDP_STRING_LEN> iceSdp{};
|
|
206
|
+
if (juice_get_local_description(ice, iceSdp.data(), iceSdp.size()) != JUICE_ERR_SUCCESS) return false;
|
|
207
|
+
const auto sdp = sending ?
|
|
208
|
+
receive::microphoneOffer(iceSdp.data(), dtls_srtp_get_local_fingerprint(crypto), localSsrc) :
|
|
209
|
+
receive::answer(*offer, iceSdp.data(), dtls_srtp_get_local_fingerprint(crypto));
|
|
210
|
+
if (!sdp) return false;
|
|
211
|
+
esp_peer_msg_t message{ESP_PEER_MSG_TYPE_SDP,
|
|
212
|
+
reinterpret_cast<uint8_t*>(const_cast<char*>(sdp->data())), int(sdp->size())};
|
|
213
|
+
if (cfg.on_msg && cfg.on_msg(&message, cfg.ctx)) return false;
|
|
214
|
+
answered = true;
|
|
215
|
+
return true;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
bool authenticate() {
|
|
219
|
+
state(ESP_PEER_STATE_CONNECTING);
|
|
220
|
+
handshakeDeadlineMs = nowMs() + 10000;
|
|
221
|
+
certificateMatched = false;
|
|
222
|
+
// IDF discards peer certificates after handshake by default. Authenticate
|
|
223
|
+
// the leaf while mbedTLS owns it; do not retain an entire certificate chain
|
|
224
|
+
// or weaken the fingerprint check when get_peer_cert() returns null.
|
|
225
|
+
mbedtls_ssl_set_verify(&crypto->ssl,
|
|
226
|
+
[](void* context, mbedtls_x509_crt* certificate, int depth, uint32_t* flags) {
|
|
227
|
+
auto* self = static_cast<Peer*>(context);
|
|
228
|
+
if (depth) return 0;
|
|
229
|
+
const auto* hash = mbedtls_md_info_from_type(MBEDTLS_MD_SHA256);
|
|
230
|
+
std::array<uint8_t, 32> digest{};
|
|
231
|
+
if (!certificate || !hash ||
|
|
232
|
+
mbedtls_md(hash, certificate->raw.p, certificate->raw.len, digest.data()))
|
|
233
|
+
return MBEDTLS_ERR_X509_CERT_VERIFY_FAILED;
|
|
234
|
+
uint8_t difference = 0;
|
|
235
|
+
for (size_t i = 0; i < digest.size(); ++i)
|
|
236
|
+
difference |= digest[i] ^ self->offer->peerFingerprint[i];
|
|
237
|
+
self->certificateMatched = difference == 0;
|
|
238
|
+
if (difference) return MBEDTLS_ERR_X509_CERT_VERIFY_FAILED;
|
|
239
|
+
// WebRTC authenticates a self-signed certificate through the SDP
|
|
240
|
+
// fingerprint, not a public CA or hostname. Only an exact match
|
|
241
|
+
// authorizes clearing the CA verification flags.
|
|
242
|
+
*flags = 0;
|
|
243
|
+
return 0;
|
|
244
|
+
}, this);
|
|
245
|
+
if (dtls_srtp_handshake(crypto) != 0 || !certificateMatched ||
|
|
246
|
+
!crypto->srtp_in || !crypto->srtp_out) return false;
|
|
247
|
+
handshakeDeadlineMs = 0;
|
|
248
|
+
authenticated = true;
|
|
249
|
+
sendReady = sending;
|
|
250
|
+
state(ESP_PEER_STATE_CONNECTED);
|
|
251
|
+
ESP_LOGI(tag, "%s SRTP authenticated", sending ? "Microphone" : "VP8/Opus");
|
|
252
|
+
return true;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
void deliverVideo(vp8::EncodedFrame frame) {
|
|
256
|
+
++videoFrames;
|
|
257
|
+
if (options.onOwnedVideo) {
|
|
258
|
+
if (options.onOwnedVideo(std::move(frame), cfg.ctx)) video->requireKeyframe();
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
esp_peer_video_frame_t output{frame.timestamp / 90, frame.bytes.data(), int(frame.bytes.size())};
|
|
262
|
+
if (cfg.on_video_data && cfg.on_video_data(&output, cfg.ctx)) video->requireKeyframe();
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
void deliverAudio(const rtp::Packet& packet) {
|
|
266
|
+
esp_peer_audio_frame_t output{packet.timestamp / 48,
|
|
267
|
+
const_cast<uint8_t*>(packet.payload.data()), int(packet.payload.size())};
|
|
268
|
+
if (cfg.on_audio_data) cfg.on_audio_data(&output, cfg.ctx);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
void feedback(bool pli) {
|
|
272
|
+
// Compound RR + SDES(CNAME), optionally PLI. This is valid whether or not
|
|
273
|
+
// reduced-size RTCP was negotiated. SRTCP tag/index room follows the bytes.
|
|
274
|
+
std::array<uint8_t, 96> bytes{0x80, 201, 0, 1};
|
|
275
|
+
rtp::write32(bytes.data() + 4, localSsrc);
|
|
276
|
+
int size = 8;
|
|
277
|
+
if (sending && sentPackets) {
|
|
278
|
+
timeval wall{};
|
|
279
|
+
gettimeofday(&wall, nullptr);
|
|
280
|
+
bytes[1] = 200; bytes[3] = 6;
|
|
281
|
+
rtp::write32(bytes.data() + 8, uint32_t(uint64_t(wall.tv_sec) + 2208988800ULL));
|
|
282
|
+
rtp::write32(bytes.data() + 12, uint32_t((uint64_t(wall.tv_usec) << 32) / 1000000));
|
|
283
|
+
rtp::write32(bytes.data() + 16, lastTimestamp + uint32_t((nowMs() - lastAudioMs) * 48));
|
|
284
|
+
rtp::write32(bytes.data() + 20, sentPackets);
|
|
285
|
+
rtp::write32(bytes.data() + 24, sentOctets);
|
|
286
|
+
size = 28;
|
|
287
|
+
}
|
|
288
|
+
bytes[size] = 0x81; bytes[size + 1] = 202; bytes[size + 3] = 4;
|
|
289
|
+
rtp::write32(bytes.data() + size + 4, localSsrc);
|
|
290
|
+
bytes[size + 8] = 1; bytes[size + 9] = 7;
|
|
291
|
+
std::memcpy(bytes.data() + size + 10, sending ? "gea-mic" : "gea-vp8", 7);
|
|
292
|
+
size += 20;
|
|
293
|
+
if (pli) {
|
|
294
|
+
const auto request = rtp::pictureLossIndication(localSsrc, videoTrack.ssrc);
|
|
295
|
+
std::memcpy(bytes.data() + size, request.data(), request.size());
|
|
296
|
+
size += int(request.size());
|
|
297
|
+
}
|
|
298
|
+
size_t protectedSize = bytes.size();
|
|
299
|
+
if (srtp_protect_rtcp(crypto->srtp_out, bytes.data(), size, bytes.data(), &protectedSize, 0) == srtp_err_status_ok &&
|
|
300
|
+
protectedSize > size_t(size) && protectedSize <= bytes.size())
|
|
301
|
+
juice_send(ice, reinterpret_cast<const char*>(bytes.data()), protectedSize);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
int enqueueAudio(esp_peer_audio_frame_t* frame) {
|
|
305
|
+
if (!sending || !sendReady || !running()) return ESP_PEER_ERR_WRONG_STATE;
|
|
306
|
+
if (!frame || !frame->data || frame->size <= 0 || frame->size > 1275)
|
|
307
|
+
return ESP_PEER_ERR_INVALID_ARG;
|
|
308
|
+
std::array<uint8_t, 1287> packet{};
|
|
309
|
+
packet[0] = 0x80;
|
|
310
|
+
packet[1] = 111;
|
|
311
|
+
packet[2] = uint8_t(sendSequence >> 8);
|
|
312
|
+
packet[3] = uint8_t(sendSequence);
|
|
313
|
+
rtp::write32(packet.data() + 4, uint32_t(uint64_t(frame->pts) * 48));
|
|
314
|
+
rtp::write32(packet.data() + 8, localSsrc);
|
|
315
|
+
std::memcpy(packet.data() + 12, frame->data, frame->size);
|
|
316
|
+
if (!outgoing.push(reinterpret_cast<const char*>(packet.data()), size_t(frame->size) + 12))
|
|
317
|
+
return ESP_PEER_ERR_WOULD_BLOCK;
|
|
318
|
+
++sendSequence;
|
|
319
|
+
wake();
|
|
320
|
+
return 0;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
bool sendAudio() {
|
|
324
|
+
// Only the peer worker touches SRTP. The encoder enqueues bounded RTP
|
|
325
|
+
// packets, avoiding concurrent crypto/report access from the two tasks.
|
|
326
|
+
Datagram packet;
|
|
327
|
+
while (outgoing.pop(packet)) {
|
|
328
|
+
const auto payloadBytes = packet.size - 12;
|
|
329
|
+
const auto timestamp = rtp::read32(packet.bytes.data() + 4);
|
|
330
|
+
size_t protectedSize = packet.bytes.size();
|
|
331
|
+
if (srtp_protect(crypto->srtp_out, packet.bytes.data(), int(packet.size),
|
|
332
|
+
packet.bytes.data(), &protectedSize, 0) != srtp_err_status_ok ||
|
|
333
|
+
protectedSize <= packet.size || protectedSize > packet.bytes.size() ||
|
|
334
|
+
juice_send(ice, reinterpret_cast<const char*>(packet.bytes.data()), protectedSize) != JUICE_ERR_SUCCESS)
|
|
335
|
+
return false;
|
|
336
|
+
++sentPackets;
|
|
337
|
+
sentOctets += uint32_t(payloadBytes);
|
|
338
|
+
lastTimestamp = timestamp;
|
|
339
|
+
lastAudioMs = nowMs();
|
|
340
|
+
}
|
|
341
|
+
return true;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
int loop() {
|
|
345
|
+
if (!running() || failed) return 0;
|
|
346
|
+
if (sending && !ice) {
|
|
347
|
+
if (!offerRequested) return 0;
|
|
348
|
+
if (!initialize() || !makeDescription()) return fail("Microphone local ICE/SDP generation failed");
|
|
349
|
+
}
|
|
350
|
+
if (!offer) {
|
|
351
|
+
std::optional<receive::Offer> next;
|
|
352
|
+
{ std::lock_guard lock(signalingMutex); next = std::move(pending); pending.reset(); }
|
|
353
|
+
if (!next) return 0;
|
|
354
|
+
if (!start(std::move(*next))) return fail("VP8 ICE/DTLS initialization failed");
|
|
355
|
+
}
|
|
356
|
+
if (!answered && !makeDescription()) return fail("VP8 local SDP generation failed");
|
|
357
|
+
const auto state = iceState.load();
|
|
358
|
+
if (state == JUICE_STATE_FAILED) return fail("VP8 ICE connectivity failed");
|
|
359
|
+
if (!authenticated) {
|
|
360
|
+
if (nowMs() - startedMs > 15000) return fail("VP8 ICE connection timed out");
|
|
361
|
+
if (state != JUICE_STATE_CONNECTED && state != JUICE_STATE_COMPLETED) return 0;
|
|
362
|
+
if (!authenticate()) return fail("VP8 DTLS or certificate fingerprint verification failed");
|
|
363
|
+
}
|
|
364
|
+
if (sending) {
|
|
365
|
+
if (!sendAudio()) return fail("Microphone SRTP send failed");
|
|
366
|
+
} else if (overflow.exchange(false) || options.requestKeyframe->exchange(false)) {
|
|
367
|
+
if (video) video->requireKeyframe();
|
|
368
|
+
}
|
|
369
|
+
if (!dtls.empty()) {
|
|
370
|
+
std::array<uint8_t, 256> control{};
|
|
371
|
+
if (dtls_srtp_read(crypto, control.data(), control.size()) == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY)
|
|
372
|
+
return fail("VP8 remote DTLS peer closed");
|
|
373
|
+
}
|
|
374
|
+
Datagram packet;
|
|
375
|
+
for (unsigned count = 0; count < 64 &&
|
|
376
|
+
(receiveMedia ? receiveMedia->pop(packet) : media.pop(packet)); ++count) {
|
|
377
|
+
int size = int(packet.size);
|
|
378
|
+
if (size < 12) continue;
|
|
379
|
+
const bool rtcp = packet.bytes[1] >= 192 && packet.bytes[1] <= 223;
|
|
380
|
+
if (rtcp) {
|
|
381
|
+
// Authenticate reports even though scheduling does not consume them yet.
|
|
382
|
+
dtls_srtp_decrypt_rtcp_packet(crypto, packet.bytes.data(), &size);
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
if (sending) continue;
|
|
386
|
+
if (dtls_srtp_decrypt_rtp_packet(crypto, packet.bytes.data(), &size) != 0 || size <= 0) {
|
|
387
|
+
++authenticationDrops;
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
const auto bytes = std::span(packet.bytes.data(), size_t(size));
|
|
391
|
+
const auto parsed = rtp::parse(bytes);
|
|
392
|
+
if (!parsed) continue;
|
|
393
|
+
if (parsed->payloadType == audioTrack.payloadType && parsed->ssrc == audioTrack.ssrc) {
|
|
394
|
+
++audioPackets;
|
|
395
|
+
audio.push(bytes, nowMs(), [&](const rtp::Packet& p) { deliverAudio(p); });
|
|
396
|
+
} else if (video && parsed->payloadType == videoTrack.payloadType && parsed->ssrc == videoTrack.ssrc) {
|
|
397
|
+
const auto distance = uint16_t(parsed->sequence - nextVideoSequence);
|
|
398
|
+
if (!videoPackets || distance < 0x8000) {
|
|
399
|
+
if (videoPackets) videoSequenceGaps += distance;
|
|
400
|
+
nextVideoSequence = uint16_t(parsed->sequence + 1);
|
|
401
|
+
} else ++videoLatePackets;
|
|
402
|
+
++videoPackets;
|
|
403
|
+
if (parsed->marker) ++videoMarkers;
|
|
404
|
+
// A lost keyframe tail must not hide the advertised stream size.
|
|
405
|
+
const auto payload = vp8::parsePayload(parsed->payload);
|
|
406
|
+
if (payload && payload->startsFrame) {
|
|
407
|
+
++videoStarts;
|
|
408
|
+
if (parsed->payload[0] & 0x20) ++videoNonReferences;
|
|
409
|
+
const auto header = payload->bytes;
|
|
410
|
+
if (!(header[0] & 1)) ++videoKeyframes;
|
|
411
|
+
if (header.size() >= 10 && !(header[0] & 1) && header[3] == 0x9d &&
|
|
412
|
+
header[4] == 0x01 && header[5] == 0x2a) {
|
|
413
|
+
const unsigned width = (header[6] | (unsigned(header[7]) << 8)) & 0x3fff;
|
|
414
|
+
const unsigned height = (header[8] | (unsigned(header[9]) << 8)) & 0x3fff;
|
|
415
|
+
if (width != videoWidth || height != videoHeight) {
|
|
416
|
+
videoWidth = width; videoHeight = height;
|
|
417
|
+
ESP_LOGI(tag, "VP8 RTP dimensions=%ux%u", width, height);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
video->push(bytes, nowMs(), [&](vp8::EncodedFrame f) { deliverVideo(std::move(f)); });
|
|
422
|
+
} else ++unmatchedPackets;
|
|
423
|
+
}
|
|
424
|
+
const auto now = nowMs();
|
|
425
|
+
if (sending) {
|
|
426
|
+
if (now - lastReportMs >= 1000) {
|
|
427
|
+
feedback(false);
|
|
428
|
+
ESP_LOGI(tag, "microphone transmitted packets=%u octets=%u", unsigned(sentPackets), unsigned(sentOctets));
|
|
429
|
+
lastReportMs = now;
|
|
430
|
+
}
|
|
431
|
+
return 0;
|
|
432
|
+
}
|
|
433
|
+
audio.poll(now, [&](const rtp::Packet& p) { deliverAudio(p); });
|
|
434
|
+
if (video) video->poll(now, [&](vp8::EncodedFrame f) { deliverVideo(std::move(f)); });
|
|
435
|
+
const bool pli = video && video->takeKeyframeRequest(now);
|
|
436
|
+
if (pli || now - lastReportMs >= 1000) { feedback(pli); lastReportMs = now; }
|
|
437
|
+
if (now - lastMediaLogMs >= 2000) {
|
|
438
|
+
ESP_LOGI(tag, "receive audio_packets=%u video_packets=%u video_frames=%u unmatched=%u queue_drops=%u needs_keyframe=%d",
|
|
439
|
+
unsigned(audioPackets), unsigned(videoPackets), unsigned(videoFrames),
|
|
440
|
+
unsigned(unmatchedPackets), unsigned(receiveDrops.load()), video && video->needsKeyframe());
|
|
441
|
+
ESP_LOGI(tag, "video RTP starts=%u keyframes=%u markers=%u sequence_gaps=%u late=%u auth_drops=%u nonreferences=%u",
|
|
442
|
+
unsigned(videoStarts), unsigned(videoKeyframes), unsigned(videoMarkers),
|
|
443
|
+
unsigned(videoSequenceGaps), unsigned(videoLatePackets), unsigned(authenticationDrops),
|
|
444
|
+
unsigned(videoNonReferences));
|
|
445
|
+
lastMediaLogMs = now;
|
|
446
|
+
}
|
|
447
|
+
return 0;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
~Peer() {
|
|
451
|
+
stopped = true;
|
|
452
|
+
if (ice) juice_destroy(ice); // Joins callbacks before their context is freed.
|
|
453
|
+
if (crypto) dtls_srtp_deinit(crypto);
|
|
454
|
+
if (receiveMedia) {
|
|
455
|
+
receiveMedia->~ReceiveQueue();
|
|
456
|
+
heap_caps_free(receiveMedia);
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
|
|
461
|
+
int open(esp_peer_cfg_t* cfg, esp_peer_handle_t* handle) {
|
|
462
|
+
if (!cfg || !handle || cfg->extra_size != sizeof(GeaVp8PeerOptions) || !cfg->extra_cfg)
|
|
463
|
+
return ESP_PEER_ERR_INVALID_ARG;
|
|
464
|
+
const bool sending = cfg->audio_dir == ESP_PEER_MEDIA_DIR_SEND_ONLY && cfg->video_dir == ESP_PEER_MEDIA_DIR_NONE;
|
|
465
|
+
const bool receiving = cfg->audio_dir == ESP_PEER_MEDIA_DIR_RECV_ONLY &&
|
|
466
|
+
(cfg->video_dir == ESP_PEER_MEDIA_DIR_NONE ||
|
|
467
|
+
(cfg->video_dir == ESP_PEER_MEDIA_DIR_RECV_ONLY && cfg->video_info.codec == GEA_PEER_VIDEO_CODEC_VP8));
|
|
468
|
+
if ((!sending && !receiving) || cfg->enable_data_channel ||
|
|
469
|
+
cfg->audio_info.codec != ESP_PEER_AUDIO_CODEC_OPUS ||
|
|
470
|
+
cfg->ice_trans_policy == ESP_PEER_ICE_TRANS_POLICY_RELAY) return ESP_PEER_ERR_NOT_SUPPORT;
|
|
471
|
+
const auto options = *static_cast<GeaVp8PeerOptions*>(cfg->extra_cfg);
|
|
472
|
+
if (!options.running || !options.requestKeyframe) return ESP_PEER_ERR_INVALID_ARG;
|
|
473
|
+
void* allocation = heap_caps_malloc(sizeof(Peer), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
|
474
|
+
if (!allocation) return ESP_PEER_ERR_NO_MEM;
|
|
475
|
+
auto* self = new(allocation) Peer;
|
|
476
|
+
self->cfg = *cfg;
|
|
477
|
+
self->options = options;
|
|
478
|
+
self->sending = sending;
|
|
479
|
+
if (!sending) {
|
|
480
|
+
auto* queue = heap_caps_malloc(sizeof(Peer::ReceiveQueue), MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT);
|
|
481
|
+
if (!queue) { self->~Peer(); heap_caps_free(self); return ESP_PEER_ERR_NO_MEM; }
|
|
482
|
+
self->receiveMedia = new(queue) Peer::ReceiveQueue;
|
|
483
|
+
}
|
|
484
|
+
*handle = self;
|
|
485
|
+
return 0;
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
int message(esp_peer_handle_t handle, esp_peer_msg_t* msg) {
|
|
489
|
+
auto* self = static_cast<Peer*>(handle);
|
|
490
|
+
if (!msg || !msg->data || msg->size <= 0) return ESP_PEER_ERR_INVALID_ARG;
|
|
491
|
+
if (msg->type != ESP_PEER_MSG_TYPE_SDP) return ESP_PEER_ERR_NOT_SUPPORT;
|
|
492
|
+
const std::string_view sdp{reinterpret_cast<const char*>(msg->data), size_t(msg->size)};
|
|
493
|
+
auto offer = self->sending ? receive::parseMicrophoneAnswer(sdp) : receive::parseOffer(sdp);
|
|
494
|
+
if (!offer) return ESP_PEER_ERR_BAD_DATA;
|
|
495
|
+
std::lock_guard lock(self->signalingMutex);
|
|
496
|
+
// One initial offer only. ICE restart/renegotiation require a fresh peer.
|
|
497
|
+
if (self->offerAccepted) return ESP_PEER_ERR_WRONG_STATE;
|
|
498
|
+
self->offerAccepted = true;
|
|
499
|
+
self->pending = std::move(offer);
|
|
500
|
+
self->wake();
|
|
501
|
+
return 0;
|
|
502
|
+
}
|
|
503
|
+
} // namespace
|
|
504
|
+
|
|
505
|
+
const esp_peer_ops_t* gea_vp8_peer_impl() {
|
|
506
|
+
static const auto ops = [] {
|
|
507
|
+
esp_peer_ops_t result{};
|
|
508
|
+
result.open = open;
|
|
509
|
+
result.send_msg = message;
|
|
510
|
+
result.new_connection = [](esp_peer_handle_t handle) -> int {
|
|
511
|
+
auto* peer = static_cast<Peer*>(handle);
|
|
512
|
+
if (!peer->sending) return ESP_PEER_ERR_NOT_SUPPORT;
|
|
513
|
+
if (peer->offerRequested.exchange(true)) return ESP_PEER_ERR_WRONG_STATE;
|
|
514
|
+
peer->wake();
|
|
515
|
+
return 0;
|
|
516
|
+
};
|
|
517
|
+
result.send_audio = [](esp_peer_handle_t handle, esp_peer_audio_frame_t* frame) {
|
|
518
|
+
return static_cast<Peer*>(handle)->enqueueAudio(frame);
|
|
519
|
+
};
|
|
520
|
+
result.main_loop = [](esp_peer_handle_t handle) {
|
|
521
|
+
auto* peer = static_cast<Peer*>(handle);
|
|
522
|
+
try { return peer->loop(); }
|
|
523
|
+
catch (...) { return peer->fail("VP8 peer allocation or media processing failed"); }
|
|
524
|
+
};
|
|
525
|
+
result.disconnect = [](esp_peer_handle_t handle) {
|
|
526
|
+
static_cast<Peer*>(handle)->stopped = true;
|
|
527
|
+
return 0;
|
|
528
|
+
};
|
|
529
|
+
result.close = [](esp_peer_handle_t handle) {
|
|
530
|
+
auto* peer = static_cast<Peer*>(handle);
|
|
531
|
+
peer->~Peer();
|
|
532
|
+
heap_caps_free(peer);
|
|
533
|
+
return 0;
|
|
534
|
+
};
|
|
535
|
+
return result;
|
|
536
|
+
}();
|
|
537
|
+
return &ops;
|
|
538
|
+
}
|