@geastack/core 0.1.33 → 0.1.35

package/app_frame_perf.cpp

@@ -30,6 +30,12 @@ const char *applicationFramePhaseName(ApplicationFramePhase phase)
 		case ApplicationFramePhase::StyleRecompute: return "style_recompute";
 		case ApplicationFramePhase::DocumentFrame: return "document_frame";
 		case ApplicationFramePhase::RefreshMounted: return "refresh_mounted";
+		case ApplicationFramePhase::CycleCollection: return "cycle_collection";
+		case ApplicationFramePhase::CallbackMicrotasks: return "callback_microtasks";
+		case ApplicationFramePhase::WebSocketCallbacks: return "websocket_callbacks";
+		case ApplicationFramePhase::RtcCallbacks: return "rtc_callbacks";
+		case ApplicationFramePhase::VideoPresentation: return "video_presentation";
+		case ApplicationFramePhase::HttpRequests: return "http_requests";
 		case ApplicationFramePhase::Idle: return "idle";
 	}
 	return "idle";
@@ -59,6 +65,12 @@ void applicationFramePerfStatsAdd(ApplicationFramePerfStats &stats, ApplicationF
 		case ApplicationFramePhase::RefreshMounted:
 			stats.refreshMountedUs += durationUs;
 			break;
+		case ApplicationFramePhase::CycleCollection:
+		case ApplicationFramePhase::CallbackMicrotasks:
+		case ApplicationFramePhase::WebSocketCallbacks:
+		case ApplicationFramePhase::RtcCallbacks:
+		case ApplicationFramePhase::VideoPresentation:
+		case ApplicationFramePhase::HttpRequests:
 		case ApplicationFramePhase::Idle:
 			break;
 	}

package/audio-worklet.d.ts

@@ -0,0 +1,49 @@
+import type { AudioContext, AudioDestinationNode } from './index'
+import type { MessagePort } from './workers'
+
+export interface AudioContextOptions {
+  sampleRate?: number
+  latencyHint?: 'interactive' | 'balanced' | 'playback' | number
+}
+
+export interface AudioWorklet {
+  addModule(moduleURL: string | URL): Promise<void>
+}
+
+export interface AudioWorkletNodeOptions {
+  numberOfInputs?: number
+  numberOfOutputs?: number
+  channelCount?: number
+  outputChannelCount?: number[]
+}
+
+export interface AudioWorkletNode {
+  readonly port: MessagePort
+  onprocessorerror: ((event: { readonly message: string }) => void) | null
+  connect(destination: AudioDestinationNode): AudioDestinationNode
+  disconnect(): void
+}
+
+export interface AudioWorkletNodeConstructor {
+  new (context: AudioContext, name: string, options?: AudioWorkletNodeOptions): AudioWorkletNode
+}
+
+export interface MediaStreamAudioSourceNode {
+  connect(destination: AudioWorkletNode): AudioWorkletNode
+  disconnect(): void
+}
+
+declare global {
+  const AudioWorkletNode: AudioWorkletNodeConstructor
+  abstract class AudioWorkletProcessor {
+    readonly port: MessagePort
+    constructor()
+    abstract process(inputs: Float32Array[][], outputs: Float32Array[][],
+      parameters: Record<string, Float32Array>): boolean
+  }
+  function registerProcessor(name: string, processorCtor: new () => AudioWorkletProcessor): void
+  /** AudioWorkletGlobalScope clock, in the context's sample-rate domain. */
+  const sampleRate: number
+  const currentFrame: number
+  const currentTime: number
+}

package/gea_app_entry.cpp

@@ -6,6 +6,7 @@
 #include "gea_perf_config.h"  // GEA_EMBEDDED_PERF master + per-subsystem perf flags
 #include "gea/embedded.h"
 #include "host/backends.h"
+#include "host/worker.h"
 #include "ui/style.h"
 #include "ui/tree_internal.h"  // Tree::mount for the post-mount clean repaint in init()
 
@@ -64,7 +65,23 @@ public:
 	{
 		if (active_) gea_cycle_collection_defer_begin();
 	}
-	~FrameCycleCollectionDeferral() { if (active_) gea_cycle_collection_defer_end(); }
+	~FrameCycleCollectionDeferral() {
+#if defined(ESP_PLATFORM) && CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS
+		const auto started = nowUs();
+#endif
+		if (active_) {
+			// Scope-exit collection runs after frame phases finish. Keep it
+			// visible to the watchdog instead of reporting a stalled frame as idle.
+			const auto previous = applicationFramePhaseRead();
+			applicationFramePhaseSet(ApplicationFramePhase::CycleCollection);
+			gea_cycle_collection_defer_end();
+			applicationFramePhaseSet(previous);
+		}
+#if defined(ESP_PLATFORM) && CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS
+		const auto elapsed = nowUs() - started;
+		if (elapsed > 30000) ESP_LOGW("frame_phase", "cycle_collection duration_ms=%lld", (long long)(elapsed / 1000));
+#endif
+	}
 
 private:
 	bool active_;
@@ -209,7 +226,7 @@ void logFramePerfWindow(int64_t windowEndUs)
 // pays zero for phase instrumentation.
 std::int64_t phaseClockUs()
 {
-#if GEA_FRAME_PHASE_TIMING
+#if GEA_FRAME_PHASE_TIMING || CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS
 	return nowUs();
 #else
 	return 0;
@@ -222,6 +239,11 @@ std::int64_t phaseClockUs()
 // neither is set.
 void recordFramePhase(ApplicationFramePhase phase, int64_t durationUs)
 {
+#if defined(ESP_PLATFORM) && CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS
+	if (durationUs > 30000)
+		ESP_LOGW("frame_phase", "%s duration_ms=%lld", applicationFramePhaseName(phase),
+			static_cast<long long>(durationUs / 1000));
+#endif
 #if GEA_EMBEDDED_FRAME_SCHEDULER_PERF_LOG
 	applicationFramePerfStatsAdd(phase, durationUs);
 #endif
@@ -273,11 +295,22 @@ void Application::frame(int timestampMs)
 	startUs = phaseClockUs();
 	applicationFramePhaseSet(ApplicationFramePhase::AnimationFrameCallbacks);
 	gea::host::runAnimationFrameCallbacks(static_cast<double>(timestampMs));
+	applicationFramePhaseSet(ApplicationFramePhase::CallbackMicrotasks);
 	generated::drainMicrotasks();
-	// Offline builds have no network events to drain. Avoid locking empty queues every frame.
+	// The build already proves whether network services are reachable. Keep their
+	// callback pumps out of offline frame loops, while still servicing local workers.
 #if !defined(GEA_EMBEDDED_NETWORK_SERVICES_DISABLED) || !GEA_EMBEDDED_NETWORK_SERVICES_DISABLED
+	applicationFramePhaseSet(ApplicationFramePhase::WebSocketCallbacks);
 	gea::host::websocket::runCallbacks();
+	applicationFramePhaseSet(ApplicationFramePhase::RtcCallbacks);
 	gea::host::rtc::runCallbacks();
+#else
+	gea::host::workers::Context::runMainPending();
+#endif
+	applicationFramePhaseSet(ApplicationFramePhase::VideoPresentation);
+	gea::host::video::presentFrames();
+#if !defined(GEA_EMBEDDED_NETWORK_SERVICES_DISABLED) || !GEA_EMBEDDED_NETWORK_SERVICES_DISABLED
+	applicationFramePhaseSet(ApplicationFramePhase::HttpRequests);
 	gea::host::http::runRequests();
 #endif
 	recordFramePhase(ApplicationFramePhase::AnimationFrameCallbacks, phaseClockUs() - startUs);

package/gea_sources.mjs

@@ -87,8 +87,8 @@ export function cxxSources(env = process.env) {
     // @geastack/host — platform layer + services
     ...cpp(host, ['input', 'wifi', 'bluetooth', 'geolocation']),
     ...cpp(join(host, 'host'), [
-      'timers', 'input', 'display', 'fetch', 'media', 'rtc', 'websocket', 'http', 'wifi',
-      'ble', 'apps', 'audio', 'image', 'touch', 'imu', 'memory', 'camera', 'geolocation',
+      'timers', 'worker', 'input', 'display', 'fetch', 'media', 'rtc', 'websocket', 'http', 'wifi',
+      'ble', 'apps', 'audio', 'audio_worklet', 'video', 'mjpeg', 'image', 'touch', 'imu', 'memory', 'camera', 'geolocation',
       'tile_loader'
     ]),
     // framework services (built on the platform abstraction — no target-specific deps)

package/include/app.h

@@ -31,6 +31,12 @@ enum class ApplicationFramePhase : std::uint8_t {
 	StyleRecompute,
 	DocumentFrame,
 	RefreshMounted,
+	CycleCollection,
+	CallbackMicrotasks,
+	WebSocketCallbacks,
+	RtcCallbacks,
+	VideoPresentation,
+	HttpRequests,
 };
 
 struct ApplicationFramePerfStats {

package/include/audio.h

@@ -5,6 +5,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <string>
+#include <functional>
 
 namespace gea::platform::audio {
 
@@ -72,6 +73,14 @@ public:
 	static bool playFile(const std::string &path);
 	static bool playPcm(const std::int16_t *samples, std::size_t sample_count, int sample_rate, int channels);
 	static void stopPlayback();
+	static void flushPlayback();
+	// Continuous 16 kHz mono source; zero means the platform could not attach it.
+	static std::uint64_t playPcmStream(std::function<std::size_t(std::int16_t*, std::size_t)> pull);
+	static void stopPcmStream(std::uint64_t stream);
+	static bool pcmStreamSettled(std::uint64_t stream);
+	// Maximum mixer and device buffering delays, in seconds.
+	static double processingLatency();
+	static double outputLatency();
 };
 
 #ifdef GEA_AUDIO_DRIVER_INTERNAL
@@ -80,8 +89,10 @@ public:
 	static bool open(int sample_rate, int channels, int bits_per_sample);
 	static bool write(const std::int16_t *pcm, std::size_t sample_count, int timeout_ms);
 	static void close();
+	static bool flush();
 	static int volume();
 	static void setVolume(int volume_percent);
+	static double outputLatency(int sample_rate);
 };
 #endif
 

package/include/audio_stream.h

@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <mutex>
+
+namespace gea::platform::audio {
+
+// The RTC decoder supplies 16 kHz mono PCM. Pulling from independent readers
+// avoids copying every incoming packet into another playback queue.
+class PcmStreamMixer {
+ public:
+  using Pull = std::function<std::size_t(std::int16_t*, std::size_t)>;
+  using Handle = std::uint64_t;
+  static constexpr std::size_t chunkFrames = 512;
+
+  Handle add(Pull pull) {
+    if (!pull) return 0;
+    std::lock_guard<std::mutex> lock(mutex_);
+    const auto id = ++next_;
+    sources_.emplace(id, Source{std::move(pull)});
+    return id;
+  }
+  void remove(Handle id) { std::lock_guard<std::mutex> lock(mutex_); sources_.erase(id); }
+  void clear() { std::lock_guard<std::mutex> lock(mutex_); sources_.clear(); }
+  bool active() const { std::lock_guard<std::mutex> lock(mutex_); return !sources_.empty(); }
+  bool settled(Handle id) const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    const auto it = sources_.find(id);
+    return it == sources_.end() || (!it->second.inFlight && !it->second.failed);
+  }
+  // Called by the sole output consumer AFTER its driver write returns. A
+  // reader being empty only proves its last block was pulled, not submitted.
+  void didWrite(bool success) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    for (auto& [id, source] : sources_) {
+      if (source.inFlight && !success) source.failed = true;
+      source.inFlight = false;
+    }
+  }
+
+  // Mix into existing stereo samples (e.g. oscillator output). Return the
+  // frames actually read: a short network packet must not acquire silence
+  // padding that lengthens speech and accumulates latency.
+  std::size_t mix(std::int16_t* stereo, std::size_t frames) {
+    frames = std::min(frames, chunkFrames);
+    std::array<std::int16_t, chunkFrames> mono;
+    std::size_t available = 0;
+    std::lock_guard<std::mutex> lock(mutex_);
+    for (auto& [id, source] : sources_) {
+      const auto count = std::min(frames, source.pull(mono.data(), frames));
+      source.inFlight = source.inFlight || count > 0;
+      available = std::max(available, count);
+      for (std::size_t i = 0; i < count; ++i) {
+        for (std::size_t channel = 0; channel < 2; ++channel) {
+          const auto sum = static_cast<int>(stereo[i * 2 + channel]) + mono[i];
+          stereo[i * 2 + channel] = static_cast<std::int16_t>(std::clamp(sum, -32768, 32767));
+        }
+      }
+    }
+    return available;
+  }
+ private:
+  mutable std::mutex mutex_;
+  struct Source { Pull pull; bool inFlight = false; bool failed = false; };
+  std::map<Handle, Source> sources_;
+  Handle next_ = 0;
+};
+}

package/include/canvas.h

@@ -88,6 +88,7 @@ public:
 	// LANDSCAPE is defined (papers3), so every other target is byte-for-byte
 	// unaffected. Supports both packed grayscale and native RGB framebuffers.
 	void bindPixelsRotatedLandscape(pixel::native_t *physPixels, int logicalWidth, int logicalHeight);
+	bool isLandscapeRotated() const { return landscapeRot_; }
 #endif
 
 	pixel::native_t *pixels();

package/include/pixel.h

@@ -658,17 +658,51 @@ inline std::uint16_t fromNative(native_t pixel)
 #endif
 }
 
-// Row copies across the RGB565-scratch <-> native-framebuffer boundary. Both
-// compile to a plain std::memcpy on RGB565 targets (native_t == std::uint16_t),
-// byte-identical to the original framebuffer code; on RGBA8888 they convert
-// per pixel (inactive path).
+// Non-overlapping native pixel spans, including cropped rows whose source and
+// destination have different vector alignment. Keep every load within the span:
+// a scalar head precedes the first aligned-down load, and the scalar tail leaves
+// room for the unaligned vector load's second aligned block.
+inline void copyNative(native_t *dst, const native_t *src, int count)
+{
+	if (count <= 0) return;
+#if GEA_PIXEL_PIE_FILL16 && !GEA_PIXEL_FORMAT_IS_8888 && !GEA_PIXEL_FORMAT_IS_GRAY
+	if (count >= 32) {
+		for (int i = 0; i < 8; ++i) { *dst++ = *src++; --count; }
+		while ((reinterpret_cast<std::uintptr_t>(dst) & 15u) != 0) {
+			*dst++ = *src++;
+			--count;
+		}
+		const int vectors = (count - 8) >> 3;
+		if (vectors > 0) {
+			const native_t *cursor = src;
+			asm volatile(
+				"ee.ld.128.usar.ip q0, %[src], 16\n"
+				"loopnez %[vectors], 1f\n"
+				"ee.ld.128.usar.ip q1, %[src], 16\n"
+				"ee.src.q.qup q2, q0, q1\n"
+				"ee.vst.128.ip q2, %[dst], 16\n"
+				"ee.orq q0, q1, q1\n"
+				"1:\n"
+				: [src] "+&r"(cursor), [dst] "+&r"(dst)
+				: [vectors] "r"(vectors)
+				: "memory");
+			src += vectors * 8;
+			count -= vectors * 8;
+		}
+	}
+#endif
+	std::memcpy(dst, src, static_cast<std::size_t>(count) * sizeof(native_t));
+}
+
+// Row copies across the RGB565-scratch <-> native-framebuffer boundary. RGB565
+// uses the native span copy above; other framebuffer formats convert per pixel.
 inline void copyNativeToRgb565(std::uint16_t *dst, const native_t *src, int count)
 {
 	if (count <= 0) return;
 #if GEA_PIXEL_FORMAT_IS_8888 || GEA_PIXEL_FORMAT_IS_GRAY
 	for (int i = 0; i < count; i++) dst[i] = fromNative(src[i]);
 #else
-	std::memcpy(dst, src, static_cast<std::size_t>(count) * sizeof(std::uint16_t));
+	copyNative(dst, src, count);
 #endif
 }
 
@@ -678,7 +712,7 @@ inline void copyRgb565ToNative(native_t *dst, const std::uint16_t *src, int coun
 #if GEA_PIXEL_FORMAT_IS_8888 || GEA_PIXEL_FORMAT_IS_GRAY
 	for (int i = 0; i < count; i++) dst[i] = toNative(src[i]);
 #else
-	std::memcpy(dst, src, static_cast<std::size_t>(count) * sizeof(std::uint16_t));
+	copyNative(dst, src, count);
 #endif
 }