@rainbow-robotics/muscat-plugin-kit 1.3.0 → 1.3.3-dev.0

package/cli/storybook-dist/iframe.html

@@ -509,7 +509,7 @@
   }
 </script>
 
-    <script type="module" crossorigin src="./assets/iframe-DsvRKHwc.js"></script>
+    <script type="module" crossorigin src="./assets/iframe-C1kyDB-R.js"></script>
   </head>
 
   <body>

package/cli/storybook-dist/project.json

@@ -1 +1 @@
-{"generatedAt":1789632370434,"userSince":1750932757926,"hasCustomBabel":false,"hasCustomWebpack":false,"hasStaticDirs":false,"hasStorybookEslint":true,"refCount":0,"testPackages":{"@vitest/browser":"3.2.4","@vitest/coverage-v8":"3.2.4","playwright":"1.59.1","vitest":"3.2.4"},"hasRouterPackage":false,"monorepo":"Turborepo","packageManager":{"type":"yarn","version":"berry","agent":"yarn@berry"},"preview":{"usesGlobals":false},"framework":{"name":"@storybook/react-vite","options":{}},"builder":"@storybook/builder-vite","renderer":"@storybook/react","portableStoriesFileCount":0,"applicationFileCount":0,"storybookVersion":"8.6.18","storybookVersionSpecifier":"^8.6.4","language":"typescript","storybookPackages":{"@chromatic-com/storybook":{"version":"3.2.7"},"@storybook/addon-essentials":{"version":"8.6.18"},"@storybook/addon-links":{"version":"8.6.18"},"@storybook/addon-onboarding":{"version":"8.6.18"},"@storybook/addon-themes":{"version":"8.6.18"},"@storybook/blocks":{"version":"8.6.18"},"@storybook/experimental-addon-test":{"version":"8.6.18"},"@storybook/react":{"version":"8.6.18"},"@storybook/react-vite":{"version":"8.6.18"},"@storybook/test":{"version":"8.6.18"},"@storybook/theming":{"version":"8.6.18"},"eslint-plugin-storybook":{"version":"0.11.6"},"storybook":{"version":"8.6.18"},"storybook-addon-react-docgen":{"version":"1.2.44"}},"addons":{"$SNIP/node_modules/@storybook/addon-links":{"version":null},"$SNIP/node_modules/@storybook/addon-essentials":{"version":null},"$SNIP/node_modules/@chromatic-com/storybook":{"version":null},"$SNIP/node_modules/@storybook/addon-themes":{"version":null},"$SNIP/node_modules/@storybook/experimental-addon-test":{"version":null}}}
+{"generatedAt":1790728570661,"userSince":1750932757926,"hasCustomBabel":false,"hasCustomWebpack":false,"hasStaticDirs":false,"hasStorybookEslint":true,"refCount":0,"testPackages":{"@vitest/browser":"3.2.4","@vitest/coverage-v8":"3.2.4","playwright":"1.59.1","vitest":"3.2.4"},"hasRouterPackage":false,"monorepo":"Turborepo","packageManager":{"type":"yarn","version":"berry","agent":"yarn@berry"},"preview":{"usesGlobals":false},"framework":{"name":"@storybook/react-vite","options":{}},"builder":"@storybook/builder-vite","renderer":"@storybook/react","portableStoriesFileCount":0,"applicationFileCount":0,"storybookVersion":"8.6.18","storybookVersionSpecifier":"^8.6.4","language":"typescript","storybookPackages":{"@chromatic-com/storybook":{"version":"3.2.7"},"@storybook/addon-essentials":{"version":"8.6.18"},"@storybook/addon-links":{"version":"8.6.18"},"@storybook/addon-onboarding":{"version":"8.6.18"},"@storybook/addon-themes":{"version":"8.6.18"},"@storybook/blocks":{"version":"8.6.18"},"@storybook/experimental-addon-test":{"version":"8.6.18"},"@storybook/react":{"version":"8.6.18"},"@storybook/react-vite":{"version":"8.6.18"},"@storybook/test":{"version":"8.6.18"},"@storybook/theming":{"version":"8.6.18"},"eslint-plugin-storybook":{"version":"0.11.6"},"storybook":{"version":"8.6.18"},"storybook-addon-react-docgen":{"version":"1.2.44"}},"addons":{"$SNIP/node_modules/@storybook/addon-links":{"version":null},"$SNIP/node_modules/@storybook/addon-essentials":{"version":null},"$SNIP/node_modules/@chromatic-com/storybook":{"version":null},"$SNIP/node_modules/@storybook/addon-themes":{"version":null},"$SNIP/node_modules/@storybook/experimental-addon-test":{"version":null}}}

package/cli/templates/common/README.md

@@ -766,15 +766,15 @@ container (your own Ubuntu 24.04 machine, or the robot PC):
 
 ```bash
 curl -fsSL https://rainbowrobotics.github.io/rb-sdk/rb-sdk-apt-repo.gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/rb-sdk-apt.gpg
-echo 'deb [signed-by=/usr/share/keyrings/rb-sdk-apt.gpg] https://rainbowrobotics.github.io/rb-sdk dev main' | sudo tee /etc/apt/sources.list.d/rb-sdk.list
-sudo apt update && sudo apt install rainbow-rb-sdk-dev=0.1.2~dev1
+echo 'deb [signed-by=/usr/share/keyrings/rb-sdk-apt.gpg] https://rainbowrobotics.github.io/rb-sdk stable main' | sudo tee /etc/apt/sources.list.d/rb-sdk.list
+sudo apt update && sudo apt install rainbow-rb-sdk-dev
 ```
 
 - CMake: `find_package(rb_sdk REQUIRED)` + `target_link_libraries(<target> rb_sdk::rb_sdk)` (C++20). pkg-config: `rb_sdk`.
 - Headers: `<rainbow/rb_sdk/manipulate/rb_manipulate_sdk.hpp>` (`RBManipulateSDK`),
   `<rainbow/rb_sdk/program/plugin_program_context.hpp>` (`rb_plugin::PluginProgramContext`, for `program.panel` commands — the header's top comment has a full `main()` example).
-- The binary is linked **dynamically** (`librb_sdk.so.<version>` by SONAME) and the robot provides that `.so`. Pin the SDK to the robot's version: the dev image default is `0.1.2~dev1`; change it with `MUSCAT_PK_RB_SDK_VERSION=<version> MUSCAT_PK_REBUILD_IMAGE=1 muscat-pk dev`. Do not bundle or static-link the SDK.
-- UI ↔ backend messaging (`RBPluginCall` pub/sub/serve) is Python-only for now.
+- The binary is linked **dynamically**; `muscat-pk build` copies the matching `librb_sdk.so` into `dist/<arch>/lib/` (the robot has no SDK installed), so each plugin keeps the SDK version it was built with. Change the version with `MUSCAT_PK_RB_SDK_VERSION=<version> MUSCAT_PK_REBUILD_IMAGE=1 muscat-pk dev`.
+- UI ↔ backend messaging works from C++ too: `rb_plugin::RBPluginCall` in `<rainbow/rb_sdk/plugin/rb_plugin.hpp>` (see the scaffolded `cpp/main.cpp`).
 
 ### ROS2 (rclpy)
 
@@ -815,6 +815,65 @@ public:
 
 ---
 
+## Video streaming (MediaMTX)
+
+Muscat runs a MediaMTX media server. The backend publishes a stream over RTSP and
+the UI plays it over WebRTC. `muscat-pk dev` runs the same server (or uses a local
+Muscat's), so the addresses are identical in dev and on the robot.
+
+| | Address |
+| --- | --- |
+| Publish (backend) | `rtsp://127.0.0.1:8554/<plugin_id>/<name>` |
+| Play (UI) | `http://<host>:8889/<plugin_id>/<name>/` |
+
+- Prefix the path with your plugin id — paths are shared with robot cameras and other plugins.
+- The robot runtime has no ffmpeg, GStreamer or OpenCV; bring your own encoder (below).
+
+**Backend (Python)** — use PyAV: add `av` and `numpy` to `requirements.txt`. Its wheels
+include FFmpeg, so they ship in the ZIP like any other dependency.
+
+```python
+import fractions, time
+import av, numpy as np
+
+out = av.open("rtsp://127.0.0.1:8554/__PLUGIN_ID__/cam", mode="w", format="rtsp",
+              options={"rtsp_transport": "tcp"})
+stream = out.add_stream("libx264", rate=25, options={"preset": "ultrafast", "tune": "zerolatency"})
+stream.width, stream.height, stream.pix_fmt = 640, 360, "yuv420p"
+
+i = 0
+while True:
+    img = np.zeros((360, 640, 3), dtype=np.uint8)          # your RGB frame here
+    frame = av.VideoFrame.from_ndarray(img, format="rgb24")
+    frame.pts, frame.time_base = i, fractions.Fraction(1, 25)
+    for packet in stream.encode(frame):
+        out.mux(packet)
+    i += 1
+    time.sleep(1 / 25)
+```
+
+Run the encoder loop in a thread (or `asyncio.to_thread`) so it doesn't block the
+`RBPluginCall` handlers.
+
+**UI** — embed MediaMTX's WebRTC player. `location.hostname` is `localhost` in the
+preview and the robot's address on the robot:
+
+```tsx
+const STREAM = "__PLUGIN_ID__/cam"; // the path the backend publishes
+
+export default function Camera() {
+  return (
+    <iframe
+      src={`${location.protocol}//${location.hostname}:8889/${STREAM}/`}
+      allow="autoplay"
+      className="aspect-video w-full border-0"
+    />
+  );
+}
+```
+
+---
+
 ## manifest.json
 
 ```jsonc
@@ -1043,6 +1102,8 @@ muscat-pk type change --to python     # → Python
 
 > Existing backend dirs (`ros2_ws/`, `cpp/`) are not deleted when switching away. Remove them manually if no longer needed.
 
+> Switching between a Python and a C++ backend asks whether to replace this README with the matching one (the old file is kept as `README.md.bak`). Add `--readme` to answer yes without the prompt.
+
 ---
 
 ## AI coding assistants (MCP)

package/cli/templates/common/py/main.py

@@ -24,6 +24,7 @@ import os
 from typing import Any
 
 from rainbow.rb_sdk import RBManipulateSDK, RBPluginCall
+from rainbow.rb_sdk.rb_sdk_cpp import ConnectOptions
 
 
 PLUGIN_NAME = "__PLUGIN_NAME__"
@@ -95,7 +96,10 @@ async def main():
         # Remote dev: connect to the robot before registering any handlers.
         # connect() closes and reopens the Zenoh session, so handlers registered
         # before this call would be lost.
-        await manipulate.connect(token=robot_token, host=robot_host)
+        opts = ConnectOptions()
+        opts.token = robot_token
+        opts.host = robot_host
+        manipulate.connect(opts)
 
     register_handlers()
 

package/cli/templates/common/requirements.txt

@@ -1,4 +1,4 @@
-rainbow-rb-sdk==0.1.1
-rainbow-rb-flat-buffers==0.1.1
-rainbow-rb-zenoh==0.1.1
+rainbow-rb-sdk==0.1.3
+rainbow-rb-flat-buffers==0.1.3
+rainbow-rb-zenoh==0.1.3
 requests==2.34.2

package/cli/templates/cpp/cpp/CMakeLists.txt

@@ -5,8 +5,7 @@ set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Rainbow Robotics C++ SDK — provided by the apt package rainbow-rb-sdk-dev
-# (pre-installed in the muscat-pk dev container). Linked dynamically: the robot
-# ships the matching librb_sdk.so, so do NOT bundle or static-link it.
+# (pre-installed in the muscat-pk dev container).
 find_package(rb_sdk REQUIRED)
 
 # The executable name must match runtime.entry in manifest.json.

package/cli/templates/cpp/cpp/main.cpp

@@ -3,14 +3,17 @@
 // The Rainbow Robotics C++ SDK (apt: rainbow-rb-sdk-dev) is installed in the
 // dev container and linked via CMake (rb_sdk::rb_sdk). Headers live under
 // <rainbow/rb_sdk/...>:
+//   - plugin/rb_plugin.hpp               rb_plugin::RBPluginCall — plugin UI <-> backend
+//                                        (serve/subscribe/publish) and external endpoints (router)
 //   - manipulate/rb_manipulate_sdk.hpp   RBManipulateSDK — control the robot
 //   - program/plugin_program_context.hpp rb_plugin::PluginProgramContext —
 //                                        program.panel commands (done/pause/stop,
-//                                        program variables); see the header's
-//                                        top comment for a full main() example
-// UI <-> backend messaging (RBPluginCall pub/sub/serve) is Python-only for now.
+//                                        program variables)
 
 #include <rainbow/rb_sdk/manipulate/rb_manipulate_sdk.hpp>
+#include <rainbow/rb_sdk/plugin/rb_plugin.hpp>
+
+#include <nlohmann/json.hpp>
 
 #include <chrono>
 #include <cstdlib>
@@ -18,13 +21,50 @@
 #include <string>
 #include <thread>
 
+using nlohmann::json;
+using rb_plugin::RBPluginCall;
+
 int main() {
-  const char* plugin_id = std::getenv("RB_PLUGIN_ID");
-  std::cout << "[__PLUGIN_NAME__] backend ready"
-            << (plugin_id ? std::string(" (id=") + plugin_id + ")" : "")
-            << std::endl;
+  const std::string name = "__PLUGIN_NAME__";
+
+  // Remote dev: connect before registering handlers (connect() reopens the session).
+  RBManipulateSDK manipulate;
+  const char* host = std::getenv("MUSCAT_ROBOT_HOST");
+  const char* token = std::getenv("MUSCAT_ROBOT_TOKEN");
+  if (host && token) manipulate.connect({.token = token, .host = host});
+
+  // plugin id comes from RB_PLUGIN_ID.
+  auto plugin = RBPluginCall::get_instance(RBPluginCall::resolve_plugin_id(std::nullopt));
+
+  // --- Plugin UI <-> backend (not reachable from outside Muscat) ---
+
+  // UI: await rb.action("grip", { width })
+  plugin->serve("grip", [&](const json& payload) {
+    const int width = payload.is_object() ? payload.value("width", 20) : 20;
+    std::cout << "[" << name << "] grip requested: width=" << width << std::endl;
+    plugin->publish("state", {{"ready", true}, {"last_grip_width", width}});  // -> rb.sub("state")
+    return json{{"ok", true}, {"width", width}};
+  });
+
+  // UI: rb.pub("event", ...)
+  auto event_sub = plugin->subscribe("event", [&](const std::string&, const json& payload) {
+    std::cout << "[" << name << "] event: " << payload.dump() << std::endl;
+  });
+
+  // --- External endpoints (REST / Socket.IO / TCP / MQTT / WebRTC) ---
+  //   curl -X POST http://<robot>/common/plugin/<plugin_id>/hello -d '{"name":"kim"}'
+  plugin->router().on("/hello", [](const json& payload, const json& meta) {
+    const std::string who = payload.is_object() ? payload.value("name", std::string("there")) : "there";
+    return json{{"msg", "hi " + who}, {"via", meta.value("transport", std::string())}};
+  });
+
+  // Example: robot control (not called automatically — add safety checks first).
+  //   IPC::MoveInput_TargetT t; t.tar_values = {0, 0, 0, 0, 0, 0};
+  //   manipulate.move.call_move_j("C500920", t);
+
+  plugin->publish("state", {{"ready", true}, {"plugin", name}});
+  std::cout << "[" << name << "] backend ready" << std::endl;
 
-  // e.g. RBManipulateSDK manipulate; manipulate.move.call_move_j(...);
   // Keep the process alive.
   while (true) {
     std::this_thread::sleep_for(std::chrono::seconds(1));