agenticros 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +192 -0
- package/README.md +90 -4
- package/dist/commands/config.d.ts +20 -0
- package/dist/commands/config.d.ts.map +1 -0
- package/dist/commands/config.js +143 -0
- package/dist/commands/config.js.map +1 -0
- package/dist/commands/doctor.d.ts +33 -0
- package/dist/commands/doctor.d.ts.map +1 -0
- package/dist/commands/doctor.js +232 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/down.d.ts +13 -0
- package/dist/commands/down.d.ts.map +1 -0
- package/dist/commands/down.js +81 -0
- package/dist/commands/down.js.map +1 -0
- package/dist/commands/init.d.ts +21 -0
- package/dist/commands/init.d.ts.map +1 -0
- package/dist/commands/init.js +259 -0
- package/dist/commands/init.js.map +1 -0
- package/dist/commands/logs.d.ts +18 -0
- package/dist/commands/logs.d.ts.map +1 -0
- package/dist/commands/logs.js +67 -0
- package/dist/commands/logs.js.map +1 -0
- package/dist/commands/status.d.ts +12 -0
- package/dist/commands/status.d.ts.map +1 -0
- package/dist/commands/status.js +52 -0
- package/dist/commands/status.js.map +1 -0
- package/dist/commands/up.d.ts +19 -0
- package/dist/commands/up.d.ts.map +1 -0
- package/dist/commands/up.js +58 -0
- package/dist/commands/up.js.map +1 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +106 -0
- package/dist/index.js.map +1 -0
- package/dist/menu.d.ts +9 -0
- package/dist/menu.d.ts.map +1 -0
- package/dist/menu.js +96 -0
- package/dist/menu.js.map +1 -0
- package/dist/runners/real-robot.d.ts +15 -0
- package/dist/runners/real-robot.d.ts.map +1 -0
- package/dist/runners/real-robot.js +46 -0
- package/dist/runners/real-robot.js.map +1 -0
- package/dist/runners/sim.d.ts +17 -0
- package/dist/runners/sim.d.ts.map +1 -0
- package/dist/runners/sim.js +51 -0
- package/dist/runners/sim.js.map +1 -0
- package/dist/util/env.d.ts +24 -0
- package/dist/util/env.d.ts.map +1 -0
- package/dist/util/env.js +53 -0
- package/dist/util/env.js.map +1 -0
- package/dist/util/logger.d.ts +24 -0
- package/dist/util/logger.d.ts.map +1 -0
- package/dist/util/logger.js +62 -0
- package/dist/util/logger.js.map +1 -0
- package/dist/util/paths.d.ts +57 -0
- package/dist/util/paths.d.ts.map +1 -0
- package/dist/util/paths.js +132 -0
- package/dist/util/paths.js.map +1 -0
- package/dist/util/pidfile.d.ts +16 -0
- package/dist/util/pidfile.d.ts.map +1 -0
- package/dist/util/pidfile.js +63 -0
- package/dist/util/pidfile.js.map +1 -0
- package/dist/util/state.d.ts +26 -0
- package/dist/util/state.d.ts.map +1 -0
- package/dist/util/state.js +55 -0
- package/dist/util/state.js.map +1 -0
- package/package.json +60 -1
- package/runtime/BUNDLE.json +11 -0
- package/runtime/LICENSE +192 -0
- package/runtime/README.md +273 -0
- package/runtime/docs/architecture.md +366 -0
- package/runtime/docs/cli.md +140 -0
- package/runtime/docs/memory.md +292 -0
- package/runtime/docs/robot-setup.md +347 -0
- package/runtime/package.json +28 -0
- package/runtime/packages/agenticros/agenticros-agenticros-0.0.1.tgz +0 -0
- package/runtime/packages/agenticros/openclaw.plugin.json +451 -0
- package/runtime/packages/agenticros/package.json +41 -0
- package/runtime/packages/agenticros/src/camera-snapshot-cache.ts +59 -0
- package/runtime/packages/agenticros/src/camera-snapshot-routes.ts +44 -0
- package/runtime/packages/agenticros/src/commands/estop.ts +41 -0
- package/runtime/packages/agenticros/src/commands/transport.ts +195 -0
- package/runtime/packages/agenticros/src/config-file.ts +136 -0
- package/runtime/packages/agenticros/src/config-page.ts +498 -0
- package/runtime/packages/agenticros/src/context/robot-context.ts +373 -0
- package/runtime/packages/agenticros/src/depth.ts +313 -0
- package/runtime/packages/agenticros/src/describer.ts +157 -0
- package/runtime/packages/agenticros/src/image-binary-trim.ts +16 -0
- package/runtime/packages/agenticros/src/index.ts +85 -0
- package/runtime/packages/agenticros/src/landing-page.ts +38 -0
- package/runtime/packages/agenticros/src/memory.ts +44 -0
- package/runtime/packages/agenticros/src/plugin-api.ts +173 -0
- package/runtime/packages/agenticros/src/plugin-image-base64.ts +69 -0
- package/runtime/packages/agenticros/src/preflight.ts +110 -0
- package/runtime/packages/agenticros/src/routes.ts +328 -0
- package/runtime/packages/agenticros/src/safety/validator.ts +43 -0
- package/runtime/packages/agenticros/src/service.ts +359 -0
- package/runtime/packages/agenticros/src/skill-api.ts +65 -0
- package/runtime/packages/agenticros/src/skill-loader.ts +146 -0
- package/runtime/packages/agenticros/src/teleop/page.ts +498 -0
- package/runtime/packages/agenticros/src/teleop/routes.ts +650 -0
- package/runtime/packages/agenticros/src/tools/index.ts +26 -0
- package/runtime/packages/agenticros/src/tools/ros2-action.ts +50 -0
- package/runtime/packages/agenticros/src/tools/ros2-camera.ts +221 -0
- package/runtime/packages/agenticros/src/tools/ros2-depth-distance.ts +58 -0
- package/runtime/packages/agenticros/src/tools/ros2-introspect.ts +62 -0
- package/runtime/packages/agenticros/src/tools/ros2-memory.ts +158 -0
- package/runtime/packages/agenticros/src/tools/ros2-param.ts +87 -0
- package/runtime/packages/agenticros/src/tools/ros2-publish.ts +52 -0
- package/runtime/packages/agenticros/src/tools/ros2-service.ts +46 -0
- package/runtime/packages/agenticros/src/tools/ros2-subscribe.ts +71 -0
- package/runtime/packages/agenticros/tsconfig.json +9 -0
- package/runtime/packages/agenticros-claude-code/README.md +260 -0
- package/runtime/packages/agenticros-claude-code/config.example.json +9 -0
- package/runtime/packages/agenticros-claude-code/dist/config.d.ts +8 -0
- package/runtime/packages/agenticros-claude-code/dist/config.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/config.js +93 -0
- package/runtime/packages/agenticros-claude-code/dist/config.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/depth.d.ts +20 -0
- package/runtime/packages/agenticros-claude-code/dist/depth.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/depth.js +126 -0
- package/runtime/packages/agenticros-claude-code/dist/depth.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/coco-classes.d.ts +6 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/coco-classes.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/coco-classes.js +36 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/coco-classes.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/find-object.d.ts +33 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/find-object.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/find-object.js +134 -0
- package/runtime/packages/agenticros-claude-code/dist/find-object/find-object.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/controller.d.ts +43 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/controller.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/controller.js +73 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/controller.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/detector.d.ts +58 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/detector.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/detector.js +251 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/detector.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/loop.d.ts +61 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/loop.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/loop.js +268 -0
- package/runtime/packages/agenticros-claude-code/dist/follow-me/loop.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/index.d.ts +3 -0
- package/runtime/packages/agenticros-claude-code/dist/index.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/index.js +111 -0
- package/runtime/packages/agenticros-claude-code/dist/index.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/memory.d.ts +17 -0
- package/runtime/packages/agenticros-claude-code/dist/memory.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/memory.js +44 -0
- package/runtime/packages/agenticros-claude-code/dist/memory.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/safety.d.ts +10 -0
- package/runtime/packages/agenticros-claude-code/dist/safety.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/safety.js +34 -0
- package/runtime/packages/agenticros-claude-code/dist/safety.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/tools.d.ts +36 -0
- package/runtime/packages/agenticros-claude-code/dist/tools.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/tools.js +777 -0
- package/runtime/packages/agenticros-claude-code/dist/tools.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/transport.d.ts +17 -0
- package/runtime/packages/agenticros-claude-code/dist/transport.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/transport.js +46 -0
- package/runtime/packages/agenticros-claude-code/dist/transport.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/zero-shot/detector.d.ts +42 -0
- package/runtime/packages/agenticros-claude-code/dist/zero-shot/detector.d.ts.map +1 -0
- package/runtime/packages/agenticros-claude-code/dist/zero-shot/detector.js +114 -0
- package/runtime/packages/agenticros-claude-code/dist/zero-shot/detector.js.map +1 -0
- package/runtime/packages/agenticros-claude-code/package.json +29 -0
- package/runtime/packages/agenticros-claude-code/src/config.ts +96 -0
- package/runtime/packages/agenticros-claude-code/src/depth.ts +173 -0
- package/runtime/packages/agenticros-claude-code/src/find-object/coco-classes.ts +38 -0
- package/runtime/packages/agenticros-claude-code/src/find-object/find-object.ts +190 -0
- package/runtime/packages/agenticros-claude-code/src/follow-me/controller.ts +109 -0
- package/runtime/packages/agenticros-claude-code/src/follow-me/depth-loop.ts +420 -0
- package/runtime/packages/agenticros-claude-code/src/follow-me/detector.ts +303 -0
- package/runtime/packages/agenticros-claude-code/src/follow-me/loop.ts +330 -0
- package/runtime/packages/agenticros-claude-code/src/index.ts +125 -0
- package/runtime/packages/agenticros-claude-code/src/memory.ts +51 -0
- package/runtime/packages/agenticros-claude-code/src/safety.ts +44 -0
- package/runtime/packages/agenticros-claude-code/src/tools.ts +891 -0
- package/runtime/packages/agenticros-claude-code/src/transport.ts +58 -0
- package/runtime/packages/agenticros-claude-code/src/zero-shot/detector.ts +169 -0
- package/runtime/packages/agenticros-claude-code/tsconfig.json +9 -0
- package/runtime/packages/agenticros-claude-code/yolo-debug.mjs +106 -0
- package/runtime/packages/agenticros-gemini/README.md +139 -0
- package/runtime/packages/agenticros-gemini/package.json +28 -0
- package/runtime/packages/agenticros-gemini/scripts/smoke-api.mjs +42 -0
- package/runtime/packages/agenticros-gemini/src/chat.ts +139 -0
- package/runtime/packages/agenticros-gemini/src/config.ts +92 -0
- package/runtime/packages/agenticros-gemini/src/depth.ts +173 -0
- package/runtime/packages/agenticros-gemini/src/index.ts +58 -0
- package/runtime/packages/agenticros-gemini/src/memory.ts +32 -0
- package/runtime/packages/agenticros-gemini/src/safety.ts +44 -0
- package/runtime/packages/agenticros-gemini/src/tools.ts +516 -0
- package/runtime/packages/agenticros-gemini/src/transport.ts +58 -0
- package/runtime/packages/agenticros-gemini/tsconfig.json +8 -0
- package/runtime/packages/core/package.json +47 -0
- package/runtime/packages/core/src/banner.ts +32 -0
- package/runtime/packages/core/src/cmd-vel-twist.ts +31 -0
- package/runtime/packages/core/src/config.ts +279 -0
- package/runtime/packages/core/src/index.ts +54 -0
- package/runtime/packages/core/src/memory/__tests__/factory.test.ts +70 -0
- package/runtime/packages/core/src/memory/__tests__/local-provider.test.ts +195 -0
- package/runtime/packages/core/src/memory/__tests__/mem0-provider.test.ts +192 -0
- package/runtime/packages/core/src/memory/__tests__/smart-defaults.test.ts +46 -0
- package/runtime/packages/core/src/memory/factory.ts +63 -0
- package/runtime/packages/core/src/memory/index.ts +10 -0
- package/runtime/packages/core/src/memory/local/provider.ts +229 -0
- package/runtime/packages/core/src/memory/mem0/provider.ts +379 -0
- package/runtime/packages/core/src/memory/types.ts +96 -0
- package/runtime/packages/core/src/topic-utils.ts +95 -0
- package/runtime/packages/core/src/transport/factory.ts +47 -0
- package/runtime/packages/core/src/transport/local/conversion.ts +333 -0
- package/runtime/packages/core/src/transport/local/entities.ts +129 -0
- package/runtime/packages/core/src/transport/local/transport.ts +386 -0
- package/runtime/packages/core/src/transport/rosbridge/actions.ts +81 -0
- package/runtime/packages/core/src/transport/rosbridge/adapter.ts +157 -0
- package/runtime/packages/core/src/transport/rosbridge/client.ts +438 -0
- package/runtime/packages/core/src/transport/rosbridge/services.ts +41 -0
- package/runtime/packages/core/src/transport/rosbridge/topics.ts +60 -0
- package/runtime/packages/core/src/transport/rosbridge/types.ts +118 -0
- package/runtime/packages/core/src/transport/transport.ts +77 -0
- package/runtime/packages/core/src/transport/types.ts +137 -0
- package/runtime/packages/core/src/transport/webrtc/signaling-client.ts +196 -0
- package/runtime/packages/core/src/transport/webrtc/signaling-types.ts +130 -0
- package/runtime/packages/core/src/transport/webrtc/transport.ts +516 -0
- package/runtime/packages/core/src/transport/zenoh/adapter.ts +357 -0
- package/runtime/packages/core/src/transport/zenoh/cdr.ts +183 -0
- package/runtime/packages/core/src/transport/zenoh/keys.ts +51 -0
- package/runtime/packages/core/tsconfig.json +9 -0
- package/runtime/packages/ros-camera/package.json +30 -0
- package/runtime/packages/ros-camera/src/index.ts +13 -0
- package/runtime/packages/ros-camera/src/snapshot.ts +372 -0
- package/runtime/packages/ros-camera/tsconfig.json +9 -0
- package/runtime/pnpm-lock.yaml +5260 -0
- package/runtime/pnpm-workspace.yaml +2 -0
- package/runtime/ros2_ws/src/agenticros_agent/agenticros_agent/__init__.py +0 -0
- package/runtime/ros2_ws/src/agenticros_agent/agenticros_agent/agent_node.py +561 -0
- package/runtime/ros2_ws/src/agenticros_agent/package.xml +25 -0
- package/runtime/ros2_ws/src/agenticros_agent/resource/agenticros_agent +0 -0
- package/runtime/ros2_ws/src/agenticros_agent/setup.cfg +4 -0
- package/runtime/ros2_ws/src/agenticros_agent/setup.py +25 -0
- package/runtime/ros2_ws/src/agenticros_bringup/README.md +128 -0
- package/runtime/ros2_ws/src/agenticros_bringup/agenticros_bringup/__init__.py +1 -0
- package/runtime/ros2_ws/src/agenticros_bringup/agenticros_bringup/cmd_vel_relay.py +33 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/cmd_vel_bridge.launch.py +58 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/gazebo_turtlebot3.launch.py +69 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/mode_a_gazebo.launch.py +55 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/mode_a_gazebo_rviz.launch.py +48 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/realsense_rosbridge.launch.py +154 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/rosbridge_gazebo.launch.py +54 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/rviz.launch.py +38 -0
- package/runtime/ros2_ws/src/agenticros_bringup/launch/turtlebot3_gazebo_rviz.launch.py +42 -0
- package/runtime/ros2_ws/src/agenticros_bringup/package.xml +24 -0
- package/runtime/ros2_ws/src/agenticros_bringup/resource/agenticros_bringup +0 -0
- package/runtime/ros2_ws/src/agenticros_bringup/rviz/turtlebot3_agenticros.rviz +174 -0
- package/runtime/ros2_ws/src/agenticros_bringup/setup.cfg +4 -0
- package/runtime/ros2_ws/src/agenticros_bringup/setup.py +28 -0
- package/runtime/ros2_ws/src/agenticros_discovery/agenticros_discovery/__init__.py +0 -0
- package/runtime/ros2_ws/src/agenticros_discovery/agenticros_discovery/discovery_node.py +172 -0
- package/runtime/ros2_ws/src/agenticros_discovery/package.xml +22 -0
- package/runtime/ros2_ws/src/agenticros_discovery/resource/agenticros_discovery +0 -0
- package/runtime/ros2_ws/src/agenticros_discovery/setup.cfg +5 -0
- package/runtime/ros2_ws/src/agenticros_discovery/setup.py +25 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/README.md +66 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/agenticros_follow_me/__init__.py +1 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/agenticros_follow_me/__main__.py +5 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/agenticros_follow_me/follow_me_node.py +278 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/agenticros_follow_me/follower_controller.py +631 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/agenticros_follow_me/person_tracker.py +635 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/package.xml +22 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/resource/agenticros_follow_me +0 -0
- package/runtime/ros2_ws/src/agenticros_follow_me/setup.py +25 -0
- package/runtime/ros2_ws/src/agenticros_msgs/CMakeLists.txt +26 -0
- package/runtime/ros2_ws/src/agenticros_msgs/msg/CapabilityManifest.msg +9 -0
- package/runtime/ros2_ws/src/agenticros_msgs/package.xml +22 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/FollowMeGetStatus.srv +11 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/FollowMeSetDistance.srv +4 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/FollowMeSetTarget.srv +6 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/FollowMeStart.srv +5 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/FollowMeStop.srv +3 -0
- package/runtime/ros2_ws/src/agenticros_msgs/srv/GetCapabilities.srv +5 -0
- package/runtime/scripts/activate_workspace.sh +285 -0
- package/runtime/scripts/agenticros-describer.policy.yaml +96 -0
- package/runtime/scripts/agenticros-proxy.cjs +99 -0
- package/runtime/scripts/agenticros-rosbridge.policy.yaml +62 -0
- package/runtime/scripts/check-cli-tarball-size.mjs +42 -0
- package/runtime/scripts/configure_agenticros.sh +200 -0
- package/runtime/scripts/fix-openclaw-control-ui-path.sh +20 -0
- package/runtime/scripts/install_cli.sh +94 -0
- package/runtime/scripts/install_rosbridge_from_source.sh +67 -0
- package/runtime/scripts/lib/agenticros-banner.sh +28 -0
- package/runtime/scripts/onboard_robot.sh +75 -0
- package/runtime/scripts/openai.policy.yaml +77 -0
- package/runtime/scripts/openclaw-dashboard-url.cjs +49 -0
- package/runtime/scripts/pack-runtime.mjs +245 -0
- package/runtime/scripts/run_demo_native.sh +43 -0
- package/runtime/scripts/run_nemoclaw_host_stack.sh +91 -0
- package/runtime/scripts/run_robot_rosbridge.sh +36 -0
- package/runtime/scripts/sandbox_rosbridge_relay.py +137 -0
- package/runtime/scripts/setup-openclaw-local.cjs +75 -0
- package/runtime/scripts/setup_gateway_plugin.sh +329 -0
- package/runtime/scripts/setup_robot.sh +113 -0
- package/runtime/scripts/setup_workspace.sh +484 -0
- package/runtime/scripts/smoke_test_nemoclaw.sh +123 -0
- package/runtime/scripts/start_demo.sh +55 -0
- package/runtime/scripts/sync-skill-tools.mjs +335 -0
- package/runtime/scripts/test-rclnodejs.mts +129 -0
- package/runtime/scripts/use-openclaw-2026.2.26.sh +19 -0
- package/runtime/scripts/use-openclaw-2026.3.11.sh +19 -0
- package/runtime/scripts/zenoh-bridge-ros2dds-robot.json5 +30 -0
- package/runtime/scripts/zenohd-agenticros.json5 +11 -0
- package/runtime/scripts/zenohd-rosclaw.json5 +11 -0
- package/runtime/tsconfig.base.json +19 -0
- package/index.js +0 -6
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YOLOv8n person detector for the MCP-side follow-me loop.
|
|
3
|
+
*
|
|
4
|
+
* Loads a YOLOv8n ONNX model and runs person-only detection on a single JPEG/PNG
|
|
5
|
+
* frame. Image decoded with sharp; inference via onnxruntime-node (CPU).
|
|
6
|
+
*
|
|
7
|
+
* Model lookup order:
|
|
8
|
+
* 1. AGENTICROS_YOLOV8_MODEL env var (absolute path)
|
|
9
|
+
* 2. ~/.agenticros/models/yolov8n.onnx
|
|
10
|
+
* If the file is missing it is downloaded from AGENTICROS_YOLOV8_URL (or a default
|
|
11
|
+
* public mirror). 6 MB, one-time.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import fs from "node:fs";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import os from "node:os";
|
|
17
|
+
import https from "node:https";
|
|
18
|
+
import http from "node:http";
|
|
19
|
+
|
|
20
|
+
// Type-only imports keep `ort` / `sharp` types available at compile time
|
|
21
|
+
// without forcing the native packages to load when this module is imported.
|
|
22
|
+
// The actual runtime modules are loaded lazily in `loadDeps()` so a missing
|
|
23
|
+
// native dep cannot crash MCP server startup — it only fails when the user
|
|
24
|
+
// actually invokes a detection tool.
|
|
25
|
+
type OrtModule = typeof import("onnxruntime-node");
|
|
26
|
+
type SharpFn = (input: Buffer | Uint8Array) => import("sharp").Sharp;
|
|
27
|
+
import type { InferenceSession as OrtInferenceSession } from "onnxruntime-node";
|
|
28
|
+
|
|
29
|
+
let ortModule: OrtModule | null = null;
|
|
30
|
+
let sharpFn: SharpFn | null = null;
|
|
31
|
+
|
|
32
|
+
async function loadDeps(): Promise<{ ort: OrtModule; sharp: SharpFn }> {
|
|
33
|
+
if (ortModule && sharpFn) return { ort: ortModule, sharp: sharpFn };
|
|
34
|
+
try {
|
|
35
|
+
const [ortMod, sharpMod] = await Promise.all([
|
|
36
|
+
import("onnxruntime-node"),
|
|
37
|
+
import("sharp"),
|
|
38
|
+
]);
|
|
39
|
+
// Both packages are CJS; under Node ESM their default export is the real
|
|
40
|
+
// module value. Fall back to the namespace if `.default` is absent.
|
|
41
|
+
const ortAny = ortMod as unknown as { default?: OrtModule };
|
|
42
|
+
ortModule = ortAny.default ?? (ortMod as unknown as OrtModule);
|
|
43
|
+
const sharpAny = sharpMod as unknown as { default?: SharpFn };
|
|
44
|
+
sharpFn = sharpAny.default ?? (sharpMod as unknown as SharpFn);
|
|
45
|
+
} catch (err) {
|
|
46
|
+
const hint = err instanceof Error ? err.message : String(err);
|
|
47
|
+
throw new Error(
|
|
48
|
+
`Local YOLO detection requires the optional packages 'onnxruntime-node' and 'sharp'. ` +
|
|
49
|
+
`Install them in this workspace (pnpm install) to enable follow-me local mode and ros2_find_object. ` +
|
|
50
|
+
`Underlying error: ${hint}`,
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
return { ort: ortModule!, sharp: sharpFn! };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const DEFAULT_MODEL_URL =
|
|
57
|
+
"https://huggingface.co/Ultralytics/YOLOv8/resolve/main/yolov8n.onnx";
|
|
58
|
+
|
|
59
|
+
const INPUT_SIZE = 640;
|
|
60
|
+
const PERSON_CLASS_ID = 0;
|
|
61
|
+
|
|
62
|
+
export interface PersonDetection {
|
|
63
|
+
/** Bounding box in original image pixel coordinates. */
|
|
64
|
+
x: number;
|
|
65
|
+
y: number;
|
|
66
|
+
width: number;
|
|
67
|
+
height: number;
|
|
68
|
+
/** Center of the bbox (image pixels). */
|
|
69
|
+
cx: number;
|
|
70
|
+
cy: number;
|
|
71
|
+
/** Detection confidence [0,1]. */
|
|
72
|
+
confidence: number;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export interface DetectorOptions {
|
|
76
|
+
/** Score threshold for filtering raw detections (default 0.4). */
|
|
77
|
+
scoreThreshold?: number;
|
|
78
|
+
/** IoU threshold for NMS (default 0.5). */
|
|
79
|
+
iouThreshold?: number;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function resolveModelPath(): string {
|
|
83
|
+
const fromEnv = process.env["AGENTICROS_YOLOV8_MODEL"];
|
|
84
|
+
if (fromEnv && fromEnv.trim().length > 0) return fromEnv.trim();
|
|
85
|
+
return path.join(os.homedir(), ".agenticros", "models", "yolov8n.onnx");
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function downloadFile(url: string, dest: string, redirectsLeft = 5): Promise<void> {
|
|
89
|
+
return new Promise((resolve, reject) => {
|
|
90
|
+
const client = url.startsWith("https:") ? https : http;
|
|
91
|
+
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
|
92
|
+
const tmp = `${dest}.partial`;
|
|
93
|
+
const file = fs.createWriteStream(tmp);
|
|
94
|
+
client
|
|
95
|
+
.get(url, (res) => {
|
|
96
|
+
const status = res.statusCode ?? 0;
|
|
97
|
+
if (status >= 300 && status < 400 && res.headers.location) {
|
|
98
|
+
file.close();
|
|
99
|
+
fs.unlink(tmp, () => {});
|
|
100
|
+
if (redirectsLeft <= 0) {
|
|
101
|
+
reject(new Error(`Too many redirects downloading ${url}`));
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
const next = new URL(res.headers.location, url).toString();
|
|
105
|
+
downloadFile(next, dest, redirectsLeft - 1).then(resolve, reject);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
if (status !== 200) {
|
|
109
|
+
file.close();
|
|
110
|
+
fs.unlink(tmp, () => {});
|
|
111
|
+
reject(new Error(`Download failed ${status} for ${url}`));
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
res.pipe(file);
|
|
115
|
+
file.on("finish", () => {
|
|
116
|
+
file.close();
|
|
117
|
+
fs.renameSync(tmp, dest);
|
|
118
|
+
resolve();
|
|
119
|
+
});
|
|
120
|
+
})
|
|
121
|
+
.on("error", (err) => {
|
|
122
|
+
file.close();
|
|
123
|
+
fs.unlink(tmp, () => {});
|
|
124
|
+
reject(err);
|
|
125
|
+
});
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async function ensureModel(): Promise<string> {
|
|
130
|
+
const modelPath = resolveModelPath();
|
|
131
|
+
if (fs.existsSync(modelPath) && fs.statSync(modelPath).size > 1_000_000) {
|
|
132
|
+
return modelPath;
|
|
133
|
+
}
|
|
134
|
+
const url = process.env["AGENTICROS_YOLOV8_URL"] || DEFAULT_MODEL_URL;
|
|
135
|
+
process.stderr.write(`[AgenticROS] follow-me: downloading YOLOv8n ONNX → ${modelPath}\n`);
|
|
136
|
+
try {
|
|
137
|
+
await downloadFile(url, modelPath);
|
|
138
|
+
} catch (err) {
|
|
139
|
+
const hint = err instanceof Error ? err.message : String(err);
|
|
140
|
+
throw new Error(
|
|
141
|
+
`Failed to download YOLOv8n model from ${url}: ${hint}. ` +
|
|
142
|
+
`Set AGENTICROS_YOLOV8_MODEL to a local path or AGENTICROS_YOLOV8_URL to an accessible mirror.`,
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
return modelPath;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function iou(a: PersonDetection, b: PersonDetection): number {
|
|
149
|
+
const ax2 = a.x + a.width;
|
|
150
|
+
const ay2 = a.y + a.height;
|
|
151
|
+
const bx2 = b.x + b.width;
|
|
152
|
+
const by2 = b.y + b.height;
|
|
153
|
+
const ix1 = Math.max(a.x, b.x);
|
|
154
|
+
const iy1 = Math.max(a.y, b.y);
|
|
155
|
+
const ix2 = Math.min(ax2, bx2);
|
|
156
|
+
const iy2 = Math.min(ay2, by2);
|
|
157
|
+
const iw = Math.max(0, ix2 - ix1);
|
|
158
|
+
const ih = Math.max(0, iy2 - iy1);
|
|
159
|
+
const inter = iw * ih;
|
|
160
|
+
const union = a.width * a.height + b.width * b.height - inter;
|
|
161
|
+
return union <= 0 ? 0 : inter / union;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function nms(detections: PersonDetection[], iouThreshold: number): PersonDetection[] {
|
|
165
|
+
const sorted = detections.slice().sort((a, b) => b.confidence - a.confidence);
|
|
166
|
+
const kept: PersonDetection[] = [];
|
|
167
|
+
for (const d of sorted) {
|
|
168
|
+
if (kept.every((k) => iou(d, k) < iouThreshold)) kept.push(d);
|
|
169
|
+
}
|
|
170
|
+
return kept;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export class PersonDetector {
|
|
174
|
+
private session: OrtInferenceSession | null = null;
|
|
175
|
+
private readonly scoreThreshold: number;
|
|
176
|
+
private readonly iouThreshold: number;
|
|
177
|
+
|
|
178
|
+
constructor(opts: DetectorOptions = {}) {
|
|
179
|
+
this.scoreThreshold = opts.scoreThreshold ?? 0.4;
|
|
180
|
+
this.iouThreshold = opts.iouThreshold ?? 0.5;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async load(): Promise<void> {
|
|
184
|
+
if (this.session) return;
|
|
185
|
+
const { ort: ortMod } = await loadDeps();
|
|
186
|
+
const modelPath = await ensureModel();
|
|
187
|
+
this.session = await ortMod.InferenceSession.create(modelPath, {
|
|
188
|
+
executionProviders: ["cpu"],
|
|
189
|
+
graphOptimizationLevel: "all",
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Detect people in a JPEG/PNG image buffer.
|
|
195
|
+
*
|
|
196
|
+
* Returns bounding boxes in the original image's pixel space.
|
|
197
|
+
*/
|
|
198
|
+
async detect(image: Buffer | Uint8Array): Promise<{ width: number; height: number; persons: PersonDetection[] }> {
|
|
199
|
+
const r = await this.detectClass(image, PERSON_CLASS_ID);
|
|
200
|
+
return { width: r.width, height: r.height, persons: r.detections };
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Detect a single COCO class (0..79) in a JPEG/PNG image buffer.
|
|
205
|
+
* 0=person, 67=cell phone, 56=chair, ... see find-object/coco-classes.ts.
|
|
206
|
+
*/
|
|
207
|
+
async detectClass(
|
|
208
|
+
image: Buffer | Uint8Array,
|
|
209
|
+
classId: number,
|
|
210
|
+
): Promise<{ width: number; height: number; detections: PersonDetection[] }> {
|
|
211
|
+
if (!this.session) await this.load();
|
|
212
|
+
const session = this.session!;
|
|
213
|
+
const { ort: ortMod, sharp: sharpFn } = await loadDeps();
|
|
214
|
+
|
|
215
|
+
const src = sharpFn(image);
|
|
216
|
+
const meta = await src.metadata();
|
|
217
|
+
const origW = meta.width ?? 0;
|
|
218
|
+
const origH = meta.height ?? 0;
|
|
219
|
+
if (!origW || !origH) {
|
|
220
|
+
throw new Error("Could not read image dimensions from camera frame.");
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// Letterbox resize to INPUT_SIZE × INPUT_SIZE (preserve aspect ratio, pad with gray).
|
|
224
|
+
const scale = Math.min(INPUT_SIZE / origW, INPUT_SIZE / origH);
|
|
225
|
+
const newW = Math.round(origW * scale);
|
|
226
|
+
const newH = Math.round(origH * scale);
|
|
227
|
+
const padX = Math.floor((INPUT_SIZE - newW) / 2);
|
|
228
|
+
const padY = Math.floor((INPUT_SIZE - newH) / 2);
|
|
229
|
+
|
|
230
|
+
const { data, info } = await sharpFn(image)
|
|
231
|
+
.resize(newW, newH, { fit: "fill" })
|
|
232
|
+
.extend({
|
|
233
|
+
top: padY,
|
|
234
|
+
bottom: INPUT_SIZE - newH - padY,
|
|
235
|
+
left: padX,
|
|
236
|
+
right: INPUT_SIZE - newW - padX,
|
|
237
|
+
background: { r: 114, g: 114, b: 114 },
|
|
238
|
+
})
|
|
239
|
+
.removeAlpha()
|
|
240
|
+
.raw()
|
|
241
|
+
.toBuffer({ resolveWithObject: true });
|
|
242
|
+
|
|
243
|
+
if (info.width !== INPUT_SIZE || info.height !== INPUT_SIZE) {
|
|
244
|
+
throw new Error(`Letterbox produced ${info.width}×${info.height}, expected ${INPUT_SIZE}²`);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
// HWC uint8 → CHW float32 normalized [0,1].
|
|
248
|
+
const pixels = INPUT_SIZE * INPUT_SIZE;
|
|
249
|
+
const input = new Float32Array(3 * pixels);
|
|
250
|
+
for (let i = 0; i < pixels; i++) {
|
|
251
|
+
input[i] = data[i * 3]! / 255;
|
|
252
|
+
input[pixels + i] = data[i * 3 + 1]! / 255;
|
|
253
|
+
input[2 * pixels + i] = data[i * 3 + 2]! / 255;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const inputName = session.inputNames[0]!;
|
|
257
|
+
const outputName = session.outputNames[0]!;
|
|
258
|
+
const tensor = new ortMod.Tensor("float32", input, [1, 3, INPUT_SIZE, INPUT_SIZE]);
|
|
259
|
+
const out = await session.run({ [inputName]: tensor });
|
|
260
|
+
const result = out[outputName]!;
|
|
261
|
+
// YOLOv8 ONNX output: [1, 84, 8400] — 4 box + 80 class scores per anchor.
|
|
262
|
+
const dims = result.dims;
|
|
263
|
+
if (dims.length !== 3 || dims[1] !== 84) {
|
|
264
|
+
throw new Error(`Unexpected YOLOv8 output shape ${dims.join("x")} — expected [1,84,N]`);
|
|
265
|
+
}
|
|
266
|
+
const nAnchors = dims[2]!;
|
|
267
|
+
const arr = result.data as Float32Array;
|
|
268
|
+
|
|
269
|
+
const raw: PersonDetection[] = [];
|
|
270
|
+
for (let i = 0; i < nAnchors; i++) {
|
|
271
|
+
const score = arr[(4 + classId) * nAnchors + i]!;
|
|
272
|
+
if (score < this.scoreThreshold) continue;
|
|
273
|
+
const cx = arr[0 * nAnchors + i]!;
|
|
274
|
+
const cy = arr[1 * nAnchors + i]!;
|
|
275
|
+
const w = arr[2 * nAnchors + i]!;
|
|
276
|
+
const h = arr[3 * nAnchors + i]!;
|
|
277
|
+
// Undo letterbox: subtract padding, divide by scale → original-image coordinates.
|
|
278
|
+
const x = (cx - w / 2 - padX) / scale;
|
|
279
|
+
const y = (cy - h / 2 - padY) / scale;
|
|
280
|
+
const ww = w / scale;
|
|
281
|
+
const hh = h / scale;
|
|
282
|
+
raw.push({
|
|
283
|
+
x: Math.max(0, x),
|
|
284
|
+
y: Math.max(0, y),
|
|
285
|
+
width: Math.min(origW - x, ww),
|
|
286
|
+
height: Math.min(origH - y, hh),
|
|
287
|
+
cx: (cx - padX) / scale,
|
|
288
|
+
cy: (cy - padY) / scale,
|
|
289
|
+
confidence: score,
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const detections = nms(raw, this.iouThreshold);
|
|
294
|
+
return { width: origW, height: origH, detections };
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
async dispose(): Promise<void> {
|
|
298
|
+
if (this.session) {
|
|
299
|
+
await this.session.release();
|
|
300
|
+
this.session = null;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
}
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* In-process follow-me loop. Subscribes to color + depth, runs YOLOv8n person
|
|
3
|
+
* detection, computes a Twist via FollowerController, and publishes cmd_vel.
|
|
4
|
+
*
|
|
5
|
+
* This is the `mode: 'local'` alternative to the agenticros_follow_me ROS2 node.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { AgenticROSConfig, RosTransport, Subscription } from "@agenticros/core";
|
|
9
|
+
import { resolveCameraSubscribeTopic, toNamespacedTopic } from "@agenticros/core";
|
|
10
|
+
import {
|
|
11
|
+
ROS_MSG_COMPRESSED_IMAGE,
|
|
12
|
+
ROS_MSG_IMAGE,
|
|
13
|
+
cameraSnapshotFromPlainMessage,
|
|
14
|
+
coerceRosImageDataToBuffer,
|
|
15
|
+
normalizeDepthImageEncoding,
|
|
16
|
+
rosBoolField,
|
|
17
|
+
rosNumericField,
|
|
18
|
+
rosStringField,
|
|
19
|
+
} from "@agenticros/ros-camera";
|
|
20
|
+
import { PersonDetector, type PersonDetection } from "./detector.js";
|
|
21
|
+
import { FollowerController, type Twist } from "./controller.js";
|
|
22
|
+
|
|
23
|
+
const DEFAULT_COLOR_TOPIC = "/camera/camera/color/image_raw/compressed";
|
|
24
|
+
const DEFAULT_DEPTH_TOPIC = "/camera/camera/depth/image_rect_raw";
|
|
25
|
+
const DEFAULT_HORIZONTAL_FOV_RAD = 1.20428; // RealSense D435 color HFOV ≈ 69°
|
|
26
|
+
const TICK_HZ = 8;
|
|
27
|
+
|
|
28
|
+
export interface FollowMeLocalStatus {
|
|
29
|
+
enabled: boolean;
|
|
30
|
+
tracking: boolean;
|
|
31
|
+
targetDistance: number;
|
|
32
|
+
targetDescription: string | null;
|
|
33
|
+
personCount: number;
|
|
34
|
+
lastPerson: { x: number; z: number; confidence: number } | null;
|
|
35
|
+
lastTwist: Twist;
|
|
36
|
+
lastError: string | null;
|
|
37
|
+
framesProcessed: number;
|
|
38
|
+
detectionsSinceStart: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export interface StartOptions {
|
|
42
|
+
targetDescription?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
interface LatestFrame {
|
|
46
|
+
buffer: Buffer;
|
|
47
|
+
receivedAt: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface LatestDepth {
|
|
51
|
+
width: number;
|
|
52
|
+
height: number;
|
|
53
|
+
step: number;
|
|
54
|
+
encoding: string;
|
|
55
|
+
isBigEndian: boolean;
|
|
56
|
+
data: Uint8Array;
|
|
57
|
+
receivedAt: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export class FollowMeLocal {
|
|
61
|
+
private readonly detector = new PersonDetector();
|
|
62
|
+
private readonly controller = new FollowerController();
|
|
63
|
+
private enabled = false;
|
|
64
|
+
private running = false;
|
|
65
|
+
private targetDescription: string | null = null;
|
|
66
|
+
private colorSub: Subscription | null = null;
|
|
67
|
+
private depthSub: Subscription | null = null;
|
|
68
|
+
private latestColor: LatestFrame | null = null;
|
|
69
|
+
private latestDepth: LatestDepth | null = null;
|
|
70
|
+
private tickHandle: NodeJS.Timeout | null = null;
|
|
71
|
+
private lastError: string | null = null;
|
|
72
|
+
private framesProcessed = 0;
|
|
73
|
+
private detectionsSinceStart = 0;
|
|
74
|
+
private lastPerson: { x: number; z: number; confidence: number } | null = null;
|
|
75
|
+
private tracking = false;
|
|
76
|
+
|
|
77
|
+
constructor(
|
|
78
|
+
private readonly config: AgenticROSConfig,
|
|
79
|
+
private readonly transport: RosTransport,
|
|
80
|
+
) {}
|
|
81
|
+
|
|
82
|
+
async start(opts: StartOptions = {}): Promise<void> {
|
|
83
|
+
if (this.enabled) return;
|
|
84
|
+
this.lastError = null;
|
|
85
|
+
this.framesProcessed = 0;
|
|
86
|
+
this.detectionsSinceStart = 0;
|
|
87
|
+
this.targetDescription = opts.targetDescription?.trim() || null;
|
|
88
|
+
this.controller.reset();
|
|
89
|
+
await this.detector.load();
|
|
90
|
+
this.subscribeFrames();
|
|
91
|
+
this.enabled = true;
|
|
92
|
+
this.tickHandle = setInterval(() => this.tickSafe(), 1000 / TICK_HZ);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
async stop(): Promise<void> {
|
|
96
|
+
this.enabled = false;
|
|
97
|
+
if (this.tickHandle) {
|
|
98
|
+
clearInterval(this.tickHandle);
|
|
99
|
+
this.tickHandle = null;
|
|
100
|
+
}
|
|
101
|
+
this.unsubscribeFrames();
|
|
102
|
+
this.controller.reset();
|
|
103
|
+
this.tracking = false;
|
|
104
|
+
this.lastPerson = null;
|
|
105
|
+
await this.publishStop();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
setTargetDistance(d: number): void {
|
|
109
|
+
this.controller.setTargetDistance(d);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
setTargetDescription(description: string): void {
|
|
113
|
+
this.targetDescription = description.trim() || null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
status(): FollowMeLocalStatus {
|
|
117
|
+
return {
|
|
118
|
+
enabled: this.enabled,
|
|
119
|
+
tracking: this.tracking,
|
|
120
|
+
targetDistance: this.controller.config.targetDistance,
|
|
121
|
+
targetDescription: this.targetDescription,
|
|
122
|
+
personCount: this.lastPerson ? 1 : 0,
|
|
123
|
+
lastPerson: this.lastPerson,
|
|
124
|
+
lastTwist: this.controller.getLastTwist(),
|
|
125
|
+
lastError: this.lastError,
|
|
126
|
+
framesProcessed: this.framesProcessed,
|
|
127
|
+
detectionsSinceStart: this.detectionsSinceStart,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
private subscribeFrames(): void {
|
|
132
|
+
const colorTopicRaw = (this.config.robot?.cameraTopic ?? "").trim() || DEFAULT_COLOR_TOPIC;
|
|
133
|
+
const colorTopic = resolveCameraSubscribeTopic(this.config, colorTopicRaw);
|
|
134
|
+
const depthTopic = resolveCameraSubscribeTopic(this.config, DEFAULT_DEPTH_TOPIC);
|
|
135
|
+
const isCompressed = colorTopic.includes("compressed");
|
|
136
|
+
|
|
137
|
+
this.colorSub = this.transport.subscribe(
|
|
138
|
+
{ topic: colorTopic, type: isCompressed ? ROS_MSG_COMPRESSED_IMAGE : ROS_MSG_IMAGE },
|
|
139
|
+
(msg) => {
|
|
140
|
+
try {
|
|
141
|
+
const payload = cameraSnapshotFromPlainMessage(isCompressed ? "CompressedImage" : "Image", msg);
|
|
142
|
+
const buf = Buffer.from(payload.dataBase64, "base64");
|
|
143
|
+
this.latestColor = { buffer: buf, receivedAt: Date.now() };
|
|
144
|
+
} catch (err) {
|
|
145
|
+
this.lastError = `color decode: ${err instanceof Error ? err.message : String(err)}`;
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
);
|
|
149
|
+
|
|
150
|
+
this.depthSub = this.transport.subscribe(
|
|
151
|
+
{ topic: depthTopic, type: ROS_MSG_IMAGE },
|
|
152
|
+
(msg) => {
|
|
153
|
+
try {
|
|
154
|
+
const encoding = normalizeDepthImageEncoding(rosStringField(msg.encoding, "16UC1"));
|
|
155
|
+
const width = rosNumericField(msg.width, "width");
|
|
156
|
+
const height = rosNumericField(msg.height, "height");
|
|
157
|
+
const bpp = encoding === "32FC1" ? 4 : 2;
|
|
158
|
+
const step =
|
|
159
|
+
msg.step != null && msg.step !== "" ? rosNumericField(msg.step, "step") : width * bpp;
|
|
160
|
+
const isBigEndian = rosBoolField(msg.is_bigendian);
|
|
161
|
+
const data = new Uint8Array(coerceRosImageDataToBuffer(msg.data));
|
|
162
|
+
this.latestDepth = { width, height, step, encoding, isBigEndian, data, receivedAt: Date.now() };
|
|
163
|
+
} catch (err) {
|
|
164
|
+
this.lastError = `depth decode: ${err instanceof Error ? err.message : String(err)}`;
|
|
165
|
+
}
|
|
166
|
+
},
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
private unsubscribeFrames(): void {
|
|
171
|
+
this.colorSub?.unsubscribe();
|
|
172
|
+
this.depthSub?.unsubscribe();
|
|
173
|
+
this.colorSub = null;
|
|
174
|
+
this.depthSub = null;
|
|
175
|
+
this.latestColor = null;
|
|
176
|
+
this.latestDepth = null;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
private async tickSafe(): Promise<void> {
|
|
180
|
+
if (this.running || !this.enabled) return;
|
|
181
|
+
this.running = true;
|
|
182
|
+
try {
|
|
183
|
+
await this.tick();
|
|
184
|
+
} catch (err) {
|
|
185
|
+
this.lastError = err instanceof Error ? err.message : String(err);
|
|
186
|
+
} finally {
|
|
187
|
+
this.running = false;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
private async tick(): Promise<void> {
|
|
192
|
+
const color = this.latestColor;
|
|
193
|
+
if (!color) return; // wait for first frame
|
|
194
|
+
if (Date.now() - color.receivedAt > 2000) {
|
|
195
|
+
// stale → controller watchdog handles cmd_vel zeroing
|
|
196
|
+
const twist = this.controller.update(null);
|
|
197
|
+
await this.publishTwist(twist);
|
|
198
|
+
this.tracking = false;
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
this.framesProcessed += 1;
|
|
203
|
+
const det = await this.detector.detect(color.buffer);
|
|
204
|
+
const persons = det.persons;
|
|
205
|
+
if (persons.length === 0) {
|
|
206
|
+
this.tracking = false;
|
|
207
|
+
this.lastPerson = null;
|
|
208
|
+
const twist = this.controller.update(null);
|
|
209
|
+
await this.publishTwist(twist);
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Pick the largest bounding box (closest person). target_description is parsed but not
|
|
214
|
+
// semantically matched yet — re-identification by description requires a CLIP-style model.
|
|
215
|
+
const target = pickTarget(persons);
|
|
216
|
+
const sample = this.projectTo3D(target, det.width, det.height);
|
|
217
|
+
if (!sample) {
|
|
218
|
+
this.tracking = false;
|
|
219
|
+
const twist = this.controller.update(null);
|
|
220
|
+
await this.publishTwist(twist);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
this.tracking = true;
|
|
225
|
+
this.detectionsSinceStart += 1;
|
|
226
|
+
this.lastPerson = { x: sample.x, z: sample.z, confidence: target.confidence };
|
|
227
|
+
const twist = this.controller.update(sample);
|
|
228
|
+
await this.publishTwist(twist);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
private projectTo3D(
|
|
232
|
+
person: PersonDetection,
|
|
233
|
+
imgW: number,
|
|
234
|
+
imgH: number,
|
|
235
|
+
): { x: number; z: number; confidence: number } | null {
|
|
236
|
+
const depth = this.latestDepth;
|
|
237
|
+
if (!depth || Date.now() - depth.receivedAt > 2000) return null;
|
|
238
|
+
|
|
239
|
+
// Map person centre from color-image space to depth-image space (assumes both share FOV
|
|
240
|
+
// and aspect ratio; RealSense color/depth are aligned closely enough for this).
|
|
241
|
+
const u = (person.cx / imgW) * depth.width;
|
|
242
|
+
const v = (person.cy / imgH) * depth.height;
|
|
243
|
+
const sampleHalf = Math.max(4, Math.floor(Math.min(person.width, person.height) * 0.1 *
|
|
244
|
+
(depth.width / imgW) * 0.5));
|
|
245
|
+
const x0 = Math.max(0, Math.floor(u - sampleHalf));
|
|
246
|
+
const x1 = Math.min(depth.width, Math.floor(u + sampleHalf));
|
|
247
|
+
const y0 = Math.max(0, Math.floor(v - sampleHalf));
|
|
248
|
+
const y1 = Math.min(depth.height, Math.floor(v + sampleHalf));
|
|
249
|
+
const samples = sampleDepthRegion(depth, x0, y0, x1, y1);
|
|
250
|
+
if (samples.length === 0) return null;
|
|
251
|
+
|
|
252
|
+
samples.sort((a, b) => a - b);
|
|
253
|
+
// Use the 25th percentile to bias toward the person surface (not background bleed).
|
|
254
|
+
const z = samples[Math.floor(samples.length * 0.25)] ?? samples[0]!;
|
|
255
|
+
if (!Number.isFinite(z) || z <= 0) return null;
|
|
256
|
+
|
|
257
|
+
// Lateral offset from horizontal FOV. Image-centre is 0; right is positive x.
|
|
258
|
+
const normX = person.cx / imgW - 0.5;
|
|
259
|
+
const x = z * Math.tan(normX * DEFAULT_HORIZONTAL_FOV_RAD);
|
|
260
|
+
return { x, z, confidence: person.confidence };
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
private async publishTwist(twist: Twist): Promise<void> {
|
|
264
|
+
const topic = resolveCmdVelTopic(this.config);
|
|
265
|
+
const message = {
|
|
266
|
+
linear: { x: twist.linearX, y: 0, z: 0 },
|
|
267
|
+
angular: { x: 0, y: 0, z: twist.angularZ },
|
|
268
|
+
};
|
|
269
|
+
try {
|
|
270
|
+
await this.transport.publish({ topic, type: "geometry_msgs/msg/Twist", msg: message });
|
|
271
|
+
} catch (err) {
|
|
272
|
+
this.lastError = `cmd_vel publish: ${err instanceof Error ? err.message : String(err)}`;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
private async publishStop(): Promise<void> {
|
|
277
|
+
await this.publishTwist({ linearX: 0, angularZ: 0 });
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function pickTarget(persons: PersonDetection[]): PersonDetection {
|
|
282
|
+
return persons.reduce((best, p) => (p.width * p.height > best.width * best.height ? p : best));
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
function sampleDepthRegion(d: LatestDepth, x0: number, y0: number, x1: number, y1: number): number[] {
|
|
286
|
+
// Reuse sampleDepthMeters by passing a custom center-fraction. Easier: inline a copy of the
|
|
287
|
+
// 16UC1/32FC1 decode here for an arbitrary rect.
|
|
288
|
+
const out: number[] = [];
|
|
289
|
+
if (d.encoding === "16UC1") {
|
|
290
|
+
for (let y = y0; y < y1; y++) {
|
|
291
|
+
for (let x = x0; x < x1; x++) {
|
|
292
|
+
const off = y * d.step + x * 2;
|
|
293
|
+
if (off + 2 > d.data.length) continue;
|
|
294
|
+
const lo = d.data[off]!;
|
|
295
|
+
const hi = d.data[off + 1]!;
|
|
296
|
+
const v = d.isBigEndian ? (lo << 8) | hi : (hi << 8) | lo;
|
|
297
|
+
if (v > 0) out.push(v / 1000);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
} else if (d.encoding === "32FC1") {
|
|
301
|
+
for (let y = y0; y < y1; y++) {
|
|
302
|
+
for (let x = x0; x < x1; x++) {
|
|
303
|
+
const off = y * d.step + x * 4;
|
|
304
|
+
if (off + 4 > d.data.length) continue;
|
|
305
|
+
const v = new DataView(d.data.buffer, d.data.byteOffset + off, 4).getFloat32(0, !d.isBigEndian);
|
|
306
|
+
if (Number.isFinite(v) && v > 0) out.push(v);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return out;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function resolveCmdVelTopic(config: AgenticROSConfig): string {
|
|
314
|
+
const raw = (config.teleop?.cmdVelTopic ?? "").trim() || "/cmd_vel";
|
|
315
|
+
const namespaced = toNamespacedTopic(config, raw);
|
|
316
|
+
// Apply same uuid → robot<uuid-no-dashes> rewrite as ros2_publish handler.
|
|
317
|
+
const match = namespaced.match(/^\/([^/]+)\/cmd_vel$/i);
|
|
318
|
+
const segment = match?.[1] ?? "";
|
|
319
|
+
if (match && !segment.toLowerCase().startsWith("robot")) {
|
|
320
|
+
return `/robot${segment.replace(/-/g, "")}/cmd_vel`;
|
|
321
|
+
}
|
|
322
|
+
return namespaced;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
let singleton: FollowMeLocal | null = null;
|
|
326
|
+
|
|
327
|
+
export function getFollowMeLocal(config: AgenticROSConfig, transport: RosTransport): FollowMeLocal {
|
|
328
|
+
if (!singleton) singleton = new FollowMeLocal(config, transport);
|
|
329
|
+
return singleton;
|
|
330
|
+
}
|