agenticros 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.md +18 -3
  2. package/dist/commands/doctor.d.ts.map +1 -1
  3. package/dist/commands/doctor.js +34 -1
  4. package/dist/commands/doctor.js.map +1 -1
  5. package/dist/commands/skills.d.ts +3 -5
  6. package/dist/commands/skills.d.ts.map +1 -1
  7. package/dist/commands/skills.js +21 -7
  8. package/dist/commands/skills.js.map +1 -1
  9. package/dist/util/skill-manifest.d.ts.map +1 -1
  10. package/dist/util/skill-manifest.js +22 -1
  11. package/dist/util/skill-manifest.js.map +1 -1
  12. package/dist/util/skills.d.ts +6 -0
  13. package/dist/util/skills.d.ts.map +1 -1
  14. package/dist/util/skills.js +69 -1
  15. package/dist/util/skills.js.map +1 -1
  16. package/package.json +2 -1
  17. package/runtime/BUNDLE.json +1 -1
  18. package/runtime/README.md +31 -6
  19. package/runtime/docs/cli.md +1 -1
  20. package/runtime/docs/robot-setup.md +1 -1
  21. package/runtime/packages/agenticros/openclaw.plugin.json +147 -1
  22. package/runtime/packages/agenticros/src/__tests__/capabilities-plugin.test.ts +2 -0
  23. package/runtime/packages/agenticros/src/index.ts +8 -1
  24. package/runtime/packages/agenticros/src/routes.ts +4 -3
  25. package/runtime/packages/agenticros/src/tools/index.ts +6 -5
  26. package/runtime/packages/agenticros/src/tools/mission-pause.ts +49 -0
  27. package/runtime/packages/agenticros/src/tools/mission-resume.ts +41 -0
  28. package/runtime/packages/agenticros/src/tools/ros2-capabilities.ts +9 -12
  29. package/runtime/packages/agenticros/src/tools/ros2-mission.ts +33 -84
  30. package/runtime/packages/agenticros-claude-code/README.md +2 -0
  31. package/runtime/packages/agenticros-claude-code/dist/config.d.ts +5 -3
  32. package/runtime/packages/agenticros-claude-code/dist/config.d.ts.map +1 -1
  33. package/runtime/packages/agenticros-claude-code/dist/config.js +32 -10
  34. package/runtime/packages/agenticros-claude-code/dist/config.js.map +1 -1
  35. package/runtime/packages/agenticros-claude-code/dist/index.js +3 -3
  36. package/runtime/packages/agenticros-claude-code/dist/index.js.map +1 -1
  37. package/runtime/packages/agenticros-claude-code/dist/tools.d.ts.map +1 -1
  38. package/runtime/packages/agenticros-claude-code/dist/tools.js +137 -118
  39. package/runtime/packages/agenticros-claude-code/dist/tools.js.map +1 -1
  40. package/runtime/packages/agenticros-claude-code/src/config.ts +33 -10
  41. package/runtime/packages/agenticros-claude-code/src/index.ts +3 -3
  42. package/runtime/packages/agenticros-claude-code/src/tools.ts +151 -107
  43. package/runtime/packages/agenticros-gemini/package.json +2 -0
  44. package/runtime/packages/agenticros-gemini/src/config.ts +27 -18
  45. package/runtime/packages/agenticros-gemini/src/find-object/coco-classes.ts +38 -0
  46. package/runtime/packages/agenticros-gemini/src/find-object/find-object.ts +191 -0
  47. package/runtime/packages/agenticros-gemini/src/follow-me/controller.ts +109 -0
  48. package/runtime/packages/agenticros-gemini/src/follow-me/depth-loop.ts +452 -0
  49. package/runtime/packages/agenticros-gemini/src/follow-me/detector.ts +303 -0
  50. package/runtime/packages/agenticros-gemini/src/follow-me/loop.ts +359 -0
  51. package/runtime/packages/agenticros-gemini/src/index.ts +2 -2
  52. package/runtime/packages/agenticros-gemini/src/tools.ts +356 -92
  53. package/runtime/packages/core/README.md +1 -1
  54. package/runtime/packages/core/package.json +1 -1
  55. package/runtime/packages/core/src/__tests__/config-persistence.test.ts +26 -0
  56. package/runtime/packages/core/src/__tests__/external-capability.test.ts +72 -0
  57. package/runtime/packages/core/src/__tests__/heartbeat-fleet.test.ts +152 -0
  58. package/runtime/packages/core/src/__tests__/mission-bindings.test.ts +101 -0
  59. package/runtime/packages/core/src/__tests__/mission-pause.test.ts +104 -0
  60. package/runtime/packages/core/src/__tests__/skill-refs.test.ts +77 -0
  61. package/runtime/packages/core/src/capabilities.ts +38 -3
  62. package/runtime/packages/core/src/capability-schema.ts +58 -0
  63. package/runtime/packages/core/src/config.ts +28 -0
  64. package/runtime/packages/core/src/discoverable-capabilities.ts +178 -0
  65. package/runtime/packages/core/src/discovery.ts +21 -3
  66. package/runtime/packages/core/src/external-capability.ts +232 -0
  67. package/runtime/packages/core/src/fleet-config.ts +91 -0
  68. package/runtime/packages/core/src/heartbeat.ts +167 -0
  69. package/runtime/packages/core/src/index.ts +85 -0
  70. package/runtime/packages/core/src/mission-bindings.ts +189 -0
  71. package/runtime/packages/core/src/mission-registry.ts +31 -3
  72. package/runtime/packages/core/src/mission.ts +64 -7
  73. package/runtime/packages/core/src/robots.ts +6 -2
  74. package/runtime/packages/core/src/skill-refs.ts +330 -0
  75. package/runtime/packages/core/src/transport/rosbridge/client.ts +7 -0
  76. package/runtime/pnpm-lock.yaml +9 -0
  77. package/templates/skills/camera/package.json +1 -1
  78. package/templates/skills/depth/package.json +1 -1
  79. package/templates/skills/robot/package.json +1 -1
@@ -7,7 +7,6 @@ import type {
7
7
  Capability,
8
8
  Mission,
9
9
  MissionToolDispatcher,
10
- CapabilityToolBindings,
11
10
  ResolvedRobot,
12
11
  } from "@agenticros/core";
13
12
  import {
@@ -15,6 +14,7 @@ import {
15
14
  toNamespacedTopic,
16
15
  toNamespacedTopicFull,
17
16
  listAllCapabilities,
17
+ listCapabilitiesWithDiscoverable,
18
18
  runMission,
19
19
  listRobots,
20
20
  getActiveRobotId,
@@ -26,6 +26,10 @@ import {
26
26
  createMemoryTranscriptSink,
27
27
  missionTranscriptNamespace,
28
28
  compileGoalToMission,
29
+ buildMissionBindings,
30
+ isExternalToolName,
31
+ capabilityIdFromExternalTool,
32
+ executeExternalCapability,
29
33
  } from "@agenticros/core";
30
34
  import { getMissionRegistry } from "./mission-registry.js";
31
35
  import {
@@ -73,9 +77,12 @@ export const NO_TRANSPORT_TOOL_NAMES = new Set<string>([
73
77
  // tool falls back to a transport-driven discovery only when the caller
74
78
  // passes online=true. See the handler for the precise branch.
75
79
  "ros2_find_robots_for",
76
- // mission_cancel only mutates the in-process MissionRegistry it
77
- // never publishes to ROS, so it must work even when zenohd is down.
80
+ // mission_cancel / mission_pause / mission_resume only mutate the
81
+ // in-process MissionRegistry — they never publish to ROS, so they
82
+ // must work even when zenohd is down.
78
83
  "mission_cancel",
84
+ "mission_pause",
85
+ "mission_resume",
79
86
  // run_mission's *outer* call doesn't need transport — its handler
80
87
  // first compiles the goal (Phase 1.g, no transport) or validates
81
88
  // the mission shape, then per-step calls go through the transport
@@ -464,6 +471,40 @@ export const TOOLS: McpTool[] = [
464
471
  required: ["mission_id"],
465
472
  },
466
473
  },
474
+ {
475
+ name: "mission_pause",
476
+ description:
477
+ "Pause a mission that's currently running in this MCP server. Pass the mission_id returned by run_mission. The runner waits at the next step boundary until mission_resume (or mission_cancel). Idempotent if already paused.",
478
+ inputSchema: {
479
+ type: "object",
480
+ properties: {
481
+ mission_id: {
482
+ type: "string",
483
+ description: "The mission_id echoed by run_mission. Required.",
484
+ },
485
+ reason: {
486
+ type: "string",
487
+ description: "Optional free-text reason — surfaced in the paused transcript entry.",
488
+ },
489
+ },
490
+ required: ["mission_id"],
491
+ },
492
+ },
493
+ {
494
+ name: "mission_resume",
495
+ description:
496
+ "Resume a mission previously paused with mission_pause. Pass the mission_id returned by run_mission. Idempotent if the mission is not paused.",
497
+ inputSchema: {
498
+ type: "object",
499
+ properties: {
500
+ mission_id: {
501
+ type: "string",
502
+ description: "The mission_id echoed by run_mission. Required.",
503
+ },
504
+ },
505
+ required: ["mission_id"],
506
+ },
507
+ },
467
508
  ];
468
509
 
469
510
  export type ToolContent = { type: "text"; text: string } | { type: "image"; data: string; mimeType: string };
@@ -509,16 +550,18 @@ async function publishFollowMeCmd(
509
550
  * Format the capability list for tool responses — shared by claude-code,
510
551
  * OpenClaw, and Gemini so every adapter returns the same JSON shape.
511
552
  */
512
- function formatCapabilitiesResponse(caps: Capability[]): string {
513
- const byKind = {
514
- intrinsic: caps.filter((c) => c.source?.kind === "builtin").length,
515
- skill: caps.filter((c) => c.source?.kind === "skill").length,
516
- };
553
+ function formatCapabilitiesResponse(
554
+ caps: Array<Capability & { discoverable?: boolean; installed?: boolean; install_ref?: string }>,
555
+ ): string {
556
+ const intrinsic = caps.filter((c) => c.source?.kind === "builtin").length;
557
+ const skill = caps.filter((c) => c.installed !== false && c.source?.kind === "skill").length;
558
+ const discoverable = caps.filter((c) => c.discoverable === true).length;
517
559
  return JSON.stringify({
518
560
  success: true,
519
561
  total: caps.length,
520
- intrinsic_count: byKind.intrinsic,
521
- skill_count: byKind.skill,
562
+ intrinsic_count: intrinsic,
563
+ skill_count: skill,
564
+ discoverable_count: discoverable,
522
565
  capabilities: caps,
523
566
  });
524
567
  }
@@ -573,95 +616,6 @@ function formatFindRobotsForResponse(result: FindRobotsForResult): string {
573
616
  });
574
617
  }
575
618
 
576
- /**
577
- * Capability → MCP tool dispatch table used by `run_mission`.
578
- *
579
- * Every capability returned by `ros2_list_capabilities` that the mission
580
- * runner can actually execute must appear here. If you add a new
581
- * intrinsic capability to BUILTIN_CAPABILITIES, mirror it here. For new
582
- * skill-declared capabilities, this is where you map them to the
583
- * corresponding MCP tool. Steps that name a capability without an entry
584
- * fail with an actionable "no mission-runner tool binding" error.
585
- */
586
- const MISSION_BINDINGS: CapabilityToolBindings = {
587
- drive_base: {
588
- tool: "ros2_publish",
589
- buildArgs: (inputs) => {
590
- const lx = Number(inputs.linear_x ?? 0) || 0;
591
- const az = Number(inputs.angular_z ?? 0) || 0;
592
- return {
593
- topic: "/cmd_vel",
594
- type: "geometry_msgs/msg/Twist",
595
- message: {
596
- linear: { x: lx, y: 0, z: 0 },
597
- angular: { x: 0, y: 0, z: az },
598
- },
599
- };
600
- },
601
- },
602
- take_snapshot: {
603
- tool: "ros2_camera_snapshot",
604
- buildArgs: (inputs) => {
605
- const out: Record<string, unknown> = {};
606
- if (typeof inputs.topic === "string") out.topic = inputs.topic;
607
- if (typeof inputs.message_type === "string") out.message_type = inputs.message_type;
608
- if (typeof inputs.timeout === "number") out.timeout = inputs.timeout;
609
- return out;
610
- },
611
- },
612
- measure_depth: {
613
- tool: "ros2_depth_distance",
614
- buildArgs: (inputs) => {
615
- const out: Record<string, unknown> = {};
616
- if (typeof inputs.topic === "string") out.topic = inputs.topic;
617
- if (typeof inputs.timeout === "number") out.timeout = inputs.timeout;
618
- return out;
619
- },
620
- },
621
- list_topics: {
622
- tool: "ros2_list_topics",
623
- buildArgs: () => ({}),
624
- },
625
- publish_topic: {
626
- tool: "ros2_publish",
627
- buildArgs: (inputs) => ({
628
- topic: String(inputs.topic ?? ""),
629
- type: String(inputs.type ?? inputs.msg_type ?? ""),
630
- message: inputs.message ?? inputs.msg ?? {},
631
- }),
632
- },
633
- subscribe_once: {
634
- tool: "ros2_subscribe_once",
635
- buildArgs: (inputs) => {
636
- const out: Record<string, unknown> = { topic: String(inputs.topic ?? "") };
637
- if (typeof inputs.type === "string") out.type = inputs.type;
638
- if (typeof inputs.timeout === "number") out.timeout = inputs.timeout;
639
- return out;
640
- },
641
- },
642
- follow_person: {
643
- tool: "ros2_follow_me_start",
644
- buildArgs: (inputs) => {
645
- const out: Record<string, unknown> = {};
646
- if (typeof inputs.target_distance === "number") out.target_distance = inputs.target_distance;
647
- if (typeof inputs.mode === "string") out.mode = inputs.mode;
648
- return out;
649
- },
650
- },
651
- find_object: {
652
- tool: "ros2_find_object",
653
- buildArgs: (inputs) => {
654
- const target = String(inputs.target ?? "");
655
- const out: Record<string, unknown> = { target };
656
- if (typeof inputs.angular_speed === "number") out.angular_speed = inputs.angular_speed;
657
- if (typeof inputs.clockwise === "boolean") out.clockwise = inputs.clockwise;
658
- if (typeof inputs.timeout_seconds === "number") out.timeout_seconds = inputs.timeout_seconds;
659
- if (typeof inputs.min_confidence === "number") out.min_confidence = inputs.min_confidence;
660
- return out;
661
- },
662
- },
663
- };
664
-
665
619
  /**
666
620
  * Phase 1.f — `mission_cancel` handler. Lives at module scope so the
667
621
  * outer `handleToolCall` can invoke it WITHOUT first resolving a
@@ -701,6 +655,70 @@ function handleMissionCancel(
701
655
  };
702
656
  }
703
657
 
658
+ function handleMissionPause(
659
+ args: Record<string, unknown>,
660
+ ): { content: ToolContent[]; isError?: boolean } {
661
+ const missionId = typeof args["mission_id"] === "string" ? args["mission_id"].trim() : "";
662
+ if (!missionId) {
663
+ return {
664
+ content: [
665
+ {
666
+ type: "text",
667
+ text: "mission_pause requires 'mission_id' (a non-empty string returned by run_mission).",
668
+ },
669
+ ],
670
+ isError: true,
671
+ };
672
+ }
673
+ const reason = typeof args["reason"] === "string" ? args["reason"] : undefined;
674
+ const outcome = getMissionRegistry().pause(missionId, reason);
675
+ return {
676
+ content: [
677
+ {
678
+ type: "text",
679
+ text: JSON.stringify({
680
+ success: true,
681
+ mission_id: missionId,
682
+ found: outcome.found,
683
+ already_paused: outcome.alreadyPaused,
684
+ reason: reason ?? null,
685
+ }),
686
+ },
687
+ ],
688
+ };
689
+ }
690
+
691
+ function handleMissionResume(
692
+ args: Record<string, unknown>,
693
+ ): { content: ToolContent[]; isError?: boolean } {
694
+ const missionId = typeof args["mission_id"] === "string" ? args["mission_id"].trim() : "";
695
+ if (!missionId) {
696
+ return {
697
+ content: [
698
+ {
699
+ type: "text",
700
+ text: "mission_resume requires 'mission_id' (a non-empty string returned by run_mission).",
701
+ },
702
+ ],
703
+ isError: true,
704
+ };
705
+ }
706
+ const outcome = getMissionRegistry().resume(missionId);
707
+ return {
708
+ content: [
709
+ {
710
+ type: "text",
711
+ text: JSON.stringify({
712
+ success: true,
713
+ mission_id: missionId,
714
+ found: outcome.found,
715
+ was_paused: outcome.wasPaused,
716
+ }),
717
+ },
718
+ ],
719
+ };
720
+ }
721
+
704
722
  /**
705
723
  * Phase 1.c + 1.f + 1.g — `run_mission` handler. Lives at module scope
706
724
  * so the outer `handleToolCall` can short-circuit BEFORE
@@ -778,6 +796,30 @@ async function handleRunMission(
778
796
  }
779
797
  }
780
798
  const dispatcher: MissionToolDispatcher = async (toolName, toolArgs) => {
799
+ if (isExternalToolName(toolName)) {
800
+ const capId = capabilityIdFromExternalTool(toolName);
801
+ const cap = caps.find((c) => c.id === capId);
802
+ if (!cap) {
803
+ return {
804
+ text: `Unknown external capability "${capId}".`,
805
+ isError: true,
806
+ };
807
+ }
808
+ let robot;
809
+ try {
810
+ robot = resolveRobotFromArgs(config, toolArgs);
811
+ } catch (err) {
812
+ return {
813
+ text: err instanceof Error ? err.message : String(err),
814
+ isError: true,
815
+ };
816
+ }
817
+ const transport = await getTransportForRobot(config, robot);
818
+ const ext = await executeExternalCapability(cap, toolArgs, transport, {
819
+ namespace: robot.namespace,
820
+ });
821
+ return { text: ext.text, outputs: ext.outputs, isError: ext.isError };
822
+ }
781
823
  const res = await handleToolCall(toolName, toolArgs, config);
782
824
  const text = res.content
783
825
  .map((c) => (c.type === "text" ? c.text : `[image: ${c.mimeType}]`))
@@ -796,7 +838,7 @@ async function handleRunMission(
796
838
 
797
839
  let result;
798
840
  try {
799
- result = await runMission(mission, caps, MISSION_BINDINGS, dispatcher, {
841
+ result = await runMission(mission, caps, buildMissionBindings(caps), dispatcher, {
800
842
  mission_id: missionId,
801
843
  cancellation: regEntry.cancellation,
802
844
  transcript,
@@ -852,7 +894,7 @@ export async function handleToolCall(
852
894
  } catch (err) {
853
895
  return robotResolveError(err);
854
896
  }
855
- const caps = listAllCapabilities(config);
897
+ const caps = await listCapabilitiesWithDiscoverable(config);
856
898
  return { content: [{ type: "text", text: formatCapabilitiesResponse(caps) }] };
857
899
  }
858
900
  // ros2_list_robots reads the multi-robot section of the config (with
@@ -915,15 +957,17 @@ export async function handleToolCall(
915
957
  };
916
958
  }
917
959
  }
918
- // mission_cancel + run_mission must NOT trigger a transport connect
919
- // at this outer layer: mission_cancel only mutates the in-process
920
- // MissionRegistry, and run_mission's handler does its OWN per-step
921
- // dispatch (each step's inner `handleToolCall` connects as needed).
922
- // Routing the outer call through `getTransportForRobot()` would
923
- // hang or fail when an agent only wanted to compile / cancel.
960
+ // mission_cancel / pause / resume + run_mission must NOT trigger a
961
+ // transport connect at this outer layer.
924
962
  if (name === "mission_cancel") {
925
963
  return handleMissionCancel(args);
926
964
  }
965
+ if (name === "mission_pause") {
966
+ return handleMissionPause(args);
967
+ }
968
+ if (name === "mission_resume") {
969
+ return handleMissionResume(args);
970
+ }
927
971
  if (name === "run_mission") {
928
972
  return handleRunMission(args, config);
929
973
  }
@@ -18,6 +18,8 @@
18
18
  "@agenticros/core": "workspace:*",
19
19
  "@agenticros/ros-camera": "workspace:*",
20
20
  "@google/genai": "^1.0.0",
21
+ "onnxruntime-node": "^1.20.0",
22
+ "sharp": "^0.33.0",
21
23
  "zod": "^3.24.0"
22
24
  },
23
25
  "devDependencies": {
@@ -2,7 +2,7 @@ import fs from "node:fs";
2
2
  import path from "node:path";
3
3
  import os from "node:os";
4
4
  import type { AgenticROSConfig } from "@agenticros/core";
5
- import { parseConfig } from "@agenticros/core";
5
+ import { parseConfig, withResolvedSkillRefs, applyCachedSkillRefs } from "@agenticros/core";
6
6
 
7
7
  /** Same as Claude Code MCP: optional `robot.namespace` override from the environment. */
8
8
  function applyMcpEnvOverrides(config: AgenticROSConfig): AgenticROSConfig {
@@ -17,11 +17,6 @@ function applyMcpEnvOverrides(config: AgenticROSConfig): AgenticROSConfig {
17
17
  };
18
18
  }
19
19
 
20
- /**
21
- * Resolve path to AgenticROS config file.
22
- * Prefer AGENTICROS_CONFIG_PATH; else ~/.agenticros/config.json.
23
- * Optional: fallback to OpenClaw config and read plugins.entries.agenticros.config.
24
- */
25
20
  function getConfigPath(): string {
26
21
  const env = process.env.AGENTICROS_CONFIG_PATH;
27
22
  if (env && env.trim().length > 0) {
@@ -30,9 +25,6 @@ function getConfigPath(): string {
30
25
  return path.join(os.homedir(), ".agenticros", "config.json");
31
26
  }
32
27
 
33
- /**
34
- * Try to read config from OpenClaw file (plugins.entries.agenticros.config).
35
- */
36
28
  function tryOpenClawConfig(): Record<string, unknown> | null {
37
29
  const openclawEnv = process.env.OPENCLAW_CONFIG;
38
30
  const openclawPath = openclawEnv && openclawEnv.trim().length > 0
@@ -57,12 +49,7 @@ function tryOpenClawConfig(): Record<string, unknown> | null {
57
49
  return null;
58
50
  }
59
51
 
60
- /**
61
- * Load and parse AgenticROS config.
62
- * 1) AGENTICROS_CONFIG_PATH or ~/.agenticros/config.json (full JSON object = config).
63
- * 2) If that file does not exist, try OpenClaw config and read plugins.entries.agenticros.config.
64
- */
65
- export function loadConfig(): AgenticROSConfig {
52
+ function loadConfigSync(): AgenticROSConfig {
66
53
  const primaryPath = getConfigPath();
67
54
  try {
68
55
  const raw = fs.readFileSync(primaryPath, "utf8");
@@ -72,14 +59,14 @@ export function loadConfig(): AgenticROSConfig {
72
59
  if (process.stderr && typeof process.stderr.write === "function") {
73
60
  process.stderr.write(`[AgenticROS] Config from ${primaryPath}\n`);
74
61
  }
75
- return applyMcpEnvOverrides(cfg);
62
+ return applyMcpEnvOverrides(applyCachedSkillRefs(cfg));
76
63
  }
77
64
  } catch (err) {
78
65
  const nodeErr = err as NodeJS.ErrnoException;
79
66
  if (nodeErr.code === "ENOENT") {
80
67
  const openclawConfig = tryOpenClawConfig();
81
68
  if (openclawConfig) {
82
- return applyMcpEnvOverrides(parseConfig(openclawConfig));
69
+ return applyMcpEnvOverrides(applyCachedSkillRefs(parseConfig(openclawConfig)));
83
70
  }
84
71
  throw new Error(
85
72
  `AgenticROS config not found at ${primaryPath}. Create it or set AGENTICROS_CONFIG_PATH. ` +
@@ -88,5 +75,27 @@ export function loadConfig(): AgenticROSConfig {
88
75
  }
89
76
  throw err;
90
77
  }
91
- return applyMcpEnvOverrides(parseConfig({}));
78
+ return applyMcpEnvOverrides(applyCachedSkillRefs(parseConfig({})));
79
+ }
80
+
81
+ export function loadConfig(): AgenticROSConfig {
82
+ return loadConfigSync();
83
+ }
84
+
85
+ export async function loadConfigAsync(): Promise<AgenticROSConfig> {
86
+ const base = loadConfigSync();
87
+ if (!(base.skillRefs?.length)) return base;
88
+ const { config, errors } = await withResolvedSkillRefs(base, {
89
+ onLog: (msg) => {
90
+ if (process.stderr && typeof process.stderr.write === "function") {
91
+ process.stderr.write(`[AgenticROS] ${msg}\n`);
92
+ }
93
+ },
94
+ });
95
+ for (const e of errors) {
96
+ if (process.stderr && typeof process.stderr.write === "function") {
97
+ process.stderr.write(`[AgenticROS] skillRef ${e.ref}: ${e.error}\n`);
98
+ }
99
+ }
100
+ return applyMcpEnvOverrides(config);
92
101
  }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * COCO 80-class names (index = YOLOv8 class ID). Lower-cased for lookup.
3
+ */
4
+
5
+ export const COCO_CLASSES = [
6
+ "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train", "truck",
7
+ "boat", "traffic light", "fire hydrant", "stop sign", "parking meter", "bench",
8
+ "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear", "zebra",
9
+ "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
10
+ "skis", "snowboard", "sports ball", "kite", "baseball bat", "baseball glove",
11
+ "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup",
12
+ "fork", "knife", "spoon", "bowl", "banana", "apple", "sandwich", "orange",
13
+ "broccoli", "carrot", "hot dog", "pizza", "donut", "cake", "chair", "couch",
14
+ "potted plant", "bed", "dining table", "toilet", "tv", "laptop", "mouse",
15
+ "remote", "keyboard", "cell phone", "microwave", "oven", "toaster", "sink",
16
+ "refrigerator", "book", "clock", "vase", "scissors", "teddy bear", "hair drier",
17
+ "toothbrush",
18
+ ] as const;
19
+
20
+ const ALIASES: Record<string, string> = {
21
+ phone: "cell phone",
22
+ cellphone: "cell phone",
23
+ mobile: "cell phone",
24
+ smartphone: "cell phone",
25
+ tv: "tv",
26
+ television: "tv",
27
+ monitor: "tv",
28
+ sofa: "couch",
29
+ plant: "potted plant",
30
+ bike: "bicycle",
31
+ };
32
+
33
+ export function resolveCocoClassId(name: string): number | null {
34
+ const key = name.trim().toLowerCase();
35
+ const canonical = ALIASES[key] ?? key;
36
+ const idx = COCO_CLASSES.indexOf(canonical as (typeof COCO_CLASSES)[number]);
37
+ return idx >= 0 ? idx : null;
38
+ }
@@ -0,0 +1,191 @@
1
+ /**
2
+ * Rotate the robot in place until a target COCO class is detected in the
3
+ * camera feed, then stop. Used by the ros2_find_object MCP tool.
4
+ */
5
+
6
+ import type { AgenticROSConfig, ResolvedRobot, RosTransport } from "@agenticros/core";
7
+ import { resolveCameraSubscribeTopic, toNamespacedTopic } from "@agenticros/core";
8
+ import {
9
+ ROS_MSG_COMPRESSED_IMAGE,
10
+ cameraSnapshotFromPlainMessage,
11
+ } from "@agenticros/ros-camera";
12
+ import { PersonDetector } from "../follow-me/detector.js";
13
+ import { resolveCocoClassId, COCO_CLASSES } from "./coco-classes.js";
14
+
15
+ const DEFAULT_COLOR_TOPIC = "/camera/camera/color/image_raw/compressed";
16
+ const DEFAULT_ANGULAR_SPEED = 0.3; // rad/s
17
+ const DEFAULT_TIMEOUT_SEC = 30;
18
+ const DEFAULT_MIN_CONFIDENCE = 0.5;
19
+ const POLL_INTERVAL_MS = 500;
20
+ const SNAPSHOT_TIMEOUT_MS = 3000;
21
+
22
+ export interface FindObjectOptions {
23
+ target: string;
24
+ angularSpeed?: number;
25
+ timeoutSeconds?: number;
26
+ minConfidence?: number;
27
+ clockwise?: boolean;
28
+ }
29
+
30
+ export interface FindObjectResult {
31
+ found: boolean;
32
+ target: string;
33
+ classId: number;
34
+ elapsedSeconds: number;
35
+ rotationDirection: "clockwise" | "counterclockwise";
36
+ angularSpeed: number;
37
+ detection?: {
38
+ confidence: number;
39
+ cx: number;
40
+ cy: number;
41
+ width: number;
42
+ height: number;
43
+ imageWidth: number;
44
+ imageHeight: number;
45
+ horizontalOffset: number;
46
+ };
47
+ error?: string;
48
+ }
49
+
50
+ export async function findObject(
51
+ robot: ResolvedRobot,
52
+ config: AgenticROSConfig,
53
+ transport: RosTransport,
54
+ opts: FindObjectOptions,
55
+ ): Promise<FindObjectResult> {
56
+ const classId = resolveCocoClassId(opts.target);
57
+ if (classId === null) {
58
+ return {
59
+ found: false,
60
+ target: opts.target,
61
+ classId: -1,
62
+ elapsedSeconds: 0,
63
+ rotationDirection: "clockwise",
64
+ angularSpeed: 0,
65
+ error:
66
+ `Unknown target "${opts.target}". Must be a COCO class name (e.g., "cell phone", "chair", "bottle"). ` +
67
+ `Supported: ${COCO_CLASSES.join(", ")}.`,
68
+ };
69
+ }
70
+
71
+ const safety = config.safety ?? {};
72
+ const maxAngular = safety.maxAngularVelocity ?? 1.5;
73
+ const requestedSpeed = Math.max(0.05, Math.min(maxAngular, opts.angularSpeed ?? DEFAULT_ANGULAR_SPEED));
74
+ const clockwise = opts.clockwise ?? true;
75
+ const angularZ = clockwise ? -requestedSpeed : requestedSpeed;
76
+ const timeoutMs = Math.max(1000, (opts.timeoutSeconds ?? DEFAULT_TIMEOUT_SEC) * 1000);
77
+ const minConfidence = opts.minConfidence ?? DEFAULT_MIN_CONFIDENCE;
78
+
79
+ const detector = new PersonDetector({ scoreThreshold: minConfidence });
80
+ await detector.load();
81
+
82
+ const cmdVelTopic = resolveCmdVelTopic(config, robot);
83
+ const colorTopic = resolveCameraSubscribeTopic(
84
+ robot.namespace,
85
+ robot.cameraTopic.trim() || DEFAULT_COLOR_TOPIC,
86
+ );
87
+
88
+ const startedAt = Date.now();
89
+ let rotating = false;
90
+ let result: FindObjectResult["detection"] | undefined;
91
+
92
+ const publishTwist = async (linearX: number, angZ: number) => {
93
+ try {
94
+ await transport.publish({
95
+ topic: cmdVelTopic,
96
+ type: "geometry_msgs/msg/Twist",
97
+ msg: { linear: { x: linearX, y: 0, z: 0 }, angular: { x: 0, y: 0, z: angZ } },
98
+ });
99
+ } catch {
100
+ // best-effort; loop will retry
101
+ }
102
+ };
103
+
104
+ try {
105
+ await publishTwist(0, angularZ);
106
+ rotating = true;
107
+
108
+ const deadline = startedAt + timeoutMs;
109
+ while (Date.now() < deadline && !result) {
110
+ // Keep the rotation alive in case the robot times out cmd_vel commands.
111
+ await publishTwist(0, angularZ);
112
+
113
+ const frame = await snapshotOnce(transport, colorTopic).catch(() => null);
114
+ if (frame) {
115
+ const det = await detector.detectClass(frame.buffer, classId);
116
+ if (det.detections.length > 0) {
117
+ const best = det.detections.reduce((a, b) => (a.confidence > b.confidence ? a : b));
118
+ result = {
119
+ confidence: best.confidence,
120
+ cx: best.cx,
121
+ cy: best.cy,
122
+ width: best.width,
123
+ height: best.height,
124
+ imageWidth: det.width,
125
+ imageHeight: det.height,
126
+ horizontalOffset: (best.cx - det.width / 2) / (det.width / 2),
127
+ };
128
+ break;
129
+ }
130
+ }
131
+
132
+ const remaining = deadline - Date.now();
133
+ if (remaining <= 0) break;
134
+ await sleep(Math.min(POLL_INTERVAL_MS, remaining));
135
+ }
136
+ } finally {
137
+ if (rotating) await publishTwist(0, 0);
138
+ await detector.dispose().catch(() => {});
139
+ }
140
+
141
+ const elapsedSeconds = (Date.now() - startedAt) / 1000;
142
+ return {
143
+ found: !!result,
144
+ target: opts.target,
145
+ classId,
146
+ elapsedSeconds,
147
+ rotationDirection: clockwise ? "clockwise" : "counterclockwise",
148
+ angularSpeed: requestedSpeed,
149
+ detection: result,
150
+ };
151
+ }
152
+
153
+ async function snapshotOnce(
154
+ transport: RosTransport,
155
+ topic: string,
156
+ ): Promise<{ buffer: Buffer } | null> {
157
+ return new Promise((resolve) => {
158
+ const sub = transport.subscribe(
159
+ { topic, type: ROS_MSG_COMPRESSED_IMAGE },
160
+ (msg: Record<string, unknown>) => {
161
+ clearTimeout(timer);
162
+ sub.unsubscribe();
163
+ try {
164
+ const payload = cameraSnapshotFromPlainMessage("CompressedImage", msg);
165
+ resolve({ buffer: Buffer.from(payload.dataBase64, "base64") });
166
+ } catch {
167
+ resolve(null);
168
+ }
169
+ },
170
+ );
171
+ const timer = setTimeout(() => {
172
+ sub.unsubscribe();
173
+ resolve(null);
174
+ }, SNAPSHOT_TIMEOUT_MS);
175
+ });
176
+ }
177
+
178
+ function sleep(ms: number): Promise<void> {
179
+ return new Promise((resolve) => setTimeout(resolve, ms));
180
+ }
181
+
182
+ function resolveCmdVelTopic(config: AgenticROSConfig, robot: ResolvedRobot): string {
183
+ const raw = (config.teleop?.cmdVelTopic ?? "").trim() || "/cmd_vel";
184
+ const namespaced = toNamespacedTopic(robot.namespace, raw);
185
+ const match = namespaced.match(/^\/([^/]+)\/cmd_vel$/i);
186
+ const segment = match?.[1] ?? "";
187
+ if (match && !segment.toLowerCase().startsWith("robot")) {
188
+ return `/robot${segment.replace(/-/g, "")}/cmd_vel`;
189
+ }
190
+ return namespaced;
191
+ }