agenticros 0.0.1 → 0.1.1

package/runtime/packages/agenticros-claude-code/src/transport.ts

@@ -0,0 +1,58 @@
+import type { RosTransport } from "@agenticros/core";
+import { createTransport, getTransportConfig } from "@agenticros/core";
+import type { AgenticROSConfig } from "@agenticros/core";
+
+let transport: RosTransport | null = null;
+
+/**
+ * Get the active transport. Throws if not connected.
+ */
+export function getTransport(): RosTransport {
+  if (!transport) {
+    throw new Error("Transport not initialized. Ensure config is loaded and connect() has been called.");
+  }
+  return transport;
+}
+
+export function getTransportOrNull(): RosTransport | null {
+  return transport;
+}
+
+const CONNECT_TIMEOUT_MS = 15_000;
+
+/**
+ * Create and connect the transport. Idempotent.
+ * Fails after CONNECT_TIMEOUT_MS if Zenoh/router is unreachable.
+ */
+export async function connect(config: AgenticROSConfig): Promise<void> {
+  if (transport && transport.getStatus() === "connected") {
+    return;
+  }
+  if (transport) {
+    await transport.disconnect();
+    transport = null;
+  }
+  const transportCfg = getTransportConfig(config);
+  const newTransport = await createTransport(transportCfg);
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    setTimeout(
+      () => reject(new Error(
+        `Transport connection timed out after ${CONNECT_TIMEOUT_MS / 1000}s. ` +
+        "Is zenohd running? (e.g. ws://localhost:10000). Check config and MCP log.",
+      )),
+      CONNECT_TIMEOUT_MS,
+    );
+  });
+  await Promise.race([newTransport.connect(), timeoutPromise]);
+  transport = newTransport;
+}
+
+/**
+ * Disconnect and clear the transport. Call on process exit.
+ */
+export async function disconnect(): Promise<void> {
+  if (transport) {
+    await transport.disconnect();
+    transport = null;
+  }
+}

package/runtime/packages/agenticros-claude-code/src/zero-shot/detector.ts

@@ -0,0 +1,169 @@
+/**
+ * Zero-shot object detector using OWL-v2 via @huggingface/transformers.
+ *
+ * Accepts arbitrary text prompts at runtime ("person", "person in red shirt",
+ * "stroller", "bottle on the table") and returns bounding boxes for any
+ * region that matches. Truly open-vocabulary — no class list baked in.
+ *
+ * Model: Xenova/owlv2-base-patch16-finetuned (downloaded on first use,
+ * cached under ~/.cache/huggingface). Override with AGENTICROS_OWLV2_MODEL.
+ *
+ * Both `@huggingface/transformers` and `sharp` are loaded lazily on first
+ * use so a missing optional dep cannot crash MCP server startup — it only
+ * surfaces when the user actually calls the zero-shot detector.
+ */
+
+// Type-only imports keep the types available at compile time without forcing
+// the runtime modules to load when this file is imported.
+type TransformersModule = typeof import("@huggingface/transformers");
+type SharpFn = (input: Buffer | Uint8Array) => import("sharp").Sharp;
+import type {
+  ZeroShotObjectDetectionPipeline,
+  RawImage as RawImageType,
+} from "@huggingface/transformers";
+
+const DEFAULT_MODEL = "Xenova/owlv2-base-patch16-finetuned";
+
+let transformersModule: TransformersModule | null = null;
+let sharpFn: SharpFn | null = null;
+
+async function loadDeps(): Promise<{
+  transformers: TransformersModule;
+  sharp: SharpFn;
+}> {
+  if (transformersModule && sharpFn) {
+    return { transformers: transformersModule, sharp: sharpFn };
+  }
+  try {
+    const [tMod, sharpMod] = await Promise.all([
+      import("@huggingface/transformers"),
+      import("sharp"),
+    ]);
+    const tAny = tMod as unknown as { default?: TransformersModule };
+    transformersModule = tAny.default ?? (tMod as unknown as TransformersModule);
+    const sharpAny = sharpMod as unknown as { default?: SharpFn };
+    sharpFn = sharpAny.default ?? (sharpMod as unknown as SharpFn);
+  } catch (err) {
+    const hint = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `Zero-shot detection requires the optional packages '@huggingface/transformers' and 'sharp'. ` +
+        `Install them in this workspace (pnpm install) to enable open-vocabulary detection. ` +
+        `Underlying error: ${hint}`,
+    );
+  }
+  return { transformers: transformersModule!, sharp: sharpFn! };
+}
+
+export interface ZeroShotDetection {
+  label: string;
+  confidence: number;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+  cx: number;
+  cy: number;
+}
+
+export interface ZeroShotDetectOptions {
+  threshold?: number;
+  topK?: number;
+}
+
+export class ZeroShotDetector {
+  private detector: ZeroShotObjectDetectionPipeline | null = null;
+  private loading: Promise<void> | null = null;
+  private readonly modelId: string;
+
+  constructor(modelId?: string) {
+    this.modelId =
+      modelId ??
+      (process.env["AGENTICROS_OWLV2_MODEL"]?.trim() || DEFAULT_MODEL);
+  }
+
+  async load(): Promise<void> {
+    if (this.detector) return;
+    if (this.loading) return this.loading;
+    this.loading = (async () => {
+      const { transformers } = await loadDeps();
+      process.stderr.write(
+        `[AgenticROS] zero-shot: loading ${this.modelId} (first run downloads ~150 MB)…\n`,
+      );
+      this.detector = (await transformers.pipeline(
+        "zero-shot-object-detection",
+        this.modelId,
+        { dtype: "q8" },
+      )) as unknown as ZeroShotObjectDetectionPipeline;
+      process.stderr.write(`[AgenticROS] zero-shot: model ready\n`);
+    })();
+    try {
+      await this.loading;
+    } finally {
+      this.loading = null;
+    }
+  }
+
+  async detectByText(
+    image: Buffer | Uint8Array,
+    prompts: string[],
+    opts: ZeroShotDetectOptions = {},
+  ): Promise<{ width: number; height: number; detections: ZeroShotDetection[] }> {
+    if (!this.detector) await this.load();
+    if (prompts.length === 0) return { width: 0, height: 0, detections: [] };
+
+    const { transformers, sharp: sharpFnLocal } = await loadDeps();
+
+    const decoded = await sharpFnLocal(image)
+      .removeAlpha()
+      .raw()
+      .toBuffer({ resolveWithObject: true });
+    const w = decoded.info.width;
+    const h = decoded.info.height;
+    const rawImg: RawImageType = new transformers.RawImage(
+      new Uint8Array(decoded.data),
+      w,
+      h,
+      3,
+    );
+
+    const pipelineOpts: { threshold: number; top_k?: number } = {
+      threshold: opts.threshold ?? 0.1,
+    };
+    if (opts.topK != null) pipelineOpts.top_k = opts.topK;
+    const out = await this.detector!(rawImg, prompts, pipelineOpts);
+
+    // Single-image input → Array<{label, score, box}>; the pipeline returns the
+    // batched form only for batched input. Normalize defensively.
+    const raw = Array.isArray(out) && Array.isArray((out as unknown[])[0])
+      ? (out as unknown[][])[0]!
+      : (out as unknown[]);
+
+    const detections: ZeroShotDetection[] = (raw as Array<{
+      label: string;
+      score: number;
+      box: { xmin: number; ymin: number; xmax: number; ymax: number };
+    }>).map((d) => {
+      const width = d.box.xmax - d.box.xmin;
+      const height = d.box.ymax - d.box.ymin;
+      return {
+        label: d.label,
+        confidence: d.score,
+        x: d.box.xmin,
+        y: d.box.ymin,
+        width,
+        height,
+        cx: d.box.xmin + width / 2,
+        cy: d.box.ymin + height / 2,
+      };
+    });
+
+    return { width: w, height: h, detections };
+  }
+
+  async dispose(): Promise<void> {
+    if (this.detector) {
+      await this.detector.dispose();
+      this.detector = null;
+    }
+  }
+}

package/runtime/packages/agenticros-claude-code/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

package/runtime/packages/agenticros-claude-code/yolo-debug.mjs

@@ -0,0 +1,106 @@
+// Debug script: capture a frame from the camera and dump ALL YOLO detections
+// across every COCO class above a very low confidence threshold.
+import { loadConfig } from "/home/ubuntu/Projects/agenticros/packages/agenticros-claude-code/dist/config.js";
+import { connect, disconnect, getTransport } from "/home/ubuntu/Projects/agenticros/packages/agenticros-claude-code/dist/transport.js";
+import { resolveCameraSubscribeTopic } from "/home/ubuntu/Projects/agenticros/packages/core/dist/index.js";
+import { ROS_MSG_COMPRESSED_IMAGE, cameraSnapshotFromPlainMessage } from "/home/ubuntu/Projects/agenticros/packages/ros-camera/dist/index.js";
+import { PersonDetector } from "/home/ubuntu/Projects/agenticros/packages/agenticros-claude-code/dist/follow-me/detector.js";
+import * as ort from "onnxruntime-node";
+import sharp from "sharp";
+import fs from "node:fs";
+
+const COCO = [
+  "person","bicycle","car","motorcycle","airplane","bus","train","truck",
+  "boat","traffic light","fire hydrant","stop sign","parking meter","bench",
+  "bird","cat","dog","horse","sheep","cow","elephant","bear","zebra",
+  "giraffe","backpack","umbrella","handbag","tie","suitcase","frisbee",
+  "skis","snowboard","sports ball","kite","baseball bat","baseball glove",
+  "skateboard","surfboard","tennis racket","bottle","wine glass","cup",
+  "fork","knife","spoon","bowl","banana","apple","sandwich","orange",
+  "broccoli","carrot","hot dog","pizza","donut","cake","chair","couch",
+  "potted plant","bed","dining table","toilet","tv","laptop","mouse",
+  "remote","keyboard","cell phone","microwave","oven","toaster","sink",
+  "refrigerator","book","clock","vase","scissors","teddy bear","hair drier",
+  "toothbrush",
+];
+
+async function captureFrame(transport, topic) {
+  return new Promise((resolve, reject) => {
+    const sub = transport.subscribe(
+      { topic, type: ROS_MSG_COMPRESSED_IMAGE },
+      (msg) => {
+        clearTimeout(timer);
+        sub.unsubscribe();
+        try {
+          const payload = cameraSnapshotFromPlainMessage("CompressedImage", msg);
+          resolve(Buffer.from(payload.dataBase64, "base64"));
+        } catch (e) { reject(e); }
+      },
+    );
+    const timer = setTimeout(() => { sub.unsubscribe(); reject(new Error("timeout")); }, 5000);
+  });
+}
+
+const INPUT_SIZE = 640;
+
+async function detectAll(image, modelPath) {
+  const session = await ort.InferenceSession.create(modelPath, { executionProviders: ["cpu"], graphOptimizationLevel: "all" });
+  const meta = await sharp(image).metadata();
+  const origW = meta.width, origH = meta.height;
+  const scale = Math.min(INPUT_SIZE / origW, INPUT_SIZE / origH);
+  const newW = Math.round(origW * scale), newH = Math.round(origH * scale);
+  const padX = Math.floor((INPUT_SIZE - newW) / 2), padY = Math.floor((INPUT_SIZE - newH) / 2);
+  const { data } = await sharp(image)
+    .resize(newW, newH, { fit: "fill" })
+    .extend({ top: padY, bottom: INPUT_SIZE - newH - padY, left: padX, right: INPUT_SIZE - newW - padX, background: { r:114,g:114,b:114 } })
+    .removeAlpha().raw().toBuffer({ resolveWithObject: true });
+  const pixels = INPUT_SIZE * INPUT_SIZE;
+  const input = new Float32Array(3 * pixels);
+  for (let i = 0; i < pixels; i++) {
+    input[i] = data[i*3] / 255;
+    input[pixels + i] = data[i*3 + 1] / 255;
+    input[2*pixels + i] = data[i*3 + 2] / 255;
+  }
+  const tensor = new ort.Tensor("float32", input, [1, 3, INPUT_SIZE, INPUT_SIZE]);
+  const inputName = session.inputNames[0];
+  const outputName = session.outputNames[0];
+  const out = await session.run({ [inputName]: tensor });
+  const result = out[outputName];
+  const [, _features, nAnchors] = result.dims;
+  const arr = result.data;
+  // For each anchor, find best class
+  const perClassMax = new Array(80).fill(0);
+  let overallMax = { score: 0, classId: -1, anchor: -1 };
+  for (let i = 0; i < nAnchors; i++) {
+    for (let c = 0; c < 80; c++) {
+      const s = arr[(4 + c) * nAnchors + i];
+      if (s > perClassMax[c]) perClassMax[c] = s;
+      if (s > overallMax.score) overallMax = { score: s, classId: c, anchor: i };
+    }
+  }
+  await session.release();
+  return { perClassMax, overallMax, imageW: origW, imageH: origH };
+}
+
+(async () => {
+  const config = loadConfig();
+  console.error("connecting transport...");
+  await connect(config);
+  const transport = getTransport();
+  const topic = resolveCameraSubscribeTopic(config, (config.robot?.cameraTopic ?? "").trim() || "/camera/camera/color/image_raw/compressed");
+  console.error(`subscribing topic=${topic}`);
+  const buf = await captureFrame(transport, topic);
+  fs.writeFileSync("/tmp/yolo-frame.jpg", buf);
+  console.error(`got frame ${buf.length} bytes, saved /tmp/yolo-frame.jpg`);
+  const modelPath = process.env.AGENTICROS_YOLOV8_MODEL || "/home/ubuntu/.agenticros/models/yolov8n.onnx";
+  const { perClassMax, overallMax, imageW, imageH } = await detectAll(buf, modelPath);
+  console.error(`image ${imageW}x${imageH}`);
+  console.error(`overall best: class=${overallMax.classId} (${COCO[overallMax.classId]}) score=${overallMax.score.toFixed(3)}`);
+  const ranked = perClassMax.map((s, c) => ({ s, c })).sort((a,b) => b.s - a.s).slice(0, 15);
+  console.error("top 15 classes (max anchor score):");
+  for (const { s, c } of ranked) {
+    console.error(`  ${s.toFixed(3)}  ${c}  ${COCO[c]}`);
+  }
+  await disconnect();
+  process.exit(0);
+})().catch(e => { console.error("ERR", e); process.exit(1); });

package/runtime/packages/agenticros-gemini/README.md

@@ -0,0 +1,139 @@
+# AgenticROS Gemini adapter
+
+CLI that uses **Google Gemini** to chat with your ROS2 robot. Same tool set as the Claude Code adapter (list topics, publish, subscribe, services, actions, params, camera snapshot, depth distance). No MCP — Gemini function calling is used directly.
+
+## Prerequisites
+
+- Node.js 20+
+- ROS2 transport available (Zenoh router, rosbridge, or local DDS)
+- **Gemini API key** ([Google AI Studio](https://aistudio.google.com/apikey))
+
+## Config
+
+Same as other adapters (Claude Code, OpenClaw):
+
+1. **`AGENTICROS_CONFIG_PATH`** — path to a JSON file
+2. **`~/.agenticros/config.json`**
+3. **OpenClaw config** — if the above are missing, uses `OPENCLAW_CONFIG` or `~/.openclaw/openclaw.json` → `plugins.entries.agenticros.config`
+
+Example `~/.agenticros/config.json`:
+
+```json
+{
+  "transport": { "mode": "zenoh" },
+  "zenoh": { "routerEndpoint": "ws://localhost:10000" },
+  "robot": {
+    "name": "MyRobot",
+    "namespace": "",
+    "cameraTopic": "/camera/camera/color/image_raw/compressed"
+  }
+}
+```
+
+## Build
+
+From the repo root:
+
+```bash
+pnpm install
+pnpm build
+```
+
+Or only this package: `pnpm --filter @agenticros/core build && pnpm --filter @agenticros/gemini build`.
+
+**Important:** Run `pnpm build` with nothing after it on the same line. If you paste extra words (e.g. from a comment like “if you haven’t already”), pnpm passes them to `tsc` and the build fails.
+
+### Quick API test (no ROS)
+
+After building, with a **real** key from [Google AI Studio](https://aistudio.google.com/apikey):
+
+```bash
+cd packages/agenticros-gemini
+export GEMINI_API_KEY="your-key-here"
+pnpm run smoke-api
+```
+
+You should see `Gemini: OK` (or similar). If you see `API_KEY_INVALID`, the key is wrong, revoked, or still the placeholder `YOUR_NEW_KEY_HERE`.
+
+### HTTP 429 / “quota exceeded” / `RESOURCE_EXHAUSTED`
+
+Your key is valid, but Google’s **free-tier limits** for that model are used up (or set to 0 for the project). This is **not** an AgenticROS bug.
+
+- Wait and retry (errors often include a suggested delay, e.g. ~12s).
+- Try another model: `export GEMINI_MODEL=gemini-2.5-flash` or `gemini-2.0-flash` (defaults may change; see [models](https://ai.google.dev/gemini-api/docs/models)).
+- Check usage and limits: [rate limits](https://ai.google.dev/gemini-api/docs/rate-limits), [AI Studio](https://aistudio.google.com/), and enable billing if you need higher quotas.
+
+The main CLI also respects **`GEMINI_MODEL`** (same as the smoke script after rebuild).
+
+## Usage
+
+Set **`GEMINI_API_KEY`** (or **`GOOGLE_API_KEY`**) and run the CLI:
+
+```bash
+export GEMINI_API_KEY=your_key_here
+pnpm --filter @agenticros/gemini exec agenticros-gemini "What do you see?"
+```
+
+Or with a message from stdin:
+
+```bash
+echo "List ROS2 topics" | GEMINI_API_KEY=xxx node packages/agenticros-gemini/dist/index.js
+```
+
+From the package directory after building:
+
+```bash
+cd packages/agenticros-gemini
+GEMINI_API_KEY=xxx node dist/index.js "Move the robot forward 0.2 m/s for 2 seconds then stop."
+```
+
+## Tested examples
+
+From the repo root:
+
+```bash
+pnpm --filter @agenticros/gemini run chat -- "Use ros2_camera_snapshot on /camera/camera/color/image_raw/compressed with message_type CompressedImage, then describe what you see in detail."
+
+pnpm --filter @agenticros/gemini run chat -- "Use ros2_depth_distance and report meters."
+
+pnpm --filter @agenticros/gemini run chat -- "Publish geometry_msgs/msg/Twist to /cmd_vel with linear.x=0.2, linear.y=0, linear.z=0, angular.x=0, angular.y=0, angular.z=0."
+
+pnpm --filter @agenticros/gemini run chat -- "Publish a stop command to cmd_vel."
+```
+
+For motion tests, send an explicit stop command as a separate command right after forward motion.
+
+## Tools
+
+The same ROS2 tools as the OpenClaw and Claude Code adapters:
+
+| Tool | Description |
+|------|-------------|
+| `ros2_list_topics` | List topics and types |
+| `ros2_publish` | Publish to a topic (e.g. cmd_vel) |
+| `ros2_subscribe_once` | Get next message from a topic |
+| `ros2_service_call` | Call a ROS2 service |
+| `ros2_action_goal` | Send goal to an action server |
+| `ros2_param_get` / `ros2_param_set` | Get/set node parameters |
+| `ros2_camera_snapshot` | One frame from camera topic (image returned to model) |
+| `ros2_depth_distance` | Distance in meters from depth camera |
+| `memory_remember` *(when enabled)* | Store a durable fact for the robot — shared with OpenClaw, Claude Desktop, Claude Code |
+| `memory_recall` *(when enabled)* | Semantic search across long-term memory |
+| `memory_forget` *(when enabled)* | Delete by id, query, or whole namespace |
+| `memory_status` *(when enabled)* | Health check + record count + embedder info |
+
+Safety limits from config (max linear/angular velocity) are applied before `ros2_publish`.
+
+### Optional: shared memory across agents
+
+The four `memory_*` tools are only exposed to Gemini when memory is **enabled** in `~/.agenticros/config.json`. Once enabled, the underlying store (`~/.mem0/vector_store.db` for the mem0 backend) is shared across every adapter for the same robot — so a fact remembered from Claude Desktop is immediately recall-able from Gemini, and vice versa. Memory tools work even when zenohd is not running. See [`docs/memory.md`](../../docs/memory.md) for setup recipes and troubleshooting.
+
+## Zenoh / transport
+
+The CLI connects to ROS2 via the same core transport as the other adapters. For Zenoh:
+
+1. Start **zenohd** with the remote-api plugin (e.g. port 10000): see `scripts/zenohd-agenticros.json5` or [docs/zenoh-agenticros.md](../../docs/zenoh-agenticros.md).
+2. Set `zenoh.routerEndpoint` in config (e.g. `ws://localhost:10000`).
+3. Run the Gemini CLI; it will connect at startup and use the tools in a loop until the model returns a final text answer.
+
+If the transport is not connected (e.g. zenohd not running), tool calls will fail with a clear error.

package/runtime/packages/agenticros-gemini/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@agenticros/gemini",
+  "version": "0.0.1",
+  "type": "module",
+  "description": "AgenticROS Gemini adapter — chat with your ROS2 robot using Google Gemini",
+  "main": "./dist/index.js",
+  "bin": {
+    "agenticros-gemini": "./dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "smoke-api": "node scripts/smoke-api.mjs",
+    "chat": "node dist/index.js"
+  },
+  "dependencies": {
+    "@agenticros/core": "workspace:*",
+    "@agenticros/ros-camera": "workspace:*",
+    "@google/genai": "^1.0.0",
+    "zod": "^3.24.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.17.0",
+    "typescript": "^5.7.0"
+  },
+  "author": "PlaiPin",
+  "license": "Apache-2.0"
+}

package/runtime/packages/agenticros-gemini/scripts/smoke-api.mjs

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+/**
+ * Minimal Gemini API check (no ROS, no AgenticROS config).
+ *
+ *   export GEMINI_API_KEY="your-real-key"
+ *   node scripts/smoke-api.mjs
+ *
+ * Run from packages/agenticros-gemini after: pnpm build
+ * (This script resolves @google/genai from the package's node_modules.)
+ */
+import { GoogleGenAI } from "@google/genai";
+
+const key = process.env.GEMINI_API_KEY ?? process.env.GOOGLE_API_KEY;
+if (!key || key.trim() === "" || key === "YOUR_NEW_KEY_HERE") {
+  console.error(
+    "Set GEMINI_API_KEY (or GOOGLE_API_KEY) to a real key from https://aistudio.google.com/apikey",
+  );
+  process.exit(1);
+}
+
+const model = (process.env.GEMINI_MODEL ?? "gemini-2.5-flash").trim();
+console.error(`[smoke-api] model=${model} (set GEMINI_MODEL to override)`);
+
+const ai = new GoogleGenAI({ apiKey: key });
+try {
+  const r = await ai.models.generateContent({
+    model,
+    contents: "Reply with exactly one word: OK",
+  });
+  console.log("Gemini:", r.text?.trim() ?? "(no text)");
+} catch (e) {
+  const msg = e instanceof Error ? e.message : String(e);
+  if (msg.includes("429") || msg.includes("RESOURCE_EXHAUSTED") || msg.includes("quota")) {
+    console.error(
+      "\nHTTP 429 / quota: free-tier limits for this model may be exhausted. Try:\n" +
+        "  • Wait a minute and retry (see Retry-After in the error)\n" +
+        "  • export GEMINI_MODEL=gemini-2.0-flash   # or another model your project allows\n" +
+        "  • https://ai.google.dev/gemini-api/docs/rate-limits and Google AI Studio billing\n",
+    );
+  }
+  throw e;
+}