@radaros/transport 0.3.44 → 0.3.46

package/dist/index.cjs

@@ -28,6 +28,7 @@ __export(index_exports, {
   createAgentRouter: () => createAgentRouter,
   createBrowserGateway: () => createBrowserGateway,
   createFileUploadMiddleware: () => createFileUploadMiddleware,
+  createVisionGateway: () => createVisionGateway,
   createVoiceGateway: () => createVoiceGateway,
   errorHandler: () => errorHandler,
   generateAgentCard: () => generateAgentCard,
@@ -2080,6 +2081,128 @@ function createAgentGateway(opts) {
   });
 }
 
+// src/socketio/vision-gateway.ts
+function createVisionGateway(opts) {
+  const ns = opts.io.of(opts.namespace ?? "/radaros-vision");
+  if (opts.authMiddleware) {
+    ns.use(opts.authMiddleware);
+  }
+  const activeSessions = /* @__PURE__ */ new Map();
+  ns.on("connection", (socket) => {
+    socket.on(
+      "vision.start",
+      async (data) => {
+        const agent = opts.agents[data.agentName];
+        if (!agent) {
+          socket.emit("vision.error", { error: `Vision agent "${data.agentName}" not found` });
+          return;
+        }
+        if (activeSessions.has(socket.id)) {
+          socket.emit("vision.error", { error: "A vision session is already active for this connection" });
+          return;
+        }
+        try {
+          const apiKey = data.apiKey ?? socket.handshake?.auth?.apiKey;
+          const userId = data.userId ?? socket.handshake?.auth?.userId;
+          const sessionId = data.sessionId ?? socket.handshake?.auth?.sessionId;
+          const session = await agent.connect({ apiKey, userId, sessionId });
+          activeSessions.set(socket.id, session);
+          session.on("audio", (ev) => {
+            socket.emit("vision.audio", {
+              data: ev.data.toString("base64"),
+              mimeType: ev.mimeType ?? "audio/pcm"
+            });
+          });
+          session.on("transcript", (ev) => {
+            socket.emit("vision.transcript", { text: ev.text, role: ev.role });
+          });
+          session.on("text", (ev) => {
+            socket.emit("vision.text", { text: ev.text });
+          });
+          session.on("tool_call_start", (ev) => {
+            socket.emit("vision.tool.call", { name: ev.name, args: ev.args });
+          });
+          session.on("tool_result", (ev) => {
+            socket.emit("vision.tool.result", { name: ev.name, result: ev.result });
+          });
+          session.on("usage", (ev) => {
+            socket.emit("vision.usage", ev);
+          });
+          session.on("interrupted", () => {
+            socket.emit("vision.interrupted");
+          });
+          session.on("error", (ev) => {
+            socket.emit("vision.error", { error: ev.error.message });
+          });
+          session.on("disconnected", () => {
+            activeSessions.delete(socket.id);
+            socket.emit("vision.stopped");
+          });
+          socket.emit("vision.started", { userId });
+        } catch (error) {
+          socket.emit("vision.error", { error: error.message });
+        }
+      }
+    );
+    socket.on("vision.audio", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.data !== "string" || data.data.length > 1e6) return;
+      try {
+        session.sendAudio(Buffer.from(data.data, "base64"));
+      } catch {
+        socket.emit("vision.error", { error: "Invalid audio data" });
+      }
+    });
+    socket.on("vision.image", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.data !== "string" || data.data.length > 5e6) return;
+      try {
+        session.sendImage(Buffer.from(data.data, "base64"), data.mimeType ?? "image/jpeg");
+      } catch {
+        socket.emit("vision.error", { error: "Invalid image data" });
+      }
+    });
+    socket.on("vision.text", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.text !== "string" || data.text.length > 1e4) return;
+      session.sendText(data.text);
+    });
+    socket.on("vision.interrupt", () => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      session.interrupt();
+    });
+    socket.on("vision.stop", async () => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      try {
+        await session.close();
+      } catch (err) {
+        console.warn("[vision-gateway] Error closing session:", err);
+      }
+      activeSessions.delete(socket.id);
+      socket.emit("vision.stopped");
+    });
+    socket.on("disconnect", async () => {
+      const session = activeSessions.get(socket.id);
+      if (session) {
+        try {
+          await session.close();
+        } catch (err) {
+          console.warn(
+            "[radaros/vision-gateway] Error closing session on disconnect:",
+            err instanceof Error ? err.message : err
+          );
+        }
+        activeSessions.delete(socket.id);
+      }
+    });
+  });
+}
+
 // src/socketio/voice-gateway.ts
 function createVoiceGateway(opts) {
   const ns = opts.io.of(opts.namespace ?? "/radaros-voice");
@@ -2221,6 +2344,7 @@ function createVoiceGateway(opts) {
   createAgentRouter,
   createBrowserGateway,
   createFileUploadMiddleware,
+  createVisionGateway,
   createVoiceGateway,
   errorHandler,
   generateAgentCard,

package/dist/index.d.cts

@@ -1,4 +1,4 @@
-import { Agent, A2AAgentCard, MCPToolProviderConfig, ToolDef, MCPToolProvider, Registry, Servable, Team, Workflow, Toolkit, EventBus, VoiceAgent } from '@radaros/core';
+import { Agent, A2AAgentCard, MCPToolProviderConfig, ToolDef, MCPToolProvider, Registry, Servable, Team, Workflow, Toolkit, EventBus, VisionAgent, VoiceAgent } from '@radaros/core';
 
 interface A2AServerOptions {
     agents: Record<string, Agent>;
@@ -332,6 +332,14 @@ interface GatewayOptions {
 
 declare function createAgentGateway(opts: GatewayOptions): void;
 
+interface VisionGatewayOptions {
+    agents: Record<string, VisionAgent>;
+    io: any;
+    namespace?: string;
+    authMiddleware?: (socket: any, next: (err?: Error) => void) => void;
+}
+declare function createVisionGateway(opts: VisionGatewayOptions): void;
+
 interface VoiceGatewayOptions {
     agents: Record<string, VoiceAgent>;
     io: any;
@@ -340,4 +348,4 @@ interface VoiceGatewayOptions {
 }
 declare function createVoiceGateway(opts: VoiceGatewayOptions): void;
 
-export { type A2AServerOptions, type AdminRouterOptions, type BrowserGatewayOptions, type FileUploadOptions, type GatewayOptions, MCPManager, type MCPServerEntry, type MCPServerSummary, type RouterOptions, type SwaggerOptions, type VoiceGatewayOptions, buildMultiModalInput, createA2AServer, createAdminRouter, createAgentGateway, createAgentRouter, createBrowserGateway, createFileUploadMiddleware, createVoiceGateway, errorHandler, generateAgentCard, generateMultiAgentCard, generateOpenAPISpec, requestLogger };
+export { type A2AServerOptions, type AdminRouterOptions, type BrowserGatewayOptions, type FileUploadOptions, type GatewayOptions, MCPManager, type MCPServerEntry, type MCPServerSummary, type RouterOptions, type SwaggerOptions, type VisionGatewayOptions, type VoiceGatewayOptions, buildMultiModalInput, createA2AServer, createAdminRouter, createAgentGateway, createAgentRouter, createBrowserGateway, createFileUploadMiddleware, createVisionGateway, createVoiceGateway, errorHandler, generateAgentCard, generateMultiAgentCard, generateOpenAPISpec, requestLogger };

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import { Agent, A2AAgentCard, MCPToolProviderConfig, ToolDef, MCPToolProvider, Registry, Servable, Team, Workflow, Toolkit, EventBus, VoiceAgent } from '@radaros/core';
+import { Agent, A2AAgentCard, MCPToolProviderConfig, ToolDef, MCPToolProvider, Registry, Servable, Team, Workflow, Toolkit, EventBus, VisionAgent, VoiceAgent } from '@radaros/core';
 
 interface A2AServerOptions {
     agents: Record<string, Agent>;
@@ -332,6 +332,14 @@ interface GatewayOptions {
 
 declare function createAgentGateway(opts: GatewayOptions): void;
 
+interface VisionGatewayOptions {
+    agents: Record<string, VisionAgent>;
+    io: any;
+    namespace?: string;
+    authMiddleware?: (socket: any, next: (err?: Error) => void) => void;
+}
+declare function createVisionGateway(opts: VisionGatewayOptions): void;
+
 interface VoiceGatewayOptions {
     agents: Record<string, VoiceAgent>;
     io: any;
@@ -340,4 +348,4 @@ interface VoiceGatewayOptions {
 }
 declare function createVoiceGateway(opts: VoiceGatewayOptions): void;
 
-export { type A2AServerOptions, type AdminRouterOptions, type BrowserGatewayOptions, type FileUploadOptions, type GatewayOptions, MCPManager, type MCPServerEntry, type MCPServerSummary, type RouterOptions, type SwaggerOptions, type VoiceGatewayOptions, buildMultiModalInput, createA2AServer, createAdminRouter, createAgentGateway, createAgentRouter, createBrowserGateway, createFileUploadMiddleware, createVoiceGateway, errorHandler, generateAgentCard, generateMultiAgentCard, generateOpenAPISpec, requestLogger };
+export { type A2AServerOptions, type AdminRouterOptions, type BrowserGatewayOptions, type FileUploadOptions, type GatewayOptions, MCPManager, type MCPServerEntry, type MCPServerSummary, type RouterOptions, type SwaggerOptions, type VisionGatewayOptions, type VoiceGatewayOptions, buildMultiModalInput, createA2AServer, createAdminRouter, createAgentGateway, createAgentRouter, createBrowserGateway, createFileUploadMiddleware, createVisionGateway, createVoiceGateway, errorHandler, generateAgentCard, generateMultiAgentCard, generateOpenAPISpec, requestLogger };

package/dist/index.js

@@ -2037,6 +2037,128 @@ function createAgentGateway(opts) {
   });
 }
 
+// src/socketio/vision-gateway.ts
+function createVisionGateway(opts) {
+  const ns = opts.io.of(opts.namespace ?? "/radaros-vision");
+  if (opts.authMiddleware) {
+    ns.use(opts.authMiddleware);
+  }
+  const activeSessions = /* @__PURE__ */ new Map();
+  ns.on("connection", (socket) => {
+    socket.on(
+      "vision.start",
+      async (data) => {
+        const agent = opts.agents[data.agentName];
+        if (!agent) {
+          socket.emit("vision.error", { error: `Vision agent "${data.agentName}" not found` });
+          return;
+        }
+        if (activeSessions.has(socket.id)) {
+          socket.emit("vision.error", { error: "A vision session is already active for this connection" });
+          return;
+        }
+        try {
+          const apiKey = data.apiKey ?? socket.handshake?.auth?.apiKey;
+          const userId = data.userId ?? socket.handshake?.auth?.userId;
+          const sessionId = data.sessionId ?? socket.handshake?.auth?.sessionId;
+          const session = await agent.connect({ apiKey, userId, sessionId });
+          activeSessions.set(socket.id, session);
+          session.on("audio", (ev) => {
+            socket.emit("vision.audio", {
+              data: ev.data.toString("base64"),
+              mimeType: ev.mimeType ?? "audio/pcm"
+            });
+          });
+          session.on("transcript", (ev) => {
+            socket.emit("vision.transcript", { text: ev.text, role: ev.role });
+          });
+          session.on("text", (ev) => {
+            socket.emit("vision.text", { text: ev.text });
+          });
+          session.on("tool_call_start", (ev) => {
+            socket.emit("vision.tool.call", { name: ev.name, args: ev.args });
+          });
+          session.on("tool_result", (ev) => {
+            socket.emit("vision.tool.result", { name: ev.name, result: ev.result });
+          });
+          session.on("usage", (ev) => {
+            socket.emit("vision.usage", ev);
+          });
+          session.on("interrupted", () => {
+            socket.emit("vision.interrupted");
+          });
+          session.on("error", (ev) => {
+            socket.emit("vision.error", { error: ev.error.message });
+          });
+          session.on("disconnected", () => {
+            activeSessions.delete(socket.id);
+            socket.emit("vision.stopped");
+          });
+          socket.emit("vision.started", { userId });
+        } catch (error) {
+          socket.emit("vision.error", { error: error.message });
+        }
+      }
+    );
+    socket.on("vision.audio", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.data !== "string" || data.data.length > 1e6) return;
+      try {
+        session.sendAudio(Buffer.from(data.data, "base64"));
+      } catch {
+        socket.emit("vision.error", { error: "Invalid audio data" });
+      }
+    });
+    socket.on("vision.image", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.data !== "string" || data.data.length > 5e6) return;
+      try {
+        session.sendImage(Buffer.from(data.data, "base64"), data.mimeType ?? "image/jpeg");
+      } catch {
+        socket.emit("vision.error", { error: "Invalid image data" });
+      }
+    });
+    socket.on("vision.text", (data) => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      if (typeof data?.text !== "string" || data.text.length > 1e4) return;
+      session.sendText(data.text);
+    });
+    socket.on("vision.interrupt", () => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      session.interrupt();
+    });
+    socket.on("vision.stop", async () => {
+      const session = activeSessions.get(socket.id);
+      if (!session) return;
+      try {
+        await session.close();
+      } catch (err) {
+        console.warn("[vision-gateway] Error closing session:", err);
+      }
+      activeSessions.delete(socket.id);
+      socket.emit("vision.stopped");
+    });
+    socket.on("disconnect", async () => {
+      const session = activeSessions.get(socket.id);
+      if (session) {
+        try {
+          await session.close();
+        } catch (err) {
+          console.warn(
+            "[radaros/vision-gateway] Error closing session on disconnect:",
+            err instanceof Error ? err.message : err
+          );
+        }
+        activeSessions.delete(socket.id);
+      }
+    });
+  });
+}
+
 // src/socketio/voice-gateway.ts
 function createVoiceGateway(opts) {
   const ns = opts.io.of(opts.namespace ?? "/radaros-voice");
@@ -2177,6 +2299,7 @@ export {
   createAgentRouter,
   createBrowserGateway,
   createFileUploadMiddleware,
+  createVisionGateway,
   createVoiceGateway,
   errorHandler,
   generateAgentCard,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@radaros/transport",
-  "version": "0.3.44",
+  "version": "0.3.46",
   "description": "HTTP and WebSocket transport layer for RadarOS agents",
   "license": "MIT",
   "repository": {
@@ -42,7 +42,7 @@
     "typescript": "^5.6.0"
   },
   "peerDependencies": {
-    "@radaros/core": "^0.3.44",
+    "@radaros/core": "^0.3.46",
     "@types/express": "^4.0.0 || ^5.0.0",
     "express": "^4.0.0 || ^5.0.0",
     "multer": ">=2.0.0",