@cuylabs/agent-physical 5.0.2

package/dist/index-pM6qWmMq.d.ts

@@ -0,0 +1,225 @@
+import { Tool } from '@cuylabs/agent-core/tool';
+import { z } from 'zod';
+import { RiskLevel } from '@cuylabs/agent-core/safety';
+
+type PhysicalSessionStatus = "idle" | "starting" | "ready" | "running" | "awaiting-input" | "complete" | "stopping" | "stopped" | "error";
+type PhysicalExecutionMode = "simulation" | "hardware" | "hybrid" | "unknown";
+type PhysicalObservationKind = "rgb" | "depth" | "point-cloud" | "joint-state" | "pose" | "text" | "video" | "artifact";
+type PhysicalFrame = "world" | "robot-base" | "camera" | "tool" | string;
+interface PhysicalVector3 {
+    x: number;
+    y: number;
+    z: number;
+}
+interface PhysicalQuaternionWxyz {
+    w: number;
+    x: number;
+    y: number;
+    z: number;
+}
+interface PhysicalPose {
+    position: PhysicalVector3;
+    orientation?: PhysicalQuaternionWxyz;
+    frame?: PhysicalFrame;
+}
+interface PhysicalImageObservation {
+    kind: "rgb" | "depth";
+    source: string;
+    width?: number;
+    height?: number;
+    mimeType?: string;
+    dataUrl?: string;
+    uri?: string;
+    frame?: PhysicalFrame;
+    timestamp?: number;
+}
+interface PhysicalTextObservation {
+    kind: "text";
+    source: string;
+    text: string;
+    timestamp?: number;
+}
+interface PhysicalStructuredObservation {
+    kind: Exclude<PhysicalObservationKind, "rgb" | "depth" | "text">;
+    source: string;
+    data: unknown;
+    frame?: PhysicalFrame;
+    timestamp?: number;
+}
+type PhysicalObservationItem = PhysicalImageObservation | PhysicalTextObservation | PhysicalStructuredObservation;
+interface PhysicalObservation {
+    sessionId: string;
+    status?: PhysicalSessionStatus;
+    taskId?: string;
+    summary?: string;
+    items: PhysicalObservationItem[];
+    metadata?: Record<string, unknown>;
+    timestamp: number;
+}
+interface PhysicalTaskSpec {
+    id: string;
+    description: string;
+    goal?: string;
+    environment?: string;
+    mode?: PhysicalExecutionMode;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalSafetyPolicy {
+    riskLevel?: RiskLevel;
+    requiresApproval?: boolean;
+    requiresHumanPresence?: boolean;
+    allowedModes?: PhysicalExecutionMode[];
+    emergencyStopRequired?: boolean;
+    notes?: string[];
+}
+interface PhysicalCapability {
+    name: string;
+    description: string;
+    safety?: PhysicalSafetyPolicy;
+    inputSchema?: unknown;
+    outputSchema?: unknown;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalExecutionRequest {
+    code: string;
+    language?: "python" | "typescript" | "javascript" | string;
+    timeoutMs?: number;
+    dryRun?: boolean;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalOutcome {
+    success: boolean;
+    reward?: number;
+    taskCompleted?: boolean;
+    terminated?: boolean;
+    truncated?: boolean;
+    reason?: string;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalArtifact {
+    id: string;
+    kind: "code" | "log" | "image" | "video" | "json" | "directory" | "other";
+    uri: string;
+    label?: string;
+    mimeType?: string;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalExecutionTrace {
+    id: string;
+    startedAt: number;
+    completedAt?: number;
+    request?: PhysicalExecutionRequest;
+    stdout?: string;
+    stderr?: string;
+    observations?: PhysicalObservation[];
+    artifacts?: PhysicalArtifact[];
+    outcome?: PhysicalOutcome;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalSessionState {
+    sessionId: string;
+    status: PhysicalSessionStatus;
+    task?: PhysicalTaskSpec;
+    mode?: PhysicalExecutionMode;
+    startedAt?: number;
+    updatedAt: number;
+    message?: string;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalSessionStartOptions {
+    task?: PhysicalTaskSpec;
+    metadata?: Record<string, unknown>;
+}
+interface PhysicalObserveOptions {
+    includeImages?: boolean;
+    includeArtifacts?: boolean;
+    maxItems?: number;
+}
+interface PhysicalListArtifactsOptions {
+    kind?: PhysicalArtifact["kind"];
+    limit?: number;
+}
+interface PhysicalSession {
+    readonly id: string;
+    readonly capabilities?: readonly PhysicalCapability[];
+    getState(): Promise<PhysicalSessionState> | PhysicalSessionState;
+    start?(options?: PhysicalSessionStartOptions): Promise<PhysicalSessionState>;
+    reset?(options?: PhysicalSessionStartOptions): Promise<PhysicalSessionState>;
+    observe?(options?: PhysicalObserveOptions): Promise<PhysicalObservation>;
+    executeCode?(request: PhysicalExecutionRequest): Promise<PhysicalExecutionTrace>;
+    listArtifacts?(options?: PhysicalListArtifactsOptions): Promise<PhysicalArtifact[]>;
+    stop(reason?: string): Promise<PhysicalSessionState>;
+}
+interface PhysicalAdapter<TOptions = unknown> {
+    readonly name: string;
+    createSession(options: TOptions): Promise<PhysicalSession> | PhysicalSession;
+}
+interface PhysicalToolMetadata {
+    physical?: {
+        sessionId: string;
+        status?: PhysicalSessionStatus;
+        observation?: PhysicalObservation;
+        trace?: PhysicalExecutionTrace;
+        artifacts?: PhysicalArtifact[];
+        state?: PhysicalSessionState;
+    };
+    [key: string]: unknown;
+}
+type PhysicalTool = Tool.Info;
+
+interface PhysicalToolOptions {
+    /**
+     * Prefix used for generated tool ids. Defaults to "physical".
+     */
+    prefix?: string;
+}
+declare function createPhysicalStatusTool(session: PhysicalSession, options?: PhysicalToolOptions): Tool.Info<z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>, PhysicalToolMetadata>;
+declare function createPhysicalObserveTool(session: PhysicalSession, options?: PhysicalToolOptions): Tool.Info<z.ZodObject<{
+    includeImages: z.ZodOptional<z.ZodBoolean>;
+    includeArtifacts: z.ZodOptional<z.ZodBoolean>;
+    maxItems: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    includeImages?: boolean | undefined;
+    includeArtifacts?: boolean | undefined;
+    maxItems?: number | undefined;
+}, {
+    includeImages?: boolean | undefined;
+    includeArtifacts?: boolean | undefined;
+    maxItems?: number | undefined;
+}>, PhysicalToolMetadata>;
+declare function createPhysicalRunCodeTool(session: PhysicalSession, options?: PhysicalToolOptions): Tool.Info<z.ZodObject<{
+    code: z.ZodString;
+    language: z.ZodOptional<z.ZodString>;
+    timeoutMs: z.ZodOptional<z.ZodNumber>;
+    dryRun: z.ZodOptional<z.ZodBoolean>;
+}, "strip", z.ZodTypeAny, {
+    code: string;
+    language?: string | undefined;
+    timeoutMs?: number | undefined;
+    dryRun?: boolean | undefined;
+}, {
+    code: string;
+    language?: string | undefined;
+    timeoutMs?: number | undefined;
+    dryRun?: boolean | undefined;
+}>, PhysicalToolMetadata>;
+declare function createPhysicalStopTool(session: PhysicalSession, options?: PhysicalToolOptions): Tool.Info<z.ZodObject<{
+    reason: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    reason?: string | undefined;
+}, {
+    reason?: string | undefined;
+}>, PhysicalToolMetadata>;
+declare function createPhysicalArtifactsTool(session: PhysicalSession, options?: PhysicalToolOptions): Tool.Info<z.ZodObject<{
+    kind: z.ZodOptional<z.ZodEnum<["code", "log", "image", "video", "json", "directory", "other"]>>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    kind?: "video" | "code" | "log" | "image" | "json" | "directory" | "other" | undefined;
+    limit?: number | undefined;
+}, {
+    kind?: "video" | "code" | "log" | "image" | "json" | "directory" | "other" | undefined;
+    limit?: number | undefined;
+}>, PhysicalToolMetadata>;
+declare function createPhysicalSessionTools(session: PhysicalSession, options?: PhysicalToolOptions): Tool.AnyInfo[];
+
+export { type PhysicalVector3 as A, createPhysicalArtifactsTool as B, createPhysicalObserveTool as C, createPhysicalRunCodeTool as D, createPhysicalSessionTools as E, createPhysicalStatusTool as F, createPhysicalStopTool as G, type PhysicalSafetyPolicy as P, type PhysicalSessionState as a, type PhysicalExecutionMode as b, type PhysicalAdapter as c, type PhysicalArtifact as d, type PhysicalCapability as e, type PhysicalExecutionRequest as f, type PhysicalExecutionTrace as g, type PhysicalFrame as h, type PhysicalImageObservation as i, type PhysicalListArtifactsOptions as j, type PhysicalObservation as k, type PhysicalObservationItem as l, type PhysicalObservationKind as m, type PhysicalObserveOptions as n, type PhysicalOutcome as o, type PhysicalPose as p, type PhysicalQuaternionWxyz as q, type PhysicalSession as r, type PhysicalSessionStartOptions as s, type PhysicalSessionStatus as t, type PhysicalStructuredObservation as u, type PhysicalTaskSpec as v, type PhysicalTextObservation as w, type PhysicalTool as x, type PhysicalToolMetadata as y, type PhysicalToolOptions as z };

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+import { P as PhysicalSafetyPolicy, a as PhysicalSessionState, b as PhysicalExecutionMode } from './index-pM6qWmMq.js';
+export { c as PhysicalAdapter, d as PhysicalArtifact, e as PhysicalCapability, f as PhysicalExecutionRequest, g as PhysicalExecutionTrace, h as PhysicalFrame, i as PhysicalImageObservation, j as PhysicalListArtifactsOptions, k as PhysicalObservation, l as PhysicalObservationItem, m as PhysicalObservationKind, n as PhysicalObserveOptions, o as PhysicalOutcome, p as PhysicalPose, q as PhysicalQuaternionWxyz, r as PhysicalSession, s as PhysicalSessionStartOptions, t as PhysicalSessionStatus, u as PhysicalStructuredObservation, v as PhysicalTaskSpec, w as PhysicalTextObservation, x as PhysicalTool, y as PhysicalToolMetadata, z as PhysicalToolOptions, A as PhysicalVector3, B as createPhysicalArtifactsTool, C as createPhysicalObserveTool, D as createPhysicalRunCodeTool, E as createPhysicalSessionTools, F as createPhysicalStatusTool, G as createPhysicalStopTool } from './index-pM6qWmMq.js';
+import '@cuylabs/agent-core/tool';
+import 'zod';
+import '@cuylabs/agent-core/safety';
+
+declare const defaultPhysicalSafetyPolicy: Required<Pick<PhysicalSafetyPolicy, "riskLevel" | "requiresApproval" | "requiresHumanPresence">>;
+declare function normalizePhysicalSafetyPolicy(policy: PhysicalSafetyPolicy | undefined): PhysicalSafetyPolicy;
+declare function isModeAllowed(mode: PhysicalExecutionMode | undefined, policy: PhysicalSafetyPolicy | undefined): boolean;
+declare function describePhysicalState(state: PhysicalSessionState): string;
+
+export { PhysicalExecutionMode, PhysicalSafetyPolicy, PhysicalSessionState, defaultPhysicalSafetyPolicy, describePhysicalState, isModeAllowed, normalizePhysicalSafetyPolicy };

package/dist/index.js

@@ -0,0 +1,25 @@
+import {
+  createPhysicalArtifactsTool,
+  createPhysicalObserveTool,
+  createPhysicalRunCodeTool,
+  createPhysicalSessionTools,
+  createPhysicalStatusTool,
+  createPhysicalStopTool,
+  defaultPhysicalSafetyPolicy,
+  describePhysicalState,
+  isModeAllowed,
+  normalizePhysicalSafetyPolicy
+} from "./chunk-G7CGE7QH.js";
+export {
+  createPhysicalArtifactsTool,
+  createPhysicalObserveTool,
+  createPhysicalRunCodeTool,
+  createPhysicalSessionTools,
+  createPhysicalStatusTool,
+  createPhysicalStopTool,
+  defaultPhysicalSafetyPolicy,
+  describePhysicalState,
+  isModeAllowed,
+  normalizePhysicalSafetyPolicy
+};
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/tools/index.d.ts

@@ -0,0 +1,4 @@
+import '@cuylabs/agent-core/tool';
+import 'zod';
+export { z as PhysicalToolOptions, B as createPhysicalArtifactsTool, C as createPhysicalObserveTool, D as createPhysicalRunCodeTool, E as createPhysicalSessionTools, F as createPhysicalStatusTool, G as createPhysicalStopTool } from '../index-pM6qWmMq.js';
+import '@cuylabs/agent-core/safety';

package/dist/tools/index.js

@@ -0,0 +1,17 @@
+import {
+  createPhysicalArtifactsTool,
+  createPhysicalObserveTool,
+  createPhysicalRunCodeTool,
+  createPhysicalSessionTools,
+  createPhysicalStatusTool,
+  createPhysicalStopTool
+} from "../chunk-G7CGE7QH.js";
+export {
+  createPhysicalArtifactsTool,
+  createPhysicalObserveTool,
+  createPhysicalRunCodeTool,
+  createPhysicalSessionTools,
+  createPhysicalStatusTool,
+  createPhysicalStopTool
+};
+//# sourceMappingURL=index.js.map

package/dist/tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/docs/README.md

@@ -0,0 +1,25 @@
+# @cuylabs/agent-physical Docs
+
+Start here:
+
+- [Architecture](./architecture.md) explains the package boundary and why
+  robotics-specific semantics live outside `agent-core`.
+- [Safety](./safety.md) describes how physical capabilities should interact
+  with approvals, replay, and stop controls.
+
+## Core Concepts
+
+`PhysicalSession` is the central contract. A session represents one simulator,
+robot, or embodied runtime episode. It can expose status, observation, artifacts,
+optional reset/start operations, optional code execution, and stop control.
+
+`PhysicalObservation` is a timestamped set of observation items. Items can be
+images, depth, point clouds, joint state, poses, text summaries, video
+references, or artifacts.
+
+`PhysicalCapability` is descriptive metadata for what a backend can do. It
+exists separately from `agent-core` tools because physical capabilities need
+safety policy, world-state assumptions, and backend metadata.
+
+`PhysicalExecutionTrace` captures what happened after a policy or command was
+executed: stdout, stderr, observations, artifacts, and outcome.

package/docs/architecture.md

@@ -0,0 +1,66 @@
+# Physical Agent Architecture
+
+`agent-physical` keeps physical-world concepts out of `agent-core` while still
+making them available to normal `agent-core` agents.
+
+```text
+Application
+  -> agent-core Agent
+    -> agent-physical tools
+      -> PhysicalSession
+        -> backend adapter
+```
+
+The shared contract should stay small. It is allowed to know about physical
+sessions, observations, capabilities, traces, outcomes, artifacts, and safety
+policy. It should not know how to launch CaP-X as a Python Code-as-Policy
+robotics framework, connect to ROS 2 robot middleware, authenticate to NVIDIA
+Isaac Sim or another robot simulator, connect to robots_realtime for hardware
+control, or send joint commands.
+
+The agent loop stays above this package. `agent-core` or another harness decides
+when to observe, how to reason over observations, whether to generate policy
+code, whether to ask for approval, and whether to continue or stop. This package
+only defines the shared physical vocabulary and generic tools those loops can
+use.
+
+## What Belongs Here
+
+- Backend-neutral session and observation types
+- Generic tools for status, observe, stop, artifacts, and optional code
+  execution
+- Safety metadata that can feed approval middleware and UIs
+- Replay semantics for physical side effects
+
+## What Belongs In Adapters
+
+- Process launch and lifecycle for a specific runtime
+- Transport protocols and service clients
+- Backend-specific artifact discovery
+- Simulator or robot-specific observation mapping
+- Backend-specific helper tools, such as turn history, skill libraries, or
+  visual frame access
+- Any command vocabulary that only makes sense for one backend
+
+## CaP-X Mapping
+
+CaP-X maps naturally as a backend adapter because it is already a Python
+robotics framework for Code-as-Policy robot manipulation:
+
+```text
+CaP-X YAML config      -> PhysicalTaskSpec
+CaP-X env/trial       -> PhysicalSession
+CaP-X render/logs     -> PhysicalObservation
+CaP-X outputs folder  -> PhysicalArtifact[]
+CaP-X reward/completed -> PhysicalOutcome
+```
+
+The adapter should not duplicate CaP-X internals. It should launch, connect to,
+or call CaP-X and translate the results into these contracts. For live
+agent-owned CaP-X loops, the adapter should talk to a Python runtime bridge such
+as `capx-agent-runtime`; `agent-physical` itself still only defines the generic
+session and safety contract.
+
+That split is what enables bring-your-own-agent. `agent-core` can use the
+generic physical tools, the CaP-X adapter can add CaP-X-specific tools, and
+CaP-X can remain the Python robotics runtime.

package/docs/safety.md

@@ -0,0 +1,31 @@
+# Physical Safety
+
+Physical tools are conservative by default.
+
+Direct execution of policy code is marked:
+
+- `riskLevel: "dangerous"`
+- `destructive: true`
+- `parallelSafe: false`
+- `onInterrupt: "cancel"`
+- replay mode `manual`
+- side effect level `external`
+
+That default is intentional. Replaying a tool call that moved a simulator or
+robot can change state again, and replaying against hardware can be unsafe.
+
+## Approval Expectations
+
+Applications should require explicit approval for hardware sessions and for any
+capability that can cause motion, contact, heating, cutting, navigation, or
+other physical side effects.
+
+Simulation-only sessions can loosen the policy, but they should still treat
+state-mutating execution as non-parallel and non-replayable unless the backend
+provides snapshots and deterministic restore.
+
+## Stop Control
+
+Every concrete `PhysicalSession` must implement `stop(reason)`. The exact
+guarantee depends on the backend, but the method is the common control-plane
+surface for UIs, middleware, and runtime interruption.

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@cuylabs/agent-physical",
+  "version": "5.0.2",
+  "description": "Physical-world agent contracts for sessions, observations, capabilities, safety, and tool adapters",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    },
+    "./tools": {
+      "types": "./dist/tools/index.d.ts",
+      "import": "./dist/tools/index.js",
+      "default": "./dist/tools/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "docs",
+    "README.md"
+  ],
+  "dependencies": {
+    "zod": "^3.25.76 || ^4.1.8",
+    "@cuylabs/agent-core": "^5.0.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.7.0",
+    "vitest": "^4.0.18"
+  },
+  "keywords": [
+    "agent",
+    "physical-ai",
+    "robotics",
+    "embodied-ai",
+    "tools",
+    "safety"
+  ],
+  "author": "cuylabs",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/cuylabs-ai/agents-ts.git",
+    "directory": "packages/agent-physical"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsup --config tsup.config.ts",
+    "dev": "tsup --config tsup.config.ts --watch",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist"
+  }
+}