@open1s/ezbos 1.2.4 → 1.3.0

package/README.md

@@ -80,6 +80,55 @@ const agent = brain.agent('name')
 const started = await agent.start();
 ```
 
+## Multimodal Content
+
+Send images, audio, and other binary content alongside text.
+
+```ts
+import { Content, ContentPart, Binary } from '@open1s/ezbos';
+
+// Text only
+const textContent = Content.text('What is 2 + 2?');
+
+// Single image (URL)
+const imageContent = Content.image('https://example.com/photo.jpg');
+
+// Audio from base64 data
+const audioContent = Content.audio(base64Data, 'mp3');
+
+// Multi-part: text + image
+const multiContent = Content.parts([
+  ContentPart.text('Describe this image'),
+  ContentPart.image('https://example.com/photo.jpg'),
+]);
+
+// Pass to any LLM method
+const result = await agent.ask(multiContent);
+await agent.stream(multiContent, (token) => {
+  if (token.type === 'Text') process.stdout.write(token.text);
+});
+```
+
+### Content API
+
+| Method | Description |
+|--------|-------------|
+| `Content.text(text)` | Simple text content |
+| `Content.image(url, name?)` | Single image (URL) |
+| `Content.audio(data, format)` | Single audio (base64) |
+| `Content.audioUrl(url, format)` | Single audio (URL) |
+| `Content.parts([...])` | Multi-part content |
+
+### ContentPart API
+
+| Method | Description |
+|--------|-------------|
+| `ContentPart.text(text)` | Create text part |
+| `ContentPart.image(url, name?)` | Create image part |
+| `ContentPart.audio(data, format)` | Create audio part (base64) |
+| `ContentPart.audioUrl(url, format)` | Create audio part (URL) |
+| `ContentPart.binary(type, data, name?)` | Create binary part |
+
 ### Resilience
 
 Configure circuit breaker and rate limiting:
@@ -322,6 +371,25 @@ const result2 = await started.react('Calculate the sum');
 const result3 = await started.runSimple('Say hello');
 ```
 
+All methods accept multimodal content (text, images, audio):
+
+```ts
+import { Content, ContentPart } from '@open1s/ezbos';
+
+// Text + image
+const content = Content.parts([
+  ContentPart.text('What do you see?'),
+  ContentPart.image('https://example.com/photo.jpg'),
+]);
+const result = await started.ask(content);
+
+// Image only
+const imageOnly = Content.image('https://example.com/photo.jpg');
+await started.stream(imageOnly, (token) => {
+  if (token.type === 'Text') process.stdout.write(token.text);
+});
+```
+
 ### Streaming
 
 ```ts
@@ -451,6 +519,8 @@ await agent.close();
 | `06-session.ts` | Multi-turn conversation, compactSession, save/restore, export/import |
 | `07-agent-advanced.ts` | Streaming, streamCollect, metrics, resilience, config |
 | `08-brainos-messaging.ts` | Query/Queryable, Caller/Callable, Publisher/Subscriber (recv + run) |
+| `09-system-prompt.ts` | System prompt management and configuration |
+| `10-multimodal.ts` | Multimodal content (images, audio), Content/ContentPart API |
 
 ```bash
 npm run example:tools
@@ -461,6 +531,8 @@ npm run example:skills
 npm run example:session
 npm run example:agent
 npm run example:messaging
+npm run example:system-prompt
+npm run example:multimodal
 ```
 
 ## License

package/dist/agent.d.ts

@@ -1,5 +1,13 @@
 import * as jsbos from '@open1s/jsbos';
 import { SkillDef } from './skills.js';
+export interface JsContent {
+    type: string;
+    text?: string;
+    contentType?: string;
+    url?: string;
+    base64?: string;
+    name?: string;
+}
 export declare class AgentBuilder {
     private _inner;
     private _tools;
@@ -56,18 +64,19 @@ export declare class AgentBuilder {
 export declare class Agent {
     private _inner;
     constructor(_inner: jsbos.Agent);
+    private _resolveContent;
     run(task: string): Promise<string>;
-    ask(prompt: string): Promise<string>;
-    runSimple(prompt: string): Promise<string>;
-    react(task: string): Promise<string>;
+    ask(prompt: string | Array<JsContent>): Promise<string>;
+    runSimple(prompt: string | Array<JsContent>): Promise<string>;
+    react(task: string | Array<JsContent>): Promise<string>;
     compactSession(): Promise<void>;
     saveSession(path: string): void;
     restoreSession(path: string): void;
     clearSession(): void;
     exportSession(): string;
     importSession(json: string): void;
-    stream(task: string, onToken: (token: any) => void): Promise<string>;
-    streamCollect(task: string): Promise<any[]>;
+    stream(task: string | Array<JsContent>, onToken: (token: any) => void): Promise<string>;
+    streamCollect(task: string | Array<JsContent>): Promise<any[]>;
     get tools(): string[];
     get config(): any;
     get inner(): jsbos.Agent;

package/dist/agent.js

@@ -232,17 +232,51 @@ export class Agent {
     constructor(_inner) {
         this._inner = _inner;
     }
+    _resolveContent(input) {
+        if (input && typeof input === 'object' && input.constructor && input.constructor.name === 'Content') {
+            if (input._text !== null && input._text !== undefined) {
+                return [{ type: 'text', text: input._text }];
+            }
+            const parts = input.toJSON();
+            if (!Array.isArray(parts))
+                return [{ type: 'text', text: typeof parts === 'string' ? parts : JSON.stringify(parts) }];
+            return parts.map((p) => {
+                if (p.type === 'text') {
+                    return { type: 'text', text: p.text || '' };
+                }
+                const b = p.binary || {};
+                const source = b.source || {};
+                const result = { type: 'binary', contentType: b.content_type || 'image/jpeg', name: b.name };
+                if (source.url !== undefined)
+                    result.url = source.url;
+                else if (source.base64 !== undefined)
+                    result.base64 = source.base64;
+                else if (source.type === 'url')
+                    result.url = source.data;
+                else if (source.type === 'base64')
+                    result.base64 = source.data;
+                return result;
+            });
+        }
+        if (typeof input === 'string') {
+            return [{ type: 'text', text: input }];
+        }
+        if (Array.isArray(input)) {
+            return input;
+        }
+        return [{ type: 'text', text: String(input) }];
+    }
     async run(task) {
-        return this._inner.runSimple(task);
+        return this._inner.runSimple(this._resolveContent(task));
     }
     async ask(prompt) {
-        return this._inner.react(prompt);
+        return this._inner.react(this._resolveContent(prompt));
     }
     async runSimple(prompt) {
-        return this._inner.runSimple(prompt);
+        return this._inner.runSimple(this._resolveContent(prompt));
     }
     async react(task) {
-        return this._inner.react(task);
+        return this._inner.react(this._resolveContent(task));
     }
     async compactSession() {
         const sessionJson = this._inner.getSessionJson();
@@ -271,7 +305,7 @@ export class Agent {
         this._inner.restoreSessionJson(json);
     }
     stream(task, onToken) {
-        return this._inner.stream(task, (err, token) => {
+        return this._inner.stream(this._resolveContent(task), (err, token) => {
             if (err) {
                 onToken({ type: 'Error', error: err.message });
             }

package/dist/content.d.ts

@@ -0,0 +1,4 @@
+import * as jsbos from '@open1s/jsbos';
+export declare const Content: typeof jsbos.Content;
+export declare const ContentPart: typeof jsbos.ContentPart;
+export declare const Binary: typeof jsbos.Binary;

package/dist/content.js

@@ -0,0 +1,4 @@
+import * as jsbos from '@open1s/jsbos';
+export const Content = jsbos.Content;
+export const ContentPart = jsbos.ContentPart;
+export const Binary = jsbos.Binary;

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export * from './mcp.js';
 export * from './skills.js';
 export * from './agent.js';
 export * from './brainos.js';
+export * from './content.js';
 export declare const version: typeof jsbos.version;
 export declare const initTracing: typeof jsbos.initTracing;
 export declare const logTestMessage: typeof jsbos.logTestMessage;

package/dist/index.js

@@ -7,6 +7,7 @@ export * from './mcp.js';
 export * from './skills.js';
 export * from './agent.js';
 export * from './brainos.js';
+export * from './content.js';
 export const version = jsbos.version;
 export const initTracing = jsbos.initTracing;
 export const logTestMessage = jsbos.logTestMessage;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@open1s/ezbos",
-  "version": "1.2.4",
+  "version": "1.3.0",
   "description": "Simple BrainOS - Easy wrapper for @open1s/jsbos",
   "type": "module",
   "main": "./dist/index.js",
@@ -28,7 +28,9 @@
     "example:skills": "tsx examples/05-skills.ts",
     "example:session": "tsx examples/06-session.ts",
     "example:agent": "tsx examples/07-agent-advanced.ts",
-    "example:messaging": "tsx examples/08-brainos-messaging.ts"
+    "example:messaging": "tsx examples/08-brainos-messaging.ts",
+    "example:system-prompt": "tsx examples/09-system-prompt.ts",
+    "example:multimodal": "tsx examples/10-multimodal.ts"
   },
   "keywords": [
     "ai",
@@ -38,7 +40,7 @@
   ],
   "license": "MIT",
   "dependencies": {
-    "@open1s/jsbos": "^2.2.11"
+    "@open1s/jsbos": "^2.3.0"
   },
   "devDependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",

package/src/agent.ts

@@ -3,6 +3,16 @@ import { jsbos as jsbosDefault, InternalToolDef } from './tool.js';
 import { HookEvent, HookCallback, mergeHooks } from './hook.js';
 import { PluginHandlers, mergePlugins } from './plugin.js';
 import { SkillDef } from './skills.js';
+import { Content } from './content.js';
+
+export interface JsContent {
+  type: string;
+  text?: string;
+  contentType?: string;
+  url?: string;
+  base64?: string;
+  name?: string;
+}
 
 const DEFAULT_MODEL = 'nvidia/meta/llama-3.1-8b-instruct';
 const DEFAULT_BASE_URL = 'https://integrate.api.nvidia.com/v1';
@@ -304,20 +314,50 @@ export class AgentBuilder {
 export class Agent {
   constructor(private _inner: jsbos.Agent) {}
 
+  private _resolveContent(input: any): Array<JsContent> {
+    if (input && typeof input === 'object' && input.constructor && input.constructor.name === 'Content') {
+      if (input._text !== null && input._text !== undefined) {
+        return [{ type: 'text', text: input._text }];
+      }
+      const parts = input.toJSON();
+      if (!Array.isArray(parts)) return [{ type: 'text', text: typeof parts === 'string' ? parts : JSON.stringify(parts) }];
+      return (parts as Array<any>).map((p: any) => {
+        if (p.type === 'text') {
+          return { type: 'text', text: p.text || '' };
+        }
+        const b = p.binary || {};
+        const source = b.source || {};
+        const result: JsContent = { type: 'binary', contentType: b.content_type || 'image/jpeg', name: b.name };
+        if (source.url !== undefined) result.url = source.url;
+        else if (source.base64 !== undefined) result.base64 = source.base64;
+        else if (source.type === 'url') result.url = source.data;
+        else if (source.type === 'base64') result.base64 = source.data;
+        return result;
+      });
+    }
+    if (typeof input === 'string') {
+      return [{ type: 'text', text: input }];
+    }
+    if (Array.isArray(input)) {
+      return input;
+    }
+    return [{ type: 'text', text: String(input) }];
+  }
+
   async run(task: string): Promise<string> {
-    return this._inner.runSimple(task);
+    return this._inner.runSimple(this._resolveContent(task) as any);
   }
 
-  async ask(prompt: string): Promise<string> {
-    return this._inner.react(prompt);
+  async ask(prompt: string | Array<JsContent>): Promise<string> {
+    return this._inner.react(this._resolveContent(prompt) as any);
   }
 
-  async runSimple(prompt: string): Promise<string> {
-    return this._inner.runSimple(prompt);
+  async runSimple(prompt: string | Array<JsContent>): Promise<string> {
+    return this._inner.runSimple(this._resolveContent(prompt) as any);
   }
 
-  async react(task: string): Promise<string> {
-    return this._inner.react(task);
+  async react(task: string | Array<JsContent>): Promise<string> {
+    return this._inner.react(this._resolveContent(task) as any);
   }
 
   async compactSession(): Promise<void> {
@@ -356,8 +396,8 @@ export class Agent {
     this._inner.restoreSessionJson(json);
   }
 
-  stream(task: string, onToken: (token: any) => void): Promise<string> {
-    return this._inner.stream(task, (err, token) => {
+  stream(task: string | Array<JsContent>, onToken: (token: any) => void): Promise<string> {
+    return this._inner.stream(this._resolveContent(task) as any, (err, token) => {
       if (err) {
         onToken({ type: 'Error', error: err.message });
       } else {
@@ -366,7 +406,7 @@ export class Agent {
     });
   }
 
-  async streamCollect(task: string): Promise<any[]> {
+  async streamCollect(task: string | Array<JsContent>): Promise<any[]> {
     const tokens: any[] = [];
     await new Promise<void>((resolve) => {
       this.stream(task, token => {

package/src/content.ts

@@ -0,0 +1,5 @@
+import * as jsbos from '@open1s/jsbos';
+
+export const Content = jsbos.Content;
+export const ContentPart = jsbos.ContentPart;
+export const Binary = jsbos.Binary;

package/src/index.ts

@@ -9,6 +9,7 @@ export * from './mcp.js';
 export * from './skills.js';
 export * from './agent.js';
 export * from './brainos.js';
+export * from './content.js';
 
 export const version = jsbos.version;
 export const initTracing = jsbos.initTracing;