@zhin.js/adapter-sandbox 6.0.5 → 7.0.1

package/CHANGELOG.md

@@ -1,5 +1,39 @@
 # @zhin.js/adapter-process
 
+## 7.0.1
+
+### Patch Changes
+
+- Updated dependencies [c8f4d45]
+  - @zhin.js/plugin-runtime@1.1.2
+  - @zhin.js/page@1.0.5
+  - @zhin.js/host-http@1.0.5
+  - @zhin.js/adapter@1.1.4
+  - @zhin.js/core@1.5.1
+
+## 7.0.0
+
+### Patch Changes
+
+- 4fbff5d: feat!: 多模态双向 Segment 一贯制（BREAKING，无兼容层）
+
+  全框架唯一媒体表达统一为 canonical `Segment` + `MediaRef{kind: url|path|base64|file, value, mime_type?, file_name?, size?}`，新增 audio/video/file 段类型；所有第二形状（legacy `data.url/file/base64` 字段、`mediaRefFromLegacyData`/`mediaRefToLegacyFields` 桥、双写）全部删除。
+
+  - **core**：`SendContent` 一等支持 `Segment[]`；endpoint 出站载荷只含 canonical 段；`resolveOutboundMediaPolicy` 改为纯声明驱动（adapter definition `segments.outboundMedia`），内置策略表删除，未声明回退 `url-or-text`；`ImageContent` 旧桥删除。
+  - **ai**：新增 `MediaContentBlock`/`MediaBlockRef`（Segment 同构）与 `UserMessage.media`（当前 turn 媒体，**不持久化**——存储层自动剥离）；`createUserMessage(text, media?)` 签名变更（`ImageContent` 删除）；provider 边界序列化器 `filterMediaBlocksForProvider` + 能力表（缺省 image-only，不支持类型降级占位文本）；ai-sdk 桥媒体块 → SDK image/file parts。
+  - **agent**：入站 turn 注入（`turn/inbound-media.ts`）——commMessage 媒体段 → 当前 turn `UserMessage.media`；图片 path 物化、音频默认 STT（`@zhin.js/speech` 可选，失败降级占位）、视频/文件占位；`publishOutboundElements` 产出 canonical Segment；`transcribeAudioPayload` 导出。
+  - **cli**：`bridgeRuntimeMessage` 回复链路媒体段透传，不再压平为文本（`$reply` 直达 normalize → adapter）。
+  - **全部 20 个平台适配器**：出站媒体只消费 `data.media`（url 直发 / base64 直发 / 平台上传 / 读盘），入站媒体产出 canonical `data.media`；`segments.outboundMedia` 声明与实际消费逐一核对修正；QQ 入站新增 canonical segments（image/audio/video/file/mention/face/reply），图片/语音/视频不再丢失。
+
+  迁移：适配器/插件产媒体一律用 `{ type, data: { media: MediaRef } }`；发送 legacy `data.url/file/base64` 形状的段会被 warn 丢弃。
+
+- Updated dependencies [7c1e63a]
+- Updated dependencies [4fbff5d]
+- Updated dependencies [5b94d9c]
+  - @zhin.js/adapter@1.1.3
+  - @zhin.js/core@1.5.0
+  - @zhin.js/client@2.1.4
+
 ## 6.0.5
 
 ### Patch Changes

package/adapters/sandbox.js

@@ -10,9 +10,10 @@ import { resolveSandboxEndpoint, } from "../lib/protocol.js";
 export { SandboxWsEndpoint } from "../lib/endpoint.js";
 export default defineAdapter({
     capabilities: ['inbound', 'outbound'],
-    // Console UI 直接渲染 base64 内联媒体；交互段由 Console 原生承载。
+    // Console UI 直连 URL 媒体、直接渲染 base64 内联媒体；path 由端点读盘
+    // 物化为 base64 内联（sandbox 无平台上传通道）。交互段由 Console 原生承载。
     segments: {
-        outboundMedia: ['base64'],
+        outboundMedia: ['url', 'base64', 'path'],
         interactive: 'native',
     },
     create(context) {

package/adapters/sandbox.ts

@@ -15,9 +15,10 @@ export type { SandboxEndpointOptions } from '../src/endpoint.js';
 
 export default defineAdapter<SandboxAdapterConfig>({
   capabilities: ['inbound', 'outbound'],
-  // Console UI 直接渲染 base64 内联媒体；交互段由 Console 原生承载。
+  // Console UI 直连 URL 媒体、直接渲染 base64 内联媒体；path 由端点读盘
+  // 物化为 base64 内联（sandbox 无平台上传通道）。交互段由 Console 原生承载。
   segments: {
-    outboundMedia: ['base64'],
+    outboundMedia: ['url', 'base64', 'path'],
     interactive: 'native',
   },
   create(context) {

package/lib/index.d.ts

@@ -1,2 +1,2 @@
-export { bindSandboxWsSocket, formatSandboxOutbound, parseSandboxWsPayload, resolveSandboxEndpoint, whenWsOpen, type MessageElement, type MessageType, type ResolvedSandboxBot, type SandboxAdapterConfig, type SandboxWsSocket, } from './protocol.js';
+export { bindSandboxWsSocket, formatSandboxOutbound, normalizeSandboxOutboundSegments, parseSandboxWsPayload, resolveSandboxEndpoint, whenWsOpen, type MessageElement, type MessageType, type ResolvedSandboxBot, type SandboxAdapterConfig, type SandboxWsSocket, } from './protocol.js';
 export { SandboxWsEndpoint, type SandboxEndpointOptions, } from './endpoint.js';

package/lib/index.js

@@ -1,2 +1,2 @@
-export { bindSandboxWsSocket, formatSandboxOutbound, parseSandboxWsPayload, resolveSandboxEndpoint, whenWsOpen, } from './protocol.js';
+export { bindSandboxWsSocket, formatSandboxOutbound, normalizeSandboxOutboundSegments, parseSandboxWsPayload, resolveSandboxEndpoint, whenWsOpen, } from './protocol.js';
 export { SandboxWsEndpoint, } from './endpoint.js';

package/lib/protocol.d.ts

@@ -54,6 +54,8 @@ export type SandboxOutboundChannel = {
     readonly endpoint?: string;
     readonly messageId?: string;
 };
+/** 出站段数组归一：媒体段走 MediaRef-only 归一，其余段原样透传。 */
+export declare function normalizeSandboxOutboundSegments(segments: readonly unknown[]): unknown[];
 /**
  * Wire-encode an already-rendered outbound payload.
  * Stamps `channel` so Console SandboxChat can filter by type+id (otherwise

package/lib/protocol.js

@@ -1,4 +1,8 @@
 /** Sandbox WebSocket wire protocol helpers (no legacy Adapter/Endpoint). */
+import { readFileSync } from 'node:fs';
+import { isMediaRef } from '@zhin.js/core';
+import { formatCompact, getLogger } from '@zhin.js/logger';
+const logger = getLogger('sandbox');
 export function resolveSandboxEndpoint(appConfig) {
     const entry = appConfig.endpoints?.find((item) => item.context === 'sandbox');
     const fixedName = typeof appConfig.name === 'string' && appConfig.name
@@ -103,6 +107,98 @@ export function parseSandboxWsPayload(raw) {
     }
     return { type, id, content, timestamp: payload.timestamp ?? Date.now(), text, action };
 }
+const MEDIA_SEGMENT_TYPES = new Set(['image', 'audio', 'video', 'file']);
+/**
+ * base64 内联值：Console UI 的 resolveMediaSrc 只识别 data: / base64://
+ * 前缀；裸 base64 补前缀（有 mime_type 时拼成可直转 data: URL 的形状）。
+ */
+function toInlineBase64Value(value, mimeType) {
+    const trimmed = value.trim();
+    if (trimmed.startsWith('base64://') || trimmed.startsWith('data:'))
+        return trimmed;
+    return mimeType
+        ? `base64://${mimeType};base64,${trimmed}`
+        : `base64://${trimmed}`;
+}
+/**
+ * 出站媒体段归一（canonical MediaRef 唯一来源，不读 legacy url/file/base64/src）：
+ * - kind=url → 浏览器直连 URL，原样透传；
+ * - kind=base64 → 内联直发（补 base64:// 前缀供 Console UI 解析）；
+ * - kind=path → 读盘物化为 base64 内联（sandbox 无平台上传通道）；
+ * - kind=file → sandbox 无不透明引用通道，丢弃。
+ * 无 canonical `data.media` 的媒体段一律 warn + 丢弃。
+ */
+function normalizeMediaSegment(segment) {
+    const data = segment.data ?? {};
+    const media = data.media;
+    if (!isMediaRef(media)) {
+        logger.warn(formatCompact({
+            op: 'sandbox_outbound_media_dropped',
+            type: segment.type,
+            reason: 'missing_media_ref',
+        }));
+        return null;
+    }
+    if (media.kind === 'url')
+        return { type: segment.type, data };
+    if (media.kind === 'base64') {
+        return {
+            type: segment.type,
+            data: {
+                ...data,
+                media: { ...media, value: toInlineBase64Value(media.value, media.mime_type) },
+            },
+        };
+    }
+    if (media.kind === 'path') {
+        try {
+            const base64 = readFileSync(media.value).toString('base64');
+            return {
+                type: segment.type,
+                data: {
+                    ...data,
+                    media: {
+                        ...media,
+                        kind: 'base64',
+                        value: toInlineBase64Value(base64, media.mime_type),
+                    },
+                },
+            };
+        }
+        catch (err) {
+            logger.warn(formatCompact({
+                op: 'sandbox_outbound_media_dropped',
+                type: segment.type,
+                reason: 'path_read_failed',
+                error: err instanceof Error ? err.message : String(err),
+            }));
+            return null;
+        }
+    }
+    logger.warn(formatCompact({
+        op: 'sandbox_outbound_media_dropped',
+        type: segment.type,
+        reason: 'unsupported_media_kind',
+    }));
+    return null;
+}
+/** 出站段数组归一：媒体段走 MediaRef-only 归一，其余段原样透传。 */
+export function normalizeSandboxOutboundSegments(segments) {
+    const out = [];
+    for (const item of segments) {
+        if (item
+            && typeof item === 'object'
+            && !Array.isArray(item)
+            && MEDIA_SEGMENT_TYPES.has(String(item.type))) {
+            const normalized = normalizeMediaSegment(item);
+            if (normalized)
+                out.push(normalized);
+            continue;
+        }
+        out.push(item);
+    }
+    return out;
+}
 /**
  * Wire-encode an already-rendered outbound payload.
  * Stamps `channel` so Console SandboxChat can filter by type+id (otherwise
@@ -130,7 +226,7 @@ export function formatSandboxOutbound(payload, channel = {}) {
     if (Array.isArray(payload)) {
         return JSON.stringify({
             ...stamp,
-            content: payload,
+            content: normalizeSandboxOutboundSegments(payload),
             timestamp: Date.now(),
         });
     }
@@ -146,6 +242,9 @@ export function formatSandboxOutbound(payload, channel = {}) {
         return JSON.stringify({
             ...stamp,
             ...envelope,
+            ...(Array.isArray(envelope.content)
+                ? { content: normalizeSandboxOutboundSegments(envelope.content) }
+                : {}),
             type: envelope.type ?? stamp.type,
             id: envelope.id ?? stamp.id,
             timestamp: typeof envelope.timestamp === 'number' ? envelope.timestamp : Date.now(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zhin.js/adapter-sandbox",
-  "version": "6.0.5",
+  "version": "7.0.1",
   "description": "Zhin.js Sandbox adapter for Plugin Runtime (WebSocket /sandbox)",
   "type": "module",
   "main": "./lib/index.js",
@@ -48,14 +48,14 @@
   "dependencies": {
     "lucide-react": "^0.525.0",
     "react": "^19.2.8",
-    "@zhin.js/adapter": "1.1.2",
-    "@zhin.js/client": "2.1.3",
+    "@zhin.js/adapter": "1.1.4",
+    "@zhin.js/client": "2.1.4",
     "@zhin.js/console-contract": "1.0.0",
-    "@zhin.js/core": "1.4.3",
-    "@zhin.js/host-http": "1.0.4",
+    "@zhin.js/core": "1.5.1",
+    "@zhin.js/host-http": "1.0.5",
     "@zhin.js/logger": "1.0.75",
-    "@zhin.js/page": "1.0.4",
-    "@zhin.js/plugin-runtime": "1.1.1"
+    "@zhin.js/page": "1.0.5",
+    "@zhin.js/plugin-runtime": "1.1.2"
   },
   "devDependencies": {
     "@types/react": "^19.2.18",
@@ -63,18 +63,18 @@
     "typescript": "^6.0.3",
     "vitest": "^4.1.10",
     "ws": "^8.21.1",
-    "@zhin.js/pagemanager": "2.0.9",
-    "@zhin.js/runtime": "1.0.5"
+    "@zhin.js/pagemanager": "2.0.11",
+    "@zhin.js/runtime": "1.0.6"
   },
   "peerDependencies": {
     "react": "^19.0.0",
-    "@zhin.js/adapter": "1.1.2",
-    "@zhin.js/client": "2.1.3",
+    "@zhin.js/adapter": "1.1.4",
+    "@zhin.js/client": "2.1.4",
     "@zhin.js/console-contract": "1.0.0",
-    "@zhin.js/core": "1.4.3",
-    "@zhin.js/host-http": "1.0.4",
-    "@zhin.js/page": "1.0.4",
-    "@zhin.js/plugin-runtime": "1.1.1"
+    "@zhin.js/core": "1.5.1",
+    "@zhin.js/host-http": "1.0.5",
+    "@zhin.js/page": "1.0.5",
+    "@zhin.js/plugin-runtime": "1.1.2"
   },
   "peerDependenciesMeta": {
     "react": {

package/pages/RichTextEditor.js

@@ -29,17 +29,17 @@ const RichTextEditor = forwardRef(({ placeholder = '输入消息...', onSend, on
                 else if (el.classList.contains('editor-image')) {
                     const imageUrl = el.dataset.url;
                     text += `[image:${imageUrl}]`;
-                    segments.push({ type: 'image', data: { url: imageUrl } });
+                    segments.push({ type: 'image', data: { media: { kind: 'url', value: imageUrl } } });
                 }
                 else if (el.classList.contains('editor-video')) {
                     const u = el.dataset.url || '';
                     text += `[video:${u}]`;
-                    segments.push({ type: 'video', data: { url: u } });
+                    segments.push({ type: 'video', data: { media: { kind: 'url', value: u } } });
                 }
                 else if (el.classList.contains('editor-audio')) {
                     const u = el.dataset.url || '';
                     text += `[audio:${u}]`;
-                    segments.push({ type: 'audio', data: { url: u } });
+                    segments.push({ type: 'audio', data: { media: { kind: 'url', value: u } } });
                 }
                 else if (el.classList.contains('editor-at')) {
                     const name = el.dataset.name;

package/pages/RichTextEditor.tsx

@@ -56,15 +56,15 @@ const RichTextEditor = forwardRef<RichTextEditorRef, RichTextEditorProps>(
                     } else if (el.classList.contains('editor-image')) {
                         const imageUrl = el.dataset.url
                         text += `[image:${imageUrl}]`
-                        segments.push({ type: 'image', data: { url: imageUrl } })
+                        segments.push({ type: 'image', data: { media: { kind: 'url', value: imageUrl } } })
                     } else if (el.classList.contains('editor-video')) {
                         const u = el.dataset.url || ''
                         text += `[video:${u}]`
-                        segments.push({ type: 'video', data: { url: u } })
+                        segments.push({ type: 'video', data: { media: { kind: 'url', value: u } } })
                     } else if (el.classList.contains('editor-audio')) {
                         const u = el.dataset.url || ''
                         text += `[audio:${u}]`
-                        segments.push({ type: 'audio', data: { url: u } })
+                        segments.push({ type: 'audio', data: { media: { kind: 'url', value: u } } })
                     } else if (el.classList.contains('editor-at')) {
                         const name = el.dataset.name
                         const id = el.dataset.id

package/pages/SandboxChat.js

@@ -197,11 +197,11 @@ export default function Sandbox() {
             else if (match[2])
                 segments.push({ type: 'face', data: { id: parseInt(match[2], 10) } });
             else if (match[3])
-                segments.push({ type: 'image', data: { url: match[3] } });
+                segments.push({ type: 'image', data: { media: { kind: 'url', value: match[3] } } });
             else if (match[4])
-                segments.push({ type: 'video', data: { url: match[4] } });
+                segments.push({ type: 'video', data: { media: { kind: 'url', value: match[4] } } });
             else if (match[5])
-                segments.push({ type: 'audio', data: { url: match[5] } });
+                segments.push({ type: 'audio', data: { media: { kind: 'url', value: match[5] } } });
             lastIndex = regex.lastIndex;
         }
         if (lastIndex < text.length) {

package/pages/SandboxChat.tsx

@@ -212,9 +212,9 @@ export default function Sandbox() {
             }
             if (match[1]) segments.push({ type: 'mention', data: { target: match[1], name: match[1] } })
             else if (match[2]) segments.push({ type: 'face', data: { id: parseInt(match[2], 10) } })
-            else if (match[3]) segments.push({ type: 'image', data: { url: match[3] } })
-            else if (match[4]) segments.push({ type: 'video', data: { url: match[4] } })
-            else if (match[5]) segments.push({ type: 'audio', data: { url: match[5] } })
+            else if (match[3]) segments.push({ type: 'image', data: { media: { kind: 'url', value: match[3] } } })
+            else if (match[4]) segments.push({ type: 'video', data: { media: { kind: 'url', value: match[4] } } })
+            else if (match[5]) segments.push({ type: 'audio', data: { media: { kind: 'url', value: match[5] } } })
             lastIndex = regex.lastIndex
         }
         if (lastIndex < text.length) {

package/src/index.ts

@@ -1,6 +1,7 @@
 export {
   bindSandboxWsSocket,
   formatSandboxOutbound,
+  normalizeSandboxOutboundSegments,
   parseSandboxWsPayload,
   resolveSandboxEndpoint,
   whenWsOpen,

package/src/protocol.ts

@@ -1,5 +1,11 @@
 /** Sandbox WebSocket wire protocol helpers (no legacy Adapter/Endpoint). */
 
+import { readFileSync } from 'node:fs';
+import { isMediaRef } from '@zhin.js/core';
+import { formatCompact, getLogger } from '@zhin.js/logger';
+
+const logger = getLogger('sandbox');
+
 export type MessageType = 'private' | 'group' | 'guild' | 'direct' | 'channel';
 
 export interface MessageElement {
@@ -179,6 +185,100 @@ export type SandboxOutboundChannel = {
   readonly messageId?: string;
 };
 
+const MEDIA_SEGMENT_TYPES = new Set(['image', 'audio', 'video', 'file']);
+
+/**
+ * base64 内联值：Console UI 的 resolveMediaSrc 只识别 data: / base64://
+ * 前缀；裸 base64 补前缀（有 mime_type 时拼成可直转 data: URL 的形状）。
+ */
+function toInlineBase64Value(value: string, mimeType?: string): string {
+  const trimmed = value.trim();
+  if (trimmed.startsWith('base64://') || trimmed.startsWith('data:')) return trimmed;
+  return mimeType
+    ? `base64://${mimeType};base64,${trimmed}`
+    : `base64://${trimmed}`;
+}
+
+/**
+ * 出站媒体段归一（canonical MediaRef 唯一来源，不读 legacy url/file/base64/src）：
+ * - kind=url → 浏览器直连 URL，原样透传；
+ * - kind=base64 → 内联直发（补 base64:// 前缀供 Console UI 解析）；
+ * - kind=path → 读盘物化为 base64 内联（sandbox 无平台上传通道）；
+ * - kind=file → sandbox 无不透明引用通道，丢弃。
+ * 无 canonical `data.media` 的媒体段一律 warn + 丢弃。
+ */
+function normalizeMediaSegment(segment: MessageElement): MessageElement | null {
+  const data = segment.data ?? {};
+  const media = data.media;
+  if (!isMediaRef(media)) {
+    logger.warn(formatCompact({
+      op: 'sandbox_outbound_media_dropped',
+      type: segment.type,
+      reason: 'missing_media_ref',
+    }));
+    return null;
+  }
+  if (media.kind === 'url') return { type: segment.type, data };
+  if (media.kind === 'base64') {
+    return {
+      type: segment.type,
+      data: {
+        ...data,
+        media: { ...media, value: toInlineBase64Value(media.value, media.mime_type) },
+      },
+    };
+  }
+  if (media.kind === 'path') {
+    try {
+      const base64 = readFileSync(media.value).toString('base64');
+      return {
+        type: segment.type,
+        data: {
+          ...data,
+          media: {
+            ...media,
+            kind: 'base64',
+            value: toInlineBase64Value(base64, media.mime_type),
+          },
+        },
+      };
+    } catch (err) {
+      logger.warn(formatCompact({
+        op: 'sandbox_outbound_media_dropped',
+        type: segment.type,
+        reason: 'path_read_failed',
+        error: err instanceof Error ? err.message : String(err),
+      }));
+      return null;
+    }
+  }
+  logger.warn(formatCompact({
+    op: 'sandbox_outbound_media_dropped',
+    type: segment.type,
+    reason: 'unsupported_media_kind',
+  }));
+  return null;
+}
+
+/** 出站段数组归一：媒体段走 MediaRef-only 归一，其余段原样透传。 */
+export function normalizeSandboxOutboundSegments(segments: readonly unknown[]): unknown[] {
+  const out: unknown[] = [];
+  for (const item of segments) {
+    if (
+      item
+      && typeof item === 'object'
+      && !Array.isArray(item)
+      && MEDIA_SEGMENT_TYPES.has(String((item as MessageElement).type))
+    ) {
+      const normalized = normalizeMediaSegment(item as MessageElement);
+      if (normalized) out.push(normalized);
+      continue;
+    }
+    out.push(item);
+  }
+  return out;
+}
+
 /**
  * Wire-encode an already-rendered outbound payload.
  * Stamps `channel` so Console SandboxChat can filter by type+id (otherwise
@@ -205,7 +305,7 @@ export function formatSandboxOutbound(
   if (Array.isArray(payload)) {
     return JSON.stringify({
       ...stamp,
-      content: payload,
+      content: normalizeSandboxOutboundSegments(payload),
       timestamp: Date.now(),
     });
   }
@@ -225,6 +325,9 @@ export function formatSandboxOutbound(
     return JSON.stringify({
       ...stamp,
       ...envelope,
+      ...(Array.isArray(envelope.content)
+        ? { content: normalizeSandboxOutboundSegments(envelope.content) }
+        : {}),
       type: envelope.type ?? stamp.type,
       id: envelope.id ?? stamp.id,
       timestamp: typeof envelope.timestamp === 'number' ? envelope.timestamp : Date.now(),