@kuralle-agents/messaging 0.7.2 → 0.8.0

package/dist/adapter/createMessagingRouter.js

@@ -3,6 +3,7 @@ import { MessageDeduplicator } from '../shared/deduplicator.js';
 import { InMemoryWindowStore } from './window-store.js';
 import { defaultSessionResolver } from './session-resolver.js';
 import { InboundResolverChain, defaultInboundChain, } from './input-resolver-chain.js';
+import { attachInboundMedia } from './inbound-media.js';
 import { StreamMapper } from './stream-mapper.js';
 import { OutboundPipeline } from './outbound-pipeline.js';
 import { windowGuard } from './middleware/window-guard.js';
@@ -100,7 +101,10 @@ export function createMessagingRouter(config) {
                 await recordInboundToHistory(config, sessionId, message, userId);
                 return;
             }
-            const { input, selection } = await inboundChain.resolve(message);
+            const { input: resolvedInput, selection } = await inboundChain.resolve(message);
+            // Multimodal intake: download any inbound media and attach it as AI SDK file
+            // parts so photos / voice notes / documents reach the model (REQ multimodal).
+            const input = await attachInboundMedia(message, resolvedInput, platform);
             try {
                 const handle = config.runtime.run({
                     input,

package/dist/adapter/inbound-media.d.ts

@@ -0,0 +1,17 @@
+import type { UserInputContent } from '@kuralle-agents/core';
+import type { InboundMessage } from '../types/messages.js';
+import type { PlatformClient } from '../types/client.js';
+/**
+ * Attach inbound media (image / audio / video / document / sticker) to the user
+ * turn as an AI SDK file part, so it reaches the model instead of being dropped.
+ *
+ * Bytes are downloaded via the platform client and base64-encoded — the durable
+ * runtime persists the user message through the session store (JSON/Redis/Postgres),
+ * so a raw `Buffer` is not safe; a base64 string is. When the platform already
+ * exposes a hosted URL on the reference, that URL is passed through instead.
+ *
+ * The caption (or any text the resolver produced) becomes a leading text part, so
+ * "here's my prescription 📷 + can you read it?" arrives as one multimodal turn.
+ * Non-media messages return `textInput` unchanged.
+ */
+export declare function attachInboundMedia(message: InboundMessage, textInput: UserInputContent, client: Pick<PlatformClient, 'downloadMedia'>): Promise<UserInputContent>;

package/dist/adapter/inbound-media.js

@@ -0,0 +1,50 @@
+/** Fallback media type when the platform omits a MIME type, keyed by message kind. */
+function fallbackMediaType(type) {
+    switch (type) {
+        case 'image':
+            return 'image/jpeg';
+        case 'audio':
+            return 'audio/ogg';
+        case 'video':
+            return 'video/mp4';
+        case 'sticker':
+            return 'image/webp';
+        default:
+            return 'application/octet-stream';
+    }
+}
+/**
+ * Attach inbound media (image / audio / video / document / sticker) to the user
+ * turn as an AI SDK file part, so it reaches the model instead of being dropped.
+ *
+ * Bytes are downloaded via the platform client and base64-encoded — the durable
+ * runtime persists the user message through the session store (JSON/Redis/Postgres),
+ * so a raw `Buffer` is not safe; a base64 string is. When the platform already
+ * exposes a hosted URL on the reference, that URL is passed through instead.
+ *
+ * The caption (or any text the resolver produced) becomes a leading text part, so
+ * "here's my prescription 📷 + can you read it?" arrives as one multimodal turn.
+ * Non-media messages return `textInput` unchanged.
+ */
+export async function attachInboundMedia(message, textInput, client) {
+    const ref = message.media;
+    if (!ref)
+        return textInput;
+    const caption = ref.caption ?? (typeof textInput === 'string' ? textInput : '');
+    let data;
+    let mediaType;
+    if (ref.url) {
+        data = ref.url;
+        mediaType = ref.mimeType || fallbackMediaType(message.type);
+    }
+    else {
+        const download = await client.downloadMedia(ref.id);
+        data = download.data.toString('base64');
+        mediaType = ref.mimeType || download.mimeType || fallbackMediaType(message.type);
+    }
+    const parts = [];
+    if (caption.trim())
+        parts.push({ type: 'text', text: caption });
+    parts.push({ type: 'file', data, mediaType, filename: ref.filename });
+    return parts;
+}

package/dist/adapter/input-resolver-chain.d.ts

@@ -1,23 +1,23 @@
-import type { ResolvedSelection } from '@kuralle-agents/core';
+import type { ResolvedSelection, UserInputContent } from '@kuralle-agents/core';
 import type { InboundMessage } from '../types/messages.js';
 export interface InboundResolverPlugin {
     readonly name: string;
     tryResolve(m: InboundMessage): Promise<{
-        input: string;
+        input: UserInputContent;
         selection?: ResolvedSelection;
     } | undefined>;
 }
 export declare class InteractiveResolver implements InboundResolverPlugin {
     readonly name = "interactive";
     tryResolve(m: InboundMessage): Promise<{
-        input: string;
+        input: UserInputContent;
         selection?: ResolvedSelection;
     } | undefined>;
 }
 export declare class TextResolver implements InboundResolverPlugin {
     readonly name = "text";
     tryResolve(m: InboundMessage): Promise<{
-        input: string;
+        input: UserInputContent;
         selection?: ResolvedSelection;
     }>;
 }
@@ -25,7 +25,7 @@ export declare class InboundResolverChain {
     private readonly plugins;
     constructor(plugins: InboundResolverPlugin[]);
     resolve(m: InboundMessage): Promise<{
-        input: string;
+        input: UserInputContent;
         selection?: ResolvedSelection;
     }>;
 }

package/dist/index.d.ts

@@ -6,6 +6,7 @@ export { SessionResolverChain, ThreadIdResolver, PhoneLookupResolver, } from './
 export type { SessionResolverPlugin } from './adapter/session-resolver-chain.js';
 export { InboundResolverChain, InteractiveResolver, TextResolver, defaultInboundChain, } from './adapter/input-resolver-chain.js';
 export type { InboundResolverPlugin } from './adapter/input-resolver-chain.js';
+export { attachInboundMedia } from './adapter/inbound-media.js';
 export { StreamMapper } from './adapter/stream-mapper.js';
 export { WindowTracker } from './adapter/window-tracker.js';
 export type { WindowStore, WindowState } from './adapter/window-store.js';

package/dist/index.js

@@ -9,6 +9,7 @@ export { createMessagingRouter } from './adapter/createMessagingRouter.js';
 export { defaultSessionResolver } from './adapter/session-resolver.js';
 export { SessionResolverChain, ThreadIdResolver, PhoneLookupResolver, } from './adapter/session-resolver-chain.js';
 export { InboundResolverChain, InteractiveResolver, TextResolver, defaultInboundChain, } from './adapter/input-resolver-chain.js';
+export { attachInboundMedia } from './adapter/inbound-media.js';
 export { StreamMapper } from './adapter/stream-mapper.js';
 export { WindowTracker } from './adapter/window-tracker.js';
 export { InMemoryWindowStore } from './adapter/window-store.js';

package/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/kuralle/kuralle-agents.git",
     "directory": "packages/kuralle-messaging"
   },
-  "version": "0.7.2",
+  "version": "0.8.0",
   "description": "Core interfaces and Kuralle adapter for messaging platforms",
   "type": "module",
   "main": "dist/index.js",
@@ -26,7 +26,7 @@
   ],
   "dependencies": {
     "hono": "^4.7.4",
-    "@kuralle-agents/core": "0.7.2"
+    "@kuralle-agents/core": "0.8.0"
   },
   "devDependencies": {
     "typescript": "^5.8.2",