@canonmsg/agent-sdk 0.6.0 → 0.7.0

package/README.md

@@ -77,6 +77,7 @@ The `message` event handler receives a context object with:
 | `replyFinal` | `(text: string, options?) => Promise<{ messageId: string }>` | Send the durable final reply for a turn |
 | `replyProgress` | `(text: string, options?) => Promise<{ turnId: string; durable: boolean; messageId: string \| null }>` | Update the live turn progress; add `durable: true` to also persist it |
 | `agent` | `AgentContext` | Trusted Canon agent identity and access context |
+| `media` | `{ materialize, uploadFile, replyWithFile }` | Canon-managed access to real media bytes via `~/.canon/media-cache` plus local-file uploads back into Canon |
 | `session` | `SessionInfo \| undefined` | Per-conversation queue/session state when sessions are enabled |
 | `turn` | `TurnController \| undefined` | Live turn-state helpers for thinking/streaming/tool/waiting-input |
 
@@ -114,6 +115,29 @@ When `sessions.enabled` is on, the SDK serializes work per conversation and expo
 
 This is the easiest way to build agents that need per-conversation memory or queue awareness.
 
+## Media
+
+Normalized Canon messages always expose `attachments[]` as the canonical media contract. The SDK also keeps `imageUrl` and `audioUrl` on messages for legacy compatibility, but new integrations should prefer `attachments`.
+
+Use the handler `media` helpers when you need the actual file bytes:
+
+```typescript
+agent.on('message', async ({ messages, media }) => {
+  const files = await media.materialize(messages[messages.length - 1]);
+  console.log(files[0]?.path); // ~/.canon/media-cache/<agent>/<conversation>/<message>/...
+});
+```
+
+- `media.materialize(message?)` downloads the message's attachments on demand into `~/.canon/media-cache`.
+- `media.uploadFile(path, options?)` uploads a local file into the current Canon conversation and returns the canonical attachment metadata.
+- `media.replyWithFile(path, text?, options?)` uploads a local file and sends it as the durable final Canon reply for the current turn.
+
+The public helpers are also available from the Node-only subpath export:
+
+```typescript
+import { materializeMessageMedia, uploadMediaFile } from '@canonmsg/agent-sdk/media';
+```
+
 ## Agent Registration
 
 Register a new agent using the static helpers (no API key needed):

package/dist/canon-agent.js

@@ -2,6 +2,7 @@ import { CanonClient, FINAL_MESSAGE_HANDOFF_MS, initRTDBAuth, mergeWorkSessionCo
 import { randomUUID } from 'node:crypto';
 import { AuthManager } from './auth.js';
 import { Debouncer } from './debouncer.js';
+import { materializeMessageMedia, uploadMediaFile, } from './media.js';
 import { PollingManager } from './polling.js';
 import { SessionManager } from './session-manager.js';
 const AUTO_MODE_THRESHOLD = 500;
@@ -153,12 +154,12 @@ export class CanonAgent {
         // Clear session state if enabled (uses cached IDs — no network call during shutdown)
         if (this.options.sessionState && this.agentId) {
             for (const id of this.cachedConversationIds) {
-                clearSessionState(id, this.agentId).catch(() => { });
+                Promise.resolve(clearSessionState(id, this.agentId)).catch(() => { });
             }
         }
         if (this.agentId) {
             for (const id of this.cachedConversationIds) {
-                clearTurnState(id, this.agentId).catch(() => { });
+                Promise.resolve(clearTurnState(id, this.agentId)).catch(() => { });
             }
         }
         await this.clearAgentRuntime();
@@ -197,7 +198,7 @@ export class CanonAgent {
     async clearAgentRuntime() {
         if (!this.agentId)
             return;
-        await rtdbWrite(`/agent-runtime/${this.agentId}`, null).catch(() => { });
+        await Promise.resolve(rtdbWrite(`/agent-runtime/${this.agentId}`, null)).catch(() => { });
     }
     async handleMessages(conversationId, messages) {
         if (!this.handler) {
@@ -226,7 +227,7 @@ export class CanonAgent {
             if (!agentId)
                 return;
             turnState = state;
-            await writeTurnState(conversationId, agentId, {
+            await Promise.resolve(writeTurnState(conversationId, agentId, {
                 turnId,
                 state,
                 queueDepth: queueDepth(),
@@ -236,7 +237,7 @@ export class CanonAgent {
                 ...(state === 'completed' || state === 'interrupted' || state === 'idle'
                     ? { completedAt: { '.sv': 'timestamp' } }
                     : {}),
-            }).catch(() => { });
+            })).catch(() => { });
         };
         const setLiveState = async (state, text, streamingStatus) => {
             await writeTurn(state);
@@ -277,6 +278,16 @@ export class CanonAgent {
             // Fetch hydrated history/context from API
             const page = await this.apiClient.getMessagesPage(conversationId, this.options.historyLimit);
             const history = page.messages;
+            const historyById = new Map(history.map((message) => [message.id, message]));
+            const hydratedMessages = messages.map((message) => {
+                const hydrated = historyById.get(message.id);
+                if (hydrated)
+                    return hydrated;
+                return {
+                    ...message,
+                    attachments: message.attachments ?? [],
+                };
+            });
             // If sessions enabled, seed the session with fetched history
             if (this.sessionManager && session) {
                 this.sessionManager.seedHistory(conversationId, history);
@@ -368,9 +379,43 @@ export class CanonAgent {
                     ...options,
                 });
             };
+            const uploadFile = (filePath, options) => uploadMediaFile(this.apiClient, conversationId, filePath, options);
+            const replyWithFile = async (filePath, text = '', options) => {
+                try {
+                    await this.apiClient.setTyping(conversationId, true, 'typing');
+                }
+                catch { }
+                try {
+                    const uploaded = await uploadFile(filePath, options);
+                    const result = await this.apiClient.sendMessage(conversationId, text, {
+                        ...(options?.replyTo ? { replyTo: options.replyTo } : {}),
+                        ...(options?.replyToPosition != null
+                            ? { replyToPosition: options.replyToPosition }
+                            : {}),
+                        ...(options?.mentions ? { mentions: options.mentions } : {}),
+                        metadata: {
+                            ...(options?.metadata ?? {}),
+                            turnId,
+                            turnSemantics: 'turn_complete',
+                            turnComplete: true,
+                        },
+                        ...(options?.workSessionId ? { workSessionId: options.workSessionId } : {}),
+                        contentType: uploaded.attachment.kind,
+                        attachments: [uploaded.attachment],
+                    });
+                    await sleep(FINAL_MESSAGE_HANDOFF_MS);
+                    return result;
+                }
+                finally {
+                    try {
+                        await this.apiClient.setTyping(conversationId, false);
+                    }
+                    catch { }
+                }
+            };
             // Invoke handler
             await this.handler({
-                messages,
+                messages: hydratedMessages,
                 history,
                 conversationId,
                 conversation,
@@ -390,6 +435,19 @@ export class CanonAgent {
                 agent,
                 workSession,
                 activeWorkSessions,
+                media: {
+                    materialize: (message = hydratedMessages[hydratedMessages.length - 1], options) => {
+                        if (!message)
+                            return Promise.resolve([]);
+                        return materializeMessageMedia(message, {
+                            agentId: agent.agentId,
+                            conversationId,
+                            ...(options ?? {}),
+                        });
+                    },
+                    uploadFile,
+                    replyWithFile,
+                },
                 session: session
                     ? {
                         id: session.id,
@@ -461,7 +519,7 @@ export class CanonAgent {
             }
             catch { }
             if (agentId && !shouldPersistTurnState) {
-                await clearTurnState(conversationId, agentId).catch(() => { });
+                await Promise.resolve(clearTurnState(conversationId, agentId)).catch(() => { });
             }
         }
     }

package/dist/index.d.ts

@@ -1,6 +1,8 @@
 export { CanonAgent } from './canon-agent.js';
 export { CanonApiError } from '@canonmsg/core';
 export { SessionManager } from './session-manager.js';
+export { getMessageAttachments, inferUploadMimeType, materializeAttachment, materializeMessageMedia, uploadMediaFile, } from './media.js';
+export type { MaterializeMediaOptions, MaterializedCanonAttachment, ReplyWithFileOptions, UploadMediaFileOptions, } from './media.js';
 export type { SessionConfig, Session } from './session-manager.js';
 export type { AgentContext, CanonMessage, CanonConversation, CanonResolvedWorkSession, CanonWorkSession, CanonWorkSessionContext, CanonWorkSessionConversationRole, CanonWorkSessionDisclosureMode, CanonWorkSessionParticipant, CanonWorkSessionStatus, CreateWorkSessionOptions, SendLinkedMessageOptions, SendLinkedMessageResult, SendMessageOptions, CreateConversationOptions, UpdateWorkSessionConversationOptions, } from '@canonmsg/core';
 export type { SDKMessage, SDKConversation, CanonAgentOptions, MessageHandler, MessageHandlerContext, ProgressMessageOptions, ProgressMessageResult, SessionInfo, SessionOptions, DeliveryMode, } from './types.js';

package/dist/index.js

@@ -1,3 +1,4 @@
 export { CanonAgent } from './canon-agent.js';
 export { CanonApiError } from '@canonmsg/core';
 export { SessionManager } from './session-manager.js';
+export { getMessageAttachments, inferUploadMimeType, materializeAttachment, materializeMessageMedia, uploadMediaFile, } from './media.js';

package/dist/media.d.ts

@@ -0,0 +1,32 @@
+import { CanonClient, type CanonMessage, type MediaAttachment, type SendMessageOptions } from '@canonmsg/core';
+export interface MaterializeMediaOptions {
+    agentId: string;
+    conversationId: string;
+    messageId: string;
+    rootDir?: string;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}
+export interface UploadMediaFileOptions {
+    fileName?: string;
+    mimeType?: string;
+}
+export interface ReplyWithFileOptions extends Omit<SendMessageOptions, 'attachments' | 'contentType'>, UploadMediaFileOptions {
+}
+export interface MaterializedCanonAttachment extends MediaAttachment {
+    index: number;
+    path: string;
+    sourceUrl: string;
+    conversationId: string;
+    messageId: string;
+}
+export declare function getMessageAttachments(message: Pick<CanonMessage, 'attachments' | 'imageUrl' | 'audioUrl' | 'audioDurationMs'>): MediaAttachment[];
+export declare function materializeAttachment(attachment: MediaAttachment, options: MaterializeMediaOptions & {
+    index?: number;
+}): Promise<MaterializedCanonAttachment>;
+export declare function materializeMessageMedia(message: Pick<CanonMessage, 'id' | 'attachments' | 'imageUrl' | 'audioUrl' | 'audioDurationMs'>, options: Omit<MaterializeMediaOptions, 'messageId'>): Promise<MaterializedCanonAttachment[]>;
+export declare function inferUploadMimeType(filePath: string, overrideMimeType?: string): string;
+export declare function uploadMediaFile(client: CanonClient, conversationId: string, filePath: string, options?: UploadMediaFileOptions): Promise<{
+    url: string;
+    attachment: MediaAttachment;
+}>;

package/dist/media.js

@@ -0,0 +1,166 @@
+import { mkdir, readFile, stat, writeFile } from 'node:fs/promises';
+import { basename, dirname, extname, join } from 'node:path';
+import { CANON_DIR, } from '@canonmsg/core';
+const DEFAULT_MEDIA_CACHE_DIR = join(CANON_DIR, 'media-cache');
+const EXTENSION_BY_MIME = {
+    'application/json': 'json',
+    'application/pdf': 'pdf',
+    'application/zip': 'zip',
+    'audio/mp4': 'm4a',
+    'audio/mpeg': 'mp3',
+    'audio/ogg': 'ogg',
+    'audio/webm': 'webm',
+    'audio/wav': 'wav',
+    'image/gif': 'gif',
+    'image/jpeg': 'jpg',
+    'image/png': 'png',
+    'image/webp': 'webp',
+    'text/plain': 'txt',
+};
+const MIME_BY_EXTENSION = {
+    '.gif': 'image/gif',
+    '.jpeg': 'image/jpeg',
+    '.jpg': 'image/jpeg',
+    '.json': 'application/json',
+    '.m4a': 'audio/mp4',
+    '.mp3': 'audio/mpeg',
+    '.ogg': 'audio/ogg',
+    '.pdf': 'application/pdf',
+    '.png': 'image/png',
+    '.txt': 'text/plain',
+    '.wav': 'audio/wav',
+    '.webm': 'audio/webm',
+    '.webp': 'image/webp',
+    '.zip': 'application/zip',
+};
+function sanitizeSegment(value, fallback) {
+    const sanitized = value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
+    return sanitized || fallback;
+}
+function sanitizeFileName(fileName, fallback) {
+    const sanitized = basename(fileName).replace(/[^a-zA-Z0-9._-]+/g, '_');
+    return sanitized || fallback;
+}
+function inferExtension(input) {
+    const explicitExtension = extname(input.attachment.fileName ?? '').toLowerCase();
+    if (explicitExtension) {
+        return explicitExtension.replace(/[^a-z0-9.]/g, '') || '.bin';
+    }
+    const mimeType = input.responseMimeType ?? input.attachment.mimeType ?? '';
+    const byMime = EXTENSION_BY_MIME[mimeType.toLowerCase()];
+    if (byMime) {
+        return `.${byMime}`;
+    }
+    const urlPath = input.attachment.url.split('?')[0] ?? '';
+    const urlExtension = extname(urlPath).toLowerCase();
+    if (urlExtension) {
+        return urlExtension.replace(/[^a-z0-9.]/g, '') || '.bin';
+    }
+    return '.bin';
+}
+function buildCachePath(input) {
+    const rootDir = input.rootDir ?? DEFAULT_MEDIA_CACHE_DIR;
+    const extension = inferExtension({
+        attachment: input.attachment,
+        responseMimeType: input.responseMimeType,
+    });
+    const fallbackName = `${input.attachment.kind}-${input.index}${extension}`;
+    const preferredName = input.attachment.fileName
+        ? sanitizeFileName(input.attachment.fileName, fallbackName)
+        : fallbackName;
+    const fileName = extname(preferredName) ? preferredName : `${preferredName}${extension}`;
+    return join(rootDir, sanitizeSegment(input.agentId, 'agent'), sanitizeSegment(input.conversationId, 'conversation'), sanitizeSegment(input.messageId, 'message'), `${String(input.index).padStart(2, '0')}-${fileName}`);
+}
+function ensureFetch(fetchImpl) {
+    if (fetchImpl)
+        return fetchImpl;
+    if (typeof fetch === 'function')
+        return fetch;
+    throw new Error('Global fetch is unavailable; provide fetchImpl explicitly.');
+}
+async function fileExists(path) {
+    try {
+        const info = await stat(path);
+        return info.isFile();
+    }
+    catch {
+        return false;
+    }
+}
+export function getMessageAttachments(message) {
+    if (Array.isArray(message.attachments) && message.attachments.length > 0) {
+        return message.attachments;
+    }
+    if (message.audioUrl) {
+        return [{
+                kind: 'audio',
+                url: message.audioUrl,
+                ...(typeof message.audioDurationMs === 'number'
+                    ? { durationMs: message.audioDurationMs }
+                    : {}),
+            }];
+    }
+    if (message.imageUrl) {
+        return [{
+                kind: 'image',
+                url: message.imageUrl,
+            }];
+    }
+    return [];
+}
+export async function materializeAttachment(attachment, options) {
+    const path = buildCachePath({
+        agentId: options.agentId,
+        conversationId: options.conversationId,
+        messageId: options.messageId,
+        attachment,
+        index: options.index ?? 0,
+        rootDir: options.rootDir,
+    });
+    await mkdir(dirname(path), { recursive: true });
+    let responseMimeType = null;
+    if (!(await fileExists(path))) {
+        const fetchImpl = ensureFetch(options.fetchImpl);
+        const response = await fetchImpl(attachment.url, {
+            signal: options.signal,
+        });
+        if (!response.ok) {
+            throw new Error(`Failed to download Canon media (${response.status} ${response.statusText})`);
+        }
+        responseMimeType = response.headers.get('content-type');
+        const body = Buffer.from(await response.arrayBuffer());
+        await writeFile(path, body);
+    }
+    return {
+        ...attachment,
+        index: options.index ?? 0,
+        path,
+        sourceUrl: attachment.url,
+        conversationId: options.conversationId,
+        messageId: options.messageId,
+        ...(attachment.mimeType
+            ? {}
+            : responseMimeType
+                ? { mimeType: responseMimeType }
+                : {}),
+    };
+}
+export async function materializeMessageMedia(message, options) {
+    const attachments = getMessageAttachments(message);
+    return Promise.all(attachments.map((attachment, index) => materializeAttachment(attachment, {
+        ...options,
+        messageId: message.id,
+        index,
+    })));
+}
+export function inferUploadMimeType(filePath, overrideMimeType) {
+    if (overrideMimeType)
+        return overrideMimeType;
+    return MIME_BY_EXTENSION[extname(filePath).toLowerCase()] ?? 'application/octet-stream';
+}
+export async function uploadMediaFile(client, conversationId, filePath, options) {
+    const buffer = await readFile(filePath);
+    const mimeType = inferUploadMimeType(filePath, options?.mimeType);
+    const fileName = options?.fileName ?? basename(filePath);
+    return client.uploadMedia(conversationId, buffer.toString('base64'), mimeType, fileName);
+}

package/dist/types.d.ts

@@ -1,5 +1,6 @@
 export type { AgentClientType, CanonMessage, CanonConversation, AgentContext, CanonResolvedWorkSession, CanonWorkSession, CanonWorkSessionContext, CanonWorkSessionConversationRole, CanonWorkSessionDisclosureMode, CanonWorkSessionParticipant, CanonWorkSessionStatus, CreateWorkSessionOptions, SendLinkedMessageOptions, SendLinkedMessageResult, SendMessageOptions, CreateConversationOptions, TurnLifecycleState, UpdateWorkSessionConversationOptions, } from '@canonmsg/core';
 import type { CanonMessage, CanonConversation, CreateWorkSessionOptions, SendMessageOptions, UpdateWorkSessionConversationOptions } from '@canonmsg/core';
+import type { MaterializeMediaOptions, MaterializedCanonAttachment, ReplyWithFileOptions, UploadMediaFileOptions } from './media.js';
 export type SDKMessage = CanonMessage;
 export type SDKConversation = CanonConversation;
 export interface ProgressMessageOptions extends SendMessageOptions {
@@ -73,6 +74,17 @@ export interface MessageHandlerContext {
     workSession?: import('@canonmsg/core').CanonWorkSessionContext | null;
     /** All active Canon work sessions currently linked to this conversation. */
     activeWorkSessions?: import('@canonmsg/core').CanonWorkSessionContext[];
+    /** Canon-managed local media access for the current conversation. */
+    media: {
+        materialize: (message?: SDKMessage, options?: Omit<MaterializeMediaOptions, 'agentId' | 'conversationId' | 'messageId'>) => Promise<MaterializedCanonAttachment[]>;
+        uploadFile: (filePath: string, options?: UploadMediaFileOptions) => Promise<{
+            url: string;
+            attachment: import('@canonmsg/core').MediaAttachment;
+        }>;
+        replyWithFile: (filePath: string, text?: string, options?: ReplyWithFileOptions) => Promise<{
+            messageId: string;
+        }>;
+    };
     /** Per-conversation session state. Present when sessions are enabled. */
     session?: SessionInfo;
     /** Turn lifecycle helpers for live-work rendering and progress reporting. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@canonmsg/agent-sdk",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Canon Agent SDK — build AI agents that participate in Canon conversations",
   "type": "module",
   "main": "dist/index.js",
@@ -9,6 +9,10 @@
     ".": {
       "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
+    },
+    "./media": {
+      "import": "./dist/media.js",
+      "types": "./dist/media.d.ts"
     }
   },
   "files": [
@@ -17,6 +21,7 @@
   "scripts": {
     "build": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\" && tsc",
     "dev": "tsc --watch",
+    "test": "vitest run",
     "prepack": "npm run build"
   },
   "engines": {
@@ -30,7 +35,8 @@
   },
   "devDependencies": {
     "@types/node": "^22.0.0",
-    "typescript": "~5.7.0"
+    "typescript": "~5.7.0",
+    "vitest": "^3.0.0"
   },
   "license": "MIT"
 }