ctod 1.0.7 → 1.1.1

package/examples/anthropic.ts

@@ -1,6 +1,6 @@
 import fs from 'fs'
 import { Anthropic } from '@anthropic-ai/sdk'
-import { CtoD, AnthropicCtodService, plugins } from '../lib/index.js'
+import { CtoD, AnthropicCtodService, plugins, paragraph } from '../lib/index.js'
 
 /**
  * @test npx esno ./examples/anthropic.ts
@@ -38,6 +38,15 @@ const brokerBuilder = ctod.createBrokerBuilder<{
                 {
                     role: 'system',
                     content: 'You are now a pharmacist skilled at categorizing indexes'
+                },
+                {
+                    role: 'user',
+                    contents: [
+                        {
+                            type: 'text',
+                            content: 'I have the following indexes'
+                        }
+                    ]
                 }
             ])
         })
@@ -49,11 +58,16 @@ const broker = brokerBuilder.create(async({ zod, data, setMessages }) => {
     setMessages([
         {
             role: 'user',
-            content: [
-                'I have the following indexes',
-                `${JSON.stringify(indexes)}`,
-                `Please help me analyze which index "${question}" might belong to`,
-                'And sort by relevance from high to low with a score ranging from 0 to 1'
+            contents: [
+                {
+                    type: 'text',
+                    content: paragraph([
+                        'I have the following indexes',
+                        `${JSON.stringify(indexes)}`,
+                        `Please help me analyze which index "${question}" might belong to`,
+                        'And sort by relevance from high to low with a score ranging from 0 to 1'
+                    ])
+                }
             ]
         }
     ])

package/examples/aws-bedrock.ts

@@ -1,5 +1,5 @@
 import { BedrockRuntimeClient, InvokeModelCommand } from '@aws-sdk/client-bedrock-runtime'
-import { CtoD, validateToJsonSchema, AnthropicChatDataGenerator, plugins } from '../lib/index.js'
+import { paragraph, CtoD, validateToJsonSchema, AnthropicChatDataGenerator, plugins } from '../lib/index.js'
 
 /**
  * @test npx esno ./examples/aws-bedrock
@@ -67,11 +67,16 @@ const broker = brokerBuilder.create(async ({ zod, data, setMessages }) => {
     setMessages([
         {
             role: 'user',
-            content: [
-                'I have the following indexes',
-                `${JSON.stringify(indexes)}`,
-                `Please help me analyze which index "${question}" might belong to`,
-                'And sort by relevance from high to low with a score ranging from 0 to 1'
+            contents: [
+                {
+                    type: 'text',
+                    content: paragraph([
+                        'I have the following indexes',
+                        `${JSON.stringify(indexes)}`,
+                        `Please help me analyze which index "${question}" might belong to`,
+                        'And sort by relevance from high to low with a score ranging from 0 to 1'
+                    ])
+                }
             ]
         }
     ])

package/examples/google.ts

@@ -1,6 +1,6 @@
 import fs from 'fs'
 import { GoogleGenAI } from '@google/genai'
-import { CtoD, GoogleCtodService, plugins } from '../lib/index.js'
+import { paragraph, CtoD, GoogleCtodService, plugins } from '../lib/index.js'
 
 /**
  * @test npx esno ./examples/google.ts
@@ -53,11 +53,16 @@ const broker = brokerBuilder.create(async({ zod, data, setMessages }) => {
     setMessages([
         {
             role: 'user',
-            content: [
-                'I have the following indexes',
-                `${JSON.stringify(indexes)}`,
-                `Please help me analyze which index "${question}" might belong to`,
-                'And sort by relevance from high to low with a score ranging from 0 to 1'
+            contents: [
+                {
+                    type: 'text',
+                    content: paragraph([
+                        'I have the following indexes',
+                        `${JSON.stringify(indexes)}`,
+                        `Please help me analyze which index "${question}" might belong to`,
+                        'And sort by relevance from high to low with a score ranging from 0 to 1'
+                    ])
+                }
             ]
         }
     ])

package/examples/openai.ts

@@ -1,5 +1,5 @@
 import fs from 'fs'
-import { CtoD, OpenAICtodService, plugins } from '../lib/index.js'
+import { CtoD, OpenAICtodService, plugins, paragraph } from '../lib/index.js'
 
 /**
  * @test npx esno ./examples/openai.ts
@@ -45,11 +45,16 @@ const broker = brokerBuilder.create(async({ zod, data, setMessages }) => {
     setMessages([
         {
             role: 'user',
-            content: [
-                'I have the following indexes',
-                `${JSON.stringify(indexes)}`,
-                `Please help me analyze which index "${question}" might belong to`,
-                'And sort by relevance from high to low with a score ranging from 0 to 1'
+            contents: [
+                {
+                    type: 'text',
+                    content: paragraph([
+                        'I have the following indexes',
+                        `${JSON.stringify(indexes)}`,
+                        `Please help me analyze which index "${question}" might belong to`,
+                        'And sort by relevance from high to low with a score ranging from 0 to 1'
+                    ])
+                }
             ]
         }
     ])

package/examples/x.ts

@@ -1,5 +1,5 @@
 import fs from 'fs'
-import { CtoD, XCtodService, plugins } from '../lib/index.js'
+import { CtoD, XCtodService, plugins, paragraph } from '../lib/index.js'
 
 /**
  * @test npx esno ./examples/x.ts
@@ -45,11 +45,16 @@ const broker = brokerBuilder.create(async({ zod, data, setMessages }) => {
     setMessages([
         {
             role: 'user',
-            content: [
-                'I have the following indexes',
-                `${JSON.stringify(indexes)}`,
-                `Please help me analyze which index "${question}" might belong to`,
-                'And sort by relevance from high to low with a score ranging from 0 to 1'
+            contents: [
+                {
+                    type: 'text',
+                    content: paragraph([
+                        'I have the following indexes',
+                        `${JSON.stringify(indexes)}`,
+                        `Please help me analyze which index "${question}" might belong to`,
+                        'And sort by relevance from high to low with a score ranging from 0 to 1'
+                    ])
+                }
             ]
         }
     ])

package/lib/broker/chat.ts

@@ -5,11 +5,18 @@ import { Translator, TranslatorParams } from '../core/translator.js'
 import { ValidateCallback, ValidateCallbackOutputs, validateToJsonSchema } from '../utils/validate.js'
 import { ParserError } from '../utils/error.js'
 import { z } from 'zod'
+import { CtoD } from '../ctod.js'
+
+export type PolymorphicMessage = {
+    type: 'text' | 'image'
+    content: string
+}
 
 export type Message = {
     role: 'system' | 'user' | 'assistant' | (string & Record<string, unknown>)
     name?: string
-    content: string
+    content?: string
+    contents?: PolymorphicMessage[]
 }
 
 export type ChatBrokerHooks<
@@ -38,7 +45,11 @@ export type ChatBrokerHooks<
             output: O
         }
         messages: Message[]
-        setPreMessages: (messages: (Omit<Message, 'content'> & { content: string | string[] })[]) => void
+        setPreMessages: (messages: (
+            Omit<Message, 'content'>
+            & { content?: string | string[] }
+            & { contents?: PolymorphicMessage[] }
+        )[]) => void
         changeMessages: (messages: Message[]) => void
         changeOutputSchema: (output: O) => void
     }
@@ -213,6 +224,15 @@ export class ChatBroker<
         }
     }
 
+    cloneFrom(ctod: CtoD<any, any>) {
+        const newParams = structuredClone(this.params)
+        newParams.request = ctod.params.request
+        if (ctod.params.plugins) {
+            newParams.plugins = ctod.params.plugins
+        }
+        return new ChatBroker(newParams)
+    }
+
     requestWithId<T extends Translator<S, O>>(data: T['__schemeType']): {
         id: string
         request: Promise<T['__outputType']>

package/lib/ctod.ts

@@ -1,4 +1,4 @@
-import { ChatBroker, Message, Params as ChatBrokerParams, ChatBrokerHooks, RequestContext } from './broker/chat.js'
+import { PolymorphicMessage, ChatBroker, Message, Params as ChatBrokerParams, ChatBrokerHooks, RequestContext } from './broker/chat.js'
 import { ChatBrokerPlugin } from './core/plugin.js'
 import * as z from 'zod'
 
@@ -29,7 +29,11 @@ export class CtoD<
                         send: (data: PS[K]['__receiveData']) => void
                     }
                 }
-                setMessages: (messages: (Omit<Message, 'content'> & { content: string | string[] })[]) => void
+                setMessages: (messages: (
+                    Omit<Message, 'content'>
+                    & { content?: string | string[] }
+                    & { contents?: PolymorphicMessage[] }
+                )[]) => void
                 metadata: Map<string, any>
             }) => Promise<O>) => {
                 return new ChatBroker<
@@ -52,7 +56,8 @@ export class CtoD<
                                     changeMessages(messages.map(e => {
                                         return {
                                             role: e.role,
-                                            content: Array.isArray(e.content) ? e.content.join('\n') : e.content
+                                            content: Array.isArray(e.content) ? e.content.join('\n') : e.content,
+                                            contents: e.contents
                                         }
                                     }))
                                 },

package/lib/index.ts

@@ -4,6 +4,7 @@ export * as chineseConverter from './utils/chinese-conv.js'
 export { CtoD } from './ctod.js'
 export { validateToJsonSchema } from './utils/validate.js'
 export { parseJSONStream } from './utils/json.js'
+export { paragraph } from './utils/paragraph.js'
 export { OpenAICtodService } from './service/openai/index.js'
 export { LlamaCppCtodService } from './service/llama-cpp/index.js'
 export { GoogleCtodService } from './service/google/index.js'
@@ -22,3 +23,4 @@ export { Translator } from './core/translator.js'
 export type { ValidateCallback } from './utils/validate.js'
 export type { TranslatorParams } from './core/translator.js'
 export type { ChatGPTMessage } from './service/openai/chat.js'
+export type { PolymorphicMessage } from './broker/chat.js'

package/lib/service/anthropic/chat.ts

@@ -1,10 +1,13 @@
 import { AnthropicCtodService } from './index.js'
+import { flow } from 'power-helper'
+import { PolymorphicMessage } from '../../broker/chat.js'
 
 type AnthropicSdk = AnthropicCtodService['anthropicSdk']
 
 export type Message = {
     role: string
-    content: string
+    content?: string
+    contents?: PolymorphicMessage[]
 }
 
 export type Config = {
@@ -30,8 +33,66 @@ export class AnthropicChatDataGenerator {
 
     private translateMessages(messages: any[]) {
         return {
-            system: messages.find(e => e.role === 'system')?.content,
-            messages: messages.filter(e => e.role !== 'system')
+            system: flow.run(() => {
+                let systemMessage = messages.find(e => e.role === 'system')
+                let output = ''
+                if (systemMessage) {
+                    if (systemMessage.content) {
+                        output += systemMessage.content
+                    }
+                    if (systemMessage.contents) {
+                        for (let contentBlock of systemMessage.contents) {
+                            if (contentBlock.type === 'text') {
+                                output += `\n${contentBlock.content}`
+                            }
+                        }
+                    }
+                }
+                return output
+            }),
+            messages: messages.filter(e => e.role !== 'system').map(e => {
+                const output: any[] = []
+                if (e.content) {
+                    output.push({
+                        type: 'text',
+                        text: e.content
+                    })
+                }
+                if (e.contents) {
+                    for (let contentBlock of e.contents) {
+                        if (contentBlock.type === 'text') {
+                            output.push({
+                                type: 'text',
+                                text: contentBlock.content
+                            })
+                        }
+                        if (contentBlock.type === 'image') {
+                            if (contentBlock.content.startsWith('http')) {
+                                output.push({
+                                    type: 'image',
+                                    source: {
+                                        type: 'url',
+                                        url: contentBlock.content
+                                    }
+                                })
+                            } else {
+                                output.push({
+                                    type: 'image',
+                                    source: {
+                                        type: 'base64',
+                                        media_type: 'image/png',
+                                        data: contentBlock.content.split(',')[1]
+                                    }
+                                })
+                            }
+                        }
+                    }
+                }
+                return {
+                    role: e.role,
+                    content: output
+                }
+            })
         }
     }
 
@@ -215,12 +276,13 @@ export class AnthropicChat {
         messages: Message[]
         onMessage: (_message: string) => void
         onThinking?: (_thinking: string) => void
-        onEnd: () => void
+        onEnd: (_params: { isManualCancelled: boolean }) => void
         onError: (_error: any) => void
     }) {
+        let isManualCancelled = false
         let stream: Extract<Awaited<ReturnType<typeof anthropic.messages.create>>, { controller: any }> | null = null
-        const anthropic = this.anthropic.anthropicSdk
         const { onThinking, onMessage, onEnd, onError } = params
+        const anthropic = this.anthropic.anthropicSdk
         const body = this.dataGenerator.createTalkStreamBody(params.messages)
         const performStreamedChat = async () => {
             try {
@@ -238,14 +300,18 @@ export class AnthropicChat {
                         }
                     }
                 }
-                onEnd()
+                onEnd({
+                    isManualCancelled
+                })
             } catch (error) {
                 onError(error)
             }
         }
         performStreamedChat()
         return {
+            isManualCancelled: () => isManualCancelled,
             cancel: () => {
+                isManualCancelled = true
                 const int = setInterval(() => {
                     if (stream && stream.controller) {
                         stream.controller.abort()

package/lib/service/anthropic/index.ts

@@ -32,12 +32,22 @@ export class AnthropicCtodService {
                     : e.content.map((content) => {
                         if (content.type === 'image_url') {
                             const url = content.image_url?.url || ''
-                            return {
-                                type: 'image',
-                                source: {
-                                    type: 'base64',
-                                    media_type: url.slice(5).split(';')[0],
-                                    data: url.split(',')[1]
+                            if (url.startsWith('http')) {
+                                return {
+                                    type: 'image',
+                                    source: {
+                                        type: url,
+                                        url: url
+                                    }
+                                }
+                            } else {
+                                return {
+                                    type: 'image',
+                                    source: {
+                                        type: 'base64',
+                                        media_type: url.slice(5).split(';')[0],
+                                        data: url.split(',')[1]
+                                    }
                                 }
                             }
                         }

package/lib/service/google/chat.ts

@@ -14,6 +14,10 @@ type Part = {
         data: string
         mimeType: string
     }
+} | {
+    fileData: {
+        fileUri: string
+    }
 }
 
 export type GoogleMessage = {
@@ -127,11 +131,12 @@ export class GoogleChat {
         system?: string
         messages: GoogleMessage[]
         onMessage: (_message: string) => void
-        onEnd: () => void
+        onEnd: (_params: { isManualCancelled: boolean }) => void
         onThinking?: (_thinking: string) => void
         onError: (_error: any) => void
     }) {
         const state = {
+            isManualCancelled: false,
             controller: new AbortController()
         }
         const model = this.google.googleGenAI.models.generateContentStream({
@@ -166,10 +171,14 @@ export class GoogleChat {
                         }
                     }
                 }
-                params.onEnd()
+                params.onEnd({
+                    isManualCancelled: state.isManualCancelled
+                })
             } catch (error) {
                 if (state.controller.signal.aborted) {
-                    params.onEnd()
+                    params.onEnd({
+                        isManualCancelled: state.isManualCancelled
+                    })
                 } else {
                     throw error
                 }
@@ -179,7 +188,9 @@ export class GoogleChat {
                 params.onError(error)
             })
         return {
+            isManualCancelled: () => state.isManualCancelled,
             cancel: () => {
+                state.isManualCancelled = true
                 state.controller.abort()
             }
         }

package/lib/service/google/index.ts

@@ -1,12 +1,13 @@
 import { validateToJsonSchema } from '../../utils/validate.js'
 import { GoogleMessage, GoogleChat, Config } from './chat.js'
 import { GoogleImagesGeneration } from './images-generation.js'
+import { OpenAIChat } from '../openai/chat.js'
 import type { GoogleGenAI } from '@google/genai'
 
 type GPTContent = {
-    type: 'image_url' | 'text'
+    type: 'image_url' | 'text' | 'input_text' | 'input_image'
     text?: string
-    image_url?: {
+    image_url?: string | {
         url: string
         detail?: string
     }
@@ -37,14 +38,21 @@ export class GoogleCtodService {
                 ]
             } else if (Array.isArray(content)) {
                 return content.map(({ type, image_url, text }): GoogleMessage['parts'][number] => {
-                    if (type === 'image_url') {
-                        // base64
-                        const url = image_url?.url || ''
-                        const mimeType = url.includes('data:image/png') ? 'image/png' : 'image/jpeg'
-                        return {
-                            inlineData: {
-                                data: url.split('base64,')[1] || '',
-                                mimeType
+                    if (type === 'image_url' || type === 'input_image') {
+                        const url = (typeof image_url === 'string' ? image_url : image_url?.url) || ''
+                        if (url.startsWith('http')) {
+                            return {
+                                fileData: {
+                                    fileUri: url
+                                }
+                            }
+                        } else {
+                            const mimeType = url.includes('data:image/png') ? 'image/png' : 'image/jpeg'
+                            return {
+                                inlineData: {
+                                    data: url.split('base64,')[1] || '',
+                                    mimeType
+                                }
                             }
                         }
                     } else {
@@ -59,7 +67,18 @@ export class GoogleCtodService {
         let system = ''
         const outputMessages: GoogleMessage[] = messages.map((message) => {
             if (message.role === 'system') {
-                system = typeof message.content === 'string' ? message.content : ''
+                if (typeof message.content === 'string') {
+                    system = message.content
+                }
+                if (Array.isArray(message.content)) {
+                    system = message.content.map(part => {
+                        if (part.type === 'text' || part.type === 'input_text') {
+                            return part.text || ''
+                        } else {
+                            return ''
+                        }
+                    }).join('\n')
+                }
                 return {
                     role: 'user',
                     parts: []
@@ -101,7 +120,7 @@ export class GoogleCtodService {
         }
         return async (messages: any[], { schema, abortController }: any) => {
             const config = typeof params.config === 'function' ? await params.config() : params.config
-            const context = GoogleCtodService.chatGPTMessageToGoogleChatMessage(messages)
+            const context = GoogleCtodService.chatGPTMessageToGoogleChatMessage(OpenAIChat.toApiMessages(messages) as GPTMessage[])
             const response = await googleGenAI.models.generateContent({
                 model: params.model,
                 contents: context.messages,

package/lib/service/llama-cpp/completion.ts

@@ -2,10 +2,12 @@ import { LlamaCppCtodService } from './index.js'
 import { flow, Once } from 'power-helper'
 import { s2t, t2s } from '../../utils/chinese-conv.js'
 import { Template } from '@huggingface/jinja'
+import { PolymorphicMessage } from '../../broker/chat.js'
 
 type Message = {
     role: string
-    content: string
+    content?: string
+    contents?: PolymorphicMessage[]
 }
 
 type Options = any
@@ -18,12 +20,13 @@ export type Config = {
 
 type Stream = {
     onMessage: (message: string) => void
-    onEnd?: () => void
+    onEnd?: (params: { isManualCancelled: boolean }) => void
     onWarn?: (error: any) => void
     onError?: (error: any) => void
 }
 
 class Requester {
+    isManualCancelled = false
     private core: LlamaCppCompletion
     private streamAbortControllers: {
         id: string
@@ -139,6 +142,7 @@ class Requester {
     }
 
     cancel() {
+        this.isManualCancelled = true
         this.streamAbortControllers.forEach(e => e.controller.abort())
         this.streamAbortControllers = []
     }
@@ -222,7 +226,13 @@ export class LlamaCppCompletion {
         const requester = new Requester(this)
         requester.stream({
             path: 'completion',
-            onEnd: params.onEnd || (() => null),
+            onEnd: () => {
+                if (params.onEnd) {
+                    params.onEnd({
+                        isManualCancelled: requester.isManualCancelled
+                    })
+                }
+            },
             onMessage: e => {
                 const message = this.config.autoConvertTraditionalChinese ? s2t(e.content) : e.content
                 params.onMessage(message)
@@ -266,10 +276,22 @@ export class LlamaCppCompletion {
                         ...(params.options || {}),
                         response_format: params.response_format,
                         messages: params.messages.map(e => {
-                            return {
+                            const output = {
                                 role: e.role,
-                                content: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
+                                content: ''
+                            }
+                            if (e.content) {
+                                output.content = this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
                             }
+                            if (e.contents) {
+                                output.content += e.contents.map(item => {
+                                    if (item.type === 'text') {
+                                        return item.content
+                                    }
+                                    return ''
+                                }).join('\n')
+                            }
+                            return output
                         })
                     }
                 })
@@ -288,7 +310,13 @@ export class LlamaCppCompletion {
         const requester = new Requester(this)
         requester.stream({
             path: 'v1/chat/completions',
-            onEnd: params.onEnd || (() => null),
+            onEnd: () => {
+                if (params.onEnd) {
+                    params.onEnd({
+                        isManualCancelled: requester.isManualCancelled
+                    })
+                }
+            },
             onMessage: e => {
                 let content = e.choices[0].delta.content
                 if (content) {
@@ -304,7 +332,7 @@ export class LlamaCppCompletion {
                 messages: params.messages.map(e => {
                     return {
                         role: e.role,
-                        content: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
+                        content: this.config.autoConvertTraditionalChinese ? t2s(e.content || '') : e.content
                     }
                 })
             }

package/lib/service/llama-cpp/index.ts

@@ -12,13 +12,13 @@ export class LlamaCppCtodService {
         talkOptions?: any
     }) {
         return async(messages: any[], { schema, onCancel }: any) => {
-            const ll3cpp = new LlamaCppCtodService()
-            const chat = ll3cpp.createCompletion()
+            const llamaCpp = new LlamaCppCtodService()
+            const chat = llamaCpp.createCompletion()
             const config = typeof params.config === 'function' ? await params.config() : params.config
             chat.setConfig(config)
             let formatSchema = validateToJsonSchema(schema.output)
             if (params.axios) {
-                ll3cpp.setAxios(params.axios)
+                llamaCpp.setAxios(params.axios)
             }
             if (chat.config.autoConvertTraditionalChinese) {
                 formatSchema = JSON.parse(t2s(JSON.stringify(formatSchema)))