ctod 1.1.1 → 1.1.2

package/lib/service/llama-cpp/completion.ts

@@ -11,6 +11,13 @@ type Message = {
 }
 
 type Options = any
+type Content = {
+    type: 'text' | 'image_url'
+    text?: string
+    image_url?: {
+        url: string
+    }
+}[]
 
 export type Config = {
     baseUrl: string
@@ -201,7 +208,22 @@ export class LlamaCppCompletion {
                 const template = new Template(props.chat_template)
                 const prompt = template.render({
                     bos_token: props.bos_token,
-                    messages: params.messages
+                    messages: params.messages.map(e => {
+                        const output: any = { role: e.role, content: [] as Content }
+                        if (e.content) {
+                            output.content.push({ type: 'text', text: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content })
+                        }
+                        if (e.contents) {
+                            e.contents.forEach(item => {
+                                if (item.type === 'text') {
+                                    output.content.push({ type: 'text', text: this.config.autoConvertTraditionalChinese ? t2s(item.content) : item.content })
+                                } else if (item.type === 'image') {
+                                    output.content.push({ type: 'image_url', image_url: { url: item.content } })
+                                }
+                            })
+                        }
+                        return output
+                    })
                 }).slice(0, props.eos_token.length * -1 - 1)
                 const result = await requester.fetch({
                     path: 'completion',
@@ -244,7 +266,22 @@ export class LlamaCppCompletion {
                 const template = new Template(props.chat_template)
                 const prompt = template.render({
                     bos_token: props.bos_token,
-                    messages: params.messages
+                    messages: params.messages.map(e => {
+                        const output: any = { role: e.role, content: [] as Content }
+                        if (e.content) {
+                            output.content.push({ type: 'text', text: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content })
+                        }
+                        if (e.contents) {
+                            e.contents.forEach(item => {
+                                if (item.type === 'text') {
+                                    output.content.push({ type: 'text', text: this.config.autoConvertTraditionalChinese ? t2s(item.content) : item.content })
+                                } else if (item.type === 'image') {
+                                    output.content.push({ type: 'image_url', image_url: { url: item.content } })
+                                }
+                            })
+                        }
+                        return output
+                    })
                 }).slice(0, props.eos_token.length * -1 - 1)
                 return {
                     ...(params.options || {}),
@@ -278,18 +315,30 @@ export class LlamaCppCompletion {
                         messages: params.messages.map(e => {
                             const output = {
                                 role: e.role,
-                                content: ''
+                                content: [] as Content
                             }
                             if (e.content) {
-                                output.content = this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
+                                output.content.push({
+                                    type: 'text',
+                                    text: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
+                                })
                             }
                             if (e.contents) {
-                                output.content += e.contents.map(item => {
+                                e.contents.forEach(item => {
                                     if (item.type === 'text') {
-                                        return item.content
+                                        output.content.push({
+                                            type: 'text',
+                                            text: this.config.autoConvertTraditionalChinese ? t2s(item.content) : item.content
+                                        })
+                                    } else if (item.type === 'image') {
+                                        output.content.push({
+                                            type: 'image_url',
+                                            image_url: {
+                                                url: item.content
+                                            }
+                                        })
                                     }
-                                    return ''
-                                }).join('\n')
+                                })
                             }
                             return output
                         })
@@ -330,10 +379,34 @@ export class LlamaCppCompletion {
                 ...(params.options || {}),
                 stream: true,
                 messages: params.messages.map(e => {
-                    return {
+                    const output = {
                         role: e.role,
-                        content: this.config.autoConvertTraditionalChinese ? t2s(e.content || '') : e.content
+                        content: [] as Content
+                    }
+                    if (e.content) {
+                        output.content.push({
+                            type: 'text',
+                            text: this.config.autoConvertTraditionalChinese ? t2s(e.content) : e.content
+                        })
+                    }
+                    if (e.contents) {
+                        e.contents.forEach(item => {
+                            if (item.type === 'text') {
+                                output.content.push({
+                                    type: 'text',
+                                    text: this.config.autoConvertTraditionalChinese ? t2s(item.content) : item.content
+                                })
+                            } else if (item.type === 'image') {
+                                output.content.push({
+                                    type: 'image_url',
+                                    image_url: {
+                                        url: item.content
+                                    }
+                                })
+                            }
+                        })
                     }
+                    return output
                 })
             }
         })

package/lib/service/llama-cpp/index.ts

@@ -33,7 +33,8 @@ export class LlamaCppCtodService {
             })
             onCancel(cancel)
             const { message } = await run()
-            return chat.config.autoConvertTraditionalChinese ? s2t(message) : message
+            const stringMessage = typeof message === 'string' ? message : JSON.stringify(message)
+            return chat.config.autoConvertTraditionalChinese ? s2t(stringMessage) : stringMessage
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ctod",
-  "version": "1.1.1",
+  "version": "1.1.2",
   "description": "CtoD Is Chat To Data Utils.",
   "main": "./dist/index.js",
   "type": "module",

package/skills/ctod/SKILL.md

@@ -0,0 +1,105 @@
+---
+name: ctod
+description: Use this skill whenever writing or generating BrokerBuilder and Broker code with CtoD. Trigger this skill whenever the user asks to add or modify a Broker, or define LLM input/output structures.
+---
+
+# CtoD — BrokerBuilder & Broker Generation Guide
+
+## Step 1: Create a BrokerBuilder
+
+Define the input type and shared system prompt.
+
+```ts
+// This is just for reference. In best practice, the ctod instance should already be initialized
+// in a shared place — no need to recreate it in every skill.
+import { CtoD } from '../lib/index.js'
+const ctod = new CtoD({ ... })
+
+const brokerBuilder = ctod.createBrokerBuilder<{
+    question: string   // input type
+}>({
+    install: ({ attach }) => {
+        attach('start', async ({ setPreMessages }) => {
+            setPreMessages([
+                { role: 'system', content: 'You are a helpful assistant.' }
+            ])
+        })
+    }
+})
+```
+
+## Step 2: Create a Broker
+
+```ts
+const broker = brokerBuilder.create(async ({ zod, data, setMessages }) => {
+    setMessages([
+        {
+            role: 'user',
+            contents: [{ type: 'text', content: `Question: ${data.question}` }]
+        }
+    ])
+    return {
+        answer: zod.string().describe('The answer'),
+        confidence: zod.number().min(0).max(1).describe('Confidence score')
+    }
+})
+
+const result = await broker.request({ question: '...' })
+```
+
+## Message Formats
+
+```ts
+// Plain text (shorthand)
+{ role: 'user', content: 'Hello' }
+
+// Multi-content (with image)
+{ role: 'user', contents: [
+    { type: 'text', content: 'What is in this image?' },
+    { type: 'image', content: 'data:image/png;base64,...' }
+]}
+
+// Multi-line text composition
+import { paragraph } from 'ctod'
+paragraph(['First paragraph', JSON.stringify(data), `Question: ${question}`])
+```
+
+## Output Schema Patterns
+
+```ts
+// Single value
+{ answer: zod.string().describe('...') }
+
+// Array
+{
+    items: zod.array(
+        zod.object({
+            name: zod.string().describe('Name'),
+            score: zod.number().describe('Score from 0 to 1')
+        })
+    ).describe('Sorted by relevance')
+}
+```
+
+## Hooks (used inside BrokerBuilder install)
+
+| Hook | Triggered when | Common params |
+|---|---|---|
+| `start` | Request starts | `setPreMessages`, `changeMessages`, `data`, `metadata` |
+| `talkBefore` | Before sending | `messages`, `lastUserMessage` |
+| `talkAfter` | After LLM responds | `changeParseText`, `parseFail` |
+| `succeeded` | Parse succeeded | `output`, `metadata` |
+| `parseFailed` | Parse failed | `retry`, `changeMessages` |
+| `done` | Request ends | `id`, `metadata` |
+
+```ts
+// metadata shares data across hooks
+attach('start', async ({ metadata }) => { metadata.set('key', value) })
+attach('done',  async ({ metadata }) => { metadata.get('key') })
+```
+
+## Notes
+
+- `zod` is injected by ctod — **no need to import it separately**
+- `setPreMessages` can only be used in the `start` hook
+- `changeMessages` can be used in multiple hooks