ctod 0.8.2 → 0.8.4

package/examples/anthropic.ts

@@ -0,0 +1,101 @@
+import fs from 'fs'
+import { Anthropic } from '@anthropic-ai/sdk'
+import { CtoD, AnthropicCtodService, plugins } from '../lib/index'
+
+/**
+ * @test npx esno ./examples/anthropic.ts
+ * 必須手動安裝 '@anthropic-ai/sdk' 套件
+ */
+
+const apiKey = fs.readFileSync('./.anthropic-api-key', 'utf-8').trim()
+
+const ctod = new CtoD({
+    plugins: () => {
+        return {
+            retry: plugins.RetryPlugin.use({
+                retry: 3,
+                printWarn: true
+            })
+        }
+    },
+    request: AnthropicCtodService.createChatRequestWithJsonSchema({
+        config: {
+            model: 'claude-3-5-haiku-latest'
+        },
+        anthropicSdk: new Anthropic({
+            apiKey
+        })
+    })
+})
+
+const brokerBuilder = ctod.createBrokerBuilder<{
+    indexes: string[]
+    question: string
+}>({
+    install: ({ attach }) => {
+        attach('start', async({ setPreMessages }) => {
+            setPreMessages([
+                {
+                    role: 'system',
+                    content: '你現在是一位擅長分類索引的藥師'
+                }
+            ])
+        })
+    }
+})
+
+const broker = brokerBuilder.create(async({ yup, data, setMessages }) => {
+    const { indexes, question } = data
+    setMessages([
+        {
+            role: 'user',
+            content: [
+                '我有以下索引',
+                `${JSON.stringify(indexes)}`,
+                `請幫我解析"${question}"可能是哪個索引`,
+                '且相關性由高到低排序並給予分數，分數由 0 ~ 1'
+            ]
+        }
+    ])
+    const item = yup.object({
+        name: yup.string().required().meta({
+            jsonSchema: {
+                description: '索引名稱'
+            }
+        }),
+        score: yup.number().required().meta({
+            jsonSchema: {
+                description: '評比分數'
+            }
+        })
+    }).required()
+    return {
+        indexes: yup.array(item).required().meta({
+            jsonSchema: {
+                description: '由高到低排序的索引'
+            }
+        })
+    }
+})
+
+broker.request({
+    indexes: ['胃痛', '腰痛', '頭痛', '喉嚨痛', '四肢疼痛'],
+    question: '喝咖啡，吃甜食，胃食道逆流'
+}).then(e => {
+    console.log('輸出結果：', e.indexes)
+    /*
+        [
+            {
+                name: '胃痛',
+                score: 1
+            },
+            {
+                name: '喉嚨痛',
+                score: 0.7
+            },
+            ...
+        ]
+     */
+}).catch(error => {
+    console.error('Error:', error)
+})

package/lib/index.ts

@@ -6,6 +6,7 @@ export { validateToJsonSchema, defineYupSchema } from './utils/validate'
 export { OpenAICtodService } from './service/openai'
 export { Llama3CppCtodService } from './service/llama3.cpp'
 export { GoogleCtodService } from './service/google'
+export { AnthropicCtodService } from './service/anthropic'
 
 export { TextParser } from './core/parser'
 export { ChatBroker } from './broker/chat'

package/lib/service/anthropic/chat.ts

@@ -0,0 +1,146 @@
+import { AnthropicCtodService } from './index'
+import { PromiseResponseType } from '../../types'
+
+export type Message = {
+    role: string
+    content: string
+}
+
+export type Config = {
+    /**
+     * @zh 選擇運行的模型。
+     * @en What model to use.
+     */
+    model: string
+    maxTokens: number
+}
+
+export class AnthropicChat {
+    anthropic: AnthropicCtodService
+    config: Config = {
+        model: 'claude-3-5-haiku-latest',
+        maxTokens: 8192
+    }
+
+    constructor(anthropic: AnthropicCtodService) {
+        this.anthropic = anthropic
+    }
+
+    /**
+     * @zh 改變對話的一些設定
+     * @en Change some settings of the conversation
+     */
+
+    setConfig(options: Partial<Config>) {
+        Object.assign(this.config, options)
+    }
+
+    /**
+     * 移除 system 訊息
+     */
+
+    private translateMessages(messages: any[]) {
+        return {
+            system: messages.find(e => e.role === 'system')?.content[0].text,
+            messages: messages.filter(e => e.role !== 'system')
+        }
+    }
+
+    /**
+     * @zh 進行對話，並且以結構化的方式輸出
+     * @en Talk to the AI and output in a structured way
+     */
+
+    async chatAndStructure(messages: Message[], jsonSchema: any) {
+        const anthropic = this.anthropic.anthropicSdk
+        const translateMessages = this.translateMessages(messages)
+        const msg = await anthropic.messages.create({
+            model: this.config.model,
+            max_tokens: this.config.maxTokens,
+            system: translateMessages.system,
+            messages: translateMessages.messages,
+            tools: [
+                {
+                    name: 'data',
+                    description: 'Response Data',
+                    input_schema: jsonSchema
+                }
+            ],
+            tool_choice: {
+                type: 'tool',
+                name: 'data'
+            }
+        })
+        let toolUseContent: any = msg.content.find(e => e.type === 'tool_use')
+        let response = toolUseContent?.input || null
+        if (response == null) {
+            return 'null'
+        }
+        return JSON.stringify(response)
+    }
+
+    /**
+     * @zh 進行對話
+     * @en Talk to the AI
+     */
+
+    async talk(messages: Message[] = []) {
+        const anthropic = this.anthropic.anthropicSdk
+        const newMessages = this.translateMessages(messages)
+        const msg = await anthropic.messages.create({
+            model: this.config.model,
+            max_tokens: this.config.maxTokens,
+            system: newMessages.system,
+            messages: newMessages.messages
+        })
+        let output = ''
+        let textContent: any = msg.content.find(e => e.type === 'text')
+        if (textContent) {
+            output = textContent.text
+        }
+        return output
+    }
+
+    /**
+     * @zh 進行對話，並且以串流的方式輸出
+     * @en Talk to the AI and output in a streaming way
+     */
+
+    talkStream(params: {
+        messages: Message[]
+        onMessage: (_message: string) => void
+        onEnd: () => void
+        onWarn: (_warn: any) => void
+        onError: (_error: any) => void
+    }) {
+        const anthropic = this.anthropic.anthropicSdk
+        const { onMessage, onEnd, onError } = params
+        const { messages, system } = this.translateMessages(params.messages)
+        const performStreamedChat = async () => {
+            try {
+                const stream = await anthropic.messages.create({
+                    model: this.config.model,
+                    max_tokens: this.config.maxTokens,
+                    system: system,
+                    stream: true,
+                    messages
+                })
+                for await (const messageStream of stream) {
+                    if (messageStream.type === 'content_block_delta') {
+                        const deltaText = 'text' in messageStream.delta ? messageStream.delta.text : ''
+                        onMessage(deltaText)
+                    }
+                }
+                onEnd()
+            } catch (error) {
+                onError(error)
+            }
+        }
+        performStreamedChat()
+        return {
+            cancel: () => null
+        }
+    }
+}
+
+export type AnthropicChatTalkResponse = PromiseResponseType<AnthropicChat['talk']>

package/lib/service/anthropic/index.ts

@@ -0,0 +1,29 @@
+import { validateToJsonSchema } from '../../utils/validate'
+import { Config, AnthropicChat } from './chat'
+import type { Anthropic } from '@anthropic-ai/sdk'
+
+export class AnthropicCtodService {
+    anthropicSdk: Anthropic
+
+    constructor(anthropicSdk: Anthropic) {
+        this.anthropicSdk = anthropicSdk
+    }
+
+    static createChatRequestWithJsonSchema(params: {
+        anthropicSdk: Anthropic
+        config?: Partial<Config>
+    }) {
+        const anthropic = new AnthropicCtodService(params.anthropicSdk)
+        const chat = anthropic.createChat()
+        chat.setConfig(params.config || {})
+        return async (messages: any[], { schema }: any) => {
+            const jsonSchema = validateToJsonSchema(schema.output)
+            const content = await chat.chatAndStructure(messages, jsonSchema)
+            return content
+        }
+    }
+
+    createChat() {
+        return new AnthropicChat(this)
+    }
+}

package/lib/service/llama3.cpp/completion.ts

@@ -1,6 +1,7 @@
 import { Llama3CppCtodService } from './index'
-import { flow } from 'power-helper'
+import { flow, Once } from 'power-helper'
 import { tify, sify } from 'chinese-conv/dist'
+import { Template } from '@huggingface/jinja'
 
 type Message = {
     role: string
@@ -16,7 +17,7 @@ export type Config = {
 }
 
 type Stream = {
-    onMessage: (data: { message: string }) => void
+    onMessage: (message: string) => void
     onEnd?: () => void
     onWarn?: (error: any) => void
     onError?: (error: any) => void
@@ -52,7 +53,7 @@ class Requester {
 
     async stream(params: {
         path: string
-        data: any
+        data: Record<string, any> | (() => Promise<any>)
         onMessage: (data: any) => void
         onEnd: () => void
         onWarn: (error: any) => void
@@ -99,7 +100,7 @@ class Requester {
         }
         fetch(`${this.core.config.baseUrl}/${params.path}`, {
             method: 'POST',
-            body: JSON.stringify(params.data),
+            body: JSON.stringify(typeof params.data === 'function' ? (await params.data()) : params.data),
             signal,
             headers: {
                 'Content-Type': 'application/json',
@@ -150,6 +151,18 @@ class Requester {
 }
 
 export class Llama3CppCompletion {
+    private getProp = new Once({
+        handler: async() => {
+            const url = `${this.config.baseUrl}/props`
+            const { data: props } = await this.core._axios.get<{
+                chat_template: string
+                bos_token: string
+                eos_token: string
+            }>(url, {})
+            return props
+        }
+    })
+
     core: Llama3CppCtodService
     config: Config = {
         baseUrl: '',
@@ -172,18 +185,6 @@ export class Llama3CppCompletion {
         options?: Options
         messages: Message[]
     }) {
-        const prompts: string[] = []
-        for (let { role, content } of params.messages) {
-            if (role === 'system') {
-                prompts.push(`<|start_header_id|>system<|end_header_id|>\n\n${content}\n\n`)
-            }
-            if (role === 'user') {
-                prompts.push(`<|start_header_id|>user<|end_header_id|>\n\n${content?.replaceAll('\n', '\\n') ?? ''}`)
-            }
-            if (role === 'assistant') {
-                prompts.push('<|start_header_id|>assistant<|end_header_id|>\n\n' + content)
-            }
-        }
         const lastMessage = params.messages.at(-1) || ''
         const requester = new Requester(this)
         return {
@@ -192,11 +193,17 @@ export class Llama3CppCompletion {
                 message: string
                 fullMessage: string
             }> => {
+                const props = await this.getProp.run()
+                const template = new Template(props.chat_template)
+                const prompt = template.render({
+                    bos_token: props.bos_token,
+                    messages: params.messages
+                }).slice(0, props.eos_token.length * -1 - 1)
                 const result = await requester.fetch({
                     path: 'completion',
                     data: {
                         ...(params.options || {}),
-                        prompt: this.config.autoConvertTraditionalChinese ? sify(prompts.join('\n')) : prompts.join('\n')
+                        prompt: this.config.autoConvertTraditionalChinese ? sify(prompt) : prompt
                     }
                 })
                 const message = this.config.autoConvertTraditionalChinese ? tify(result.data.content) : result.data.content
@@ -212,33 +219,28 @@ export class Llama3CppCompletion {
         messages: Message[]
         options?: Options
     }) {
-        const prompts: string[] = []
-        for (let { role, content } of params.messages) {
-            if (role === 'system') {
-                prompts.push(`<|start_header_id|>system<|end_header_id|>\n\n${content}\n\n`)
-            }
-            if (role === 'user') {
-                prompts.push(`<|start_header_id|>user<|end_header_id|>\n\n${content?.replaceAll('\n', '\\n') ?? ''}`)
-            }
-            if (role === 'assistant') {
-                prompts.push('<|start_header_id|>assistant<|end_header_id|>\n\n' + content)
-            }
-        }
         const requester = new Requester(this)
         requester.stream({
             path: 'completion',
             onEnd: params.onEnd || (() => null),
             onMessage: e => {
-                params.onMessage({
-                    message: this.config.autoConvertTraditionalChinese ? tify(e.content) : e.content
-                })
+                const message = this.config.autoConvertTraditionalChinese ? tify(e.content) : e.content
+                params.onMessage(message)
             },
             onWarn: params.onWarn || (() => null),
             onError: params.onError || (() => null),
-            data: {
-                ...(params.options || {}),
-                prompt: this.config.autoConvertTraditionalChinese ? sify(prompts.join('\n')) : prompts.join('\n'),
-                stream: true
+            data: async() => {
+                const props = await this.getProp.run()
+                const template = new Template(props.chat_template)
+                const prompt = template.render({
+                    bos_token: props.bos_token,
+                    messages: params.messages
+                }).slice(0, props.eos_token.length * -1 - 1)
+                return {
+                    ...(params.options || {}),
+                    prompt: this.config.autoConvertTraditionalChinese ? sify(prompt) : prompt,
+                    stream: true
+                }
             }
         })
         return requester.export()
@@ -290,9 +292,8 @@ export class Llama3CppCompletion {
             onMessage: e => {
                 let content = e.choices[0].delta.content
                 if (content) {
-                    params.onMessage({
-                        message: this.config.autoConvertTraditionalChinese ? tify(content) : content
-                    })
+                    const message = this.config.autoConvertTraditionalChinese ? tify(content) : content
+                    params.onMessage(message)
                 }
             },
             onWarn: params.onWarn || (() => null),

package/lib/service/openai/chat.ts

@@ -35,8 +35,8 @@ export type Config = {
      */
     n: number
     /**
-     * @zh 選擇運行的模型，建議: 'gpt-4' | 'gpt-3.5-turbo' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1' | 'o1-mini'
-     * @en What model to use, recommended: 'gpt-4' | 'gpt-3.5-turbo' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1' | 'o1-mini'
+     * @zh 選擇運行的模型'
+     * @en The model to use for this chat completion.
      */
     model: string
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ctod",
-  "version": "0.8.2",
+  "version": "0.8.4",
   "description": "CtoD Is Chat To Data Utils.",
   "main": "./dist/index.js",
   "type": "module",
@@ -26,9 +26,11 @@
   },
   "homepage": "https://github.com/KHC-ZhiHao/ctod#readme",
   "devDependencies": {
+    "@anthropic-ai/sdk": "^0.39.0",
     "@babel/core": "^7.4.5",
     "@babel/preset-env": "^7.4.5",
     "@google/generative-ai": "^0.21.0",
+    "@stylistic/eslint-plugin": "^2.12.1",
     "@types/chai": "^4.2.22",
     "@types/inquirer": "8.2.5",
     "@types/jsdom": "^16.2.13",
@@ -50,16 +52,16 @@
     "nyc": "^15.1.0",
     "ts-loader": "^9.3.1",
     "typescript": "^4.7.4",
-    "typescript-eslint": "^8.18.1",
-    "@stylistic/eslint-plugin": "^2.12.1"
+    "typescript-eslint": "^8.18.1"
   },
   "dependencies": {
+    "@huggingface/jinja": "^0.3.3",
     "@sodaru/yup-to-json-schema": "^2.0.1",
     "axios": "^1.4.0",
     "chinese-conv": "^3.2.2",
     "handlebars": "^4.7.7",
     "json5": "^2.2.3",
-    "power-helper": "^0.8.0",
+    "power-helper": "^0.8.1",
     "yup": "^1.4.0"
   }
 }

package/types/examples/anthropic.d.ts

@@ -0,0 +1 @@
+export {};

package/types/lib/index.d.ts

@@ -5,6 +5,7 @@ export { validateToJsonSchema, defineYupSchema } from './utils/validate';
 export { OpenAICtodService } from './service/openai';
 export { Llama3CppCtodService } from './service/llama3.cpp';
 export { GoogleCtodService } from './service/google';
+export { AnthropicCtodService } from './service/anthropic';
 export { TextParser } from './core/parser';
 export { ChatBroker } from './broker/chat';
 export { ChatBrokerPlugin } from './core/plugin';

package/types/lib/service/anthropic/chat.d.ts

@@ -0,0 +1,52 @@
+import { AnthropicCtodService } from './index';
+import { PromiseResponseType } from '../../types';
+export type Message = {
+    role: string;
+    content: string;
+};
+export type Config = {
+    /**
+     * @zh 選擇運行的模型。
+     * @en What model to use.
+     */
+    model: string;
+    maxTokens: number;
+};
+export declare class AnthropicChat {
+    anthropic: AnthropicCtodService;
+    config: Config;
+    constructor(anthropic: AnthropicCtodService);
+    /**
+     * @zh 改變對話的一些設定
+     * @en Change some settings of the conversation
+     */
+    setConfig(options: Partial<Config>): void;
+    /**
+     * 移除 system 訊息
+     */
+    private translateMessages;
+    /**
+     * @zh 進行對話，並且以結構化的方式輸出
+     * @en Talk to the AI and output in a structured way
+     */
+    chatAndStructure(messages: Message[], jsonSchema: any): Promise<string>;
+    /**
+     * @zh 進行對話
+     * @en Talk to the AI
+     */
+    talk(messages?: Message[]): Promise<string>;
+    /**
+     * @zh 進行對話，並且以串流的方式輸出
+     * @en Talk to the AI and output in a streaming way
+     */
+    talkStream(params: {
+        messages: Message[];
+        onMessage: (_message: string) => void;
+        onEnd: () => void;
+        onWarn: (_warn: any) => void;
+        onError: (_error: any) => void;
+    }): {
+        cancel: () => null;
+    };
+}
+export type AnthropicChatTalkResponse = PromiseResponseType<AnthropicChat['talk']>;

package/types/lib/service/anthropic/index.d.ts

@@ -0,0 +1,11 @@
+import { Config, AnthropicChat } from './chat';
+import type { Anthropic } from '@anthropic-ai/sdk';
+export declare class AnthropicCtodService {
+    anthropicSdk: Anthropic;
+    constructor(anthropicSdk: Anthropic);
+    static createChatRequestWithJsonSchema(params: {
+        anthropicSdk: Anthropic;
+        config?: Partial<Config>;
+    }): (messages: any[], { schema }: any) => Promise<string>;
+    createChat(): AnthropicChat;
+}

package/types/lib/service/llama3.cpp/completion.d.ts

@@ -10,14 +10,13 @@ export type Config = {
     autoConvertTraditionalChinese: boolean;
 };
 type Stream = {
-    onMessage: (data: {
-        message: string;
-    }) => void;
+    onMessage: (message: string) => void;
     onEnd?: () => void;
     onWarn?: (error: any) => void;
     onError?: (error: any) => void;
 };
 export declare class Llama3CppCompletion {
+    private getProp;
     core: Llama3CppCtodService;
     config: Config;
     constructor(core: Llama3CppCtodService);

package/types/lib/service/openai/chat.d.ts

@@ -31,8 +31,8 @@ export type Config = {
      */
     n: number;
     /**
-     * @zh 選擇運行的模型，建議: 'gpt-4' | 'gpt-3.5-turbo' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1' | 'o1-mini'
-     * @en What model to use, recommended: 'gpt-4' | 'gpt-3.5-turbo' | 'gpt-4-turbo' | 'gpt-4o' | 'gpt-4o-mini' | 'o1-preview' | 'o1' | 'o1-mini'
+     * @zh 選擇運行的模型'
+     * @en The model to use for this chat completion.
      */
     model: string;
     /**