@llumiverse/core 0.13.0 → 0.15.0

package/package.json

@@ -1,89 +1,91 @@
 {
-    "name": "@llumiverse/core",
-    "version": "0.13.0",
-    "type": "module",
-    "description": "Provide an universal API to LLMs. Support for existing LLMs can be added by writing a driver.",
-    "files": [
-        "lib",
-        "src"
-    ],
-    "keywords": [
-        "llm",
-        "ai",
-        "prompt",
-        "prompt engineering",
-        "ml",
-        "machine learning",
-        "embeddings",
-        "training",
-        "model",
-        "universal",
-        "api",
-        "chatgpt",
-        "openai",
-        "vertexai",
-        "bedrock",
-        "replicate",
-        "huggingface",
-        "togetherai"
-    ],
-    "types": "./lib/types/index.d.ts",
-    "typesVersions": {
-        "*": {
-            "async": [
-                "./lib/types/async.d.ts"
-            ],
-            "formatters": [
-                "./lib/types/formatters/index.d.ts"
-            ]
-        }
-    },
-    "exports": {
-        ".": {
-            "types": "./lib/types/index.d.ts",
-            "import": "./lib/esm/index.js",
-            "require": "./lib/cjs/index.js"
-        },
-        "./async": {
-            "types": "./lib/types/async.d.ts",
-            "import": "./lib/esm/async.js",
-            "require": "./lib/cjs/async.js"
-        },
-        "./formatters": {
-            "types": "./lib/types/formatters/index.d.ts",
-            "import": "./lib/esm/formatters/index.js",
-            "require": "./lib/cjs/formatters/index.js"
-        }
-    },
-    "scripts": {
-        "test": "vitest run",
-        "build": "tsmod build",
-        "clean": "rimraf ./lib tsconfig.tsbuildinfo"
-    },
-    "author": "Llumiverse",
-    "license": "Apache-2.0",
-    "homepage": "https://github.com/llumiverse/llumiverse",
-    "repository": {
-        "type": "git",
-        "url": "git+ssh://git@github.com/llumiverse/llumiverse.git"
-    },
-    "devDependencies": {
-        "@types/eventsource": "^1.1.15",
-        "@types/json-schema": "^7.0.15",
-        "api-fetch-client": "^0.8.6",
-        "rimraf": "^5.0.5",
-        "ts-dual-module": "^0.6.3",
-        "typescript": "^5.4.2",
-        "vitest": "^1.4.0"
+  "name": "@llumiverse/core",
+  "version": "0.15.0",
+  "type": "module",
+  "description": "Provide an universal API to LLMs. Support for existing LLMs can be added by writing a driver.",
+  "files": [
+    "lib",
+    "src"
+  ],
+  "keywords": [
+    "llm",
+    "ai",
+    "prompt",
+    "prompt engineering",
+    "ml",
+    "machine learning",
+    "embeddings",
+    "training",
+    "model",
+    "universal",
+    "api",
+    "chatgpt",
+    "openai",
+    "vertexai",
+    "bedrock",
+    "replicate",
+    "huggingface",
+    "togetherai"
+  ],
+  "types": "./lib/types/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "async": [
+        "./lib/types/async.d.ts"
+      ],
+      "formatters": [
+        "./lib/types/formatters/index.d.ts"
+      ]
+    }
+  },
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "import": "./lib/esm/index.js",
+      "require": "./lib/cjs/index.js"
     },
-    "dependencies": {
-        "json-schema": "^0.4.0"
+    "./async": {
+      "types": "./lib/types/async.d.ts",
+      "import": "./lib/esm/async.js",
+      "require": "./lib/cjs/async.js"
     },
-    "ts_dual_module": {
-        "outDir": "lib",
-        "exports": {
-            "async": "async.js",
-            "formatters": "formatters/index.js"
-        }
+    "./formatters": {
+      "types": "./lib/types/formatters/index.d.ts",
+      "import": "./lib/esm/formatters/index.js",
+      "require": "./lib/cjs/formatters/index.js"
+    }
+  },
+  "author": "Llumiverse",
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/llumiverse/llumiverse",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/llumiverse/llumiverse.git"
+  },
+  "devDependencies": {
+    "@types/eventsource": "^1.1.15",
+    "@types/json-schema": "^7.0.15",
+    "api-fetch-client": "^0.13.0",
+    "rimraf": "^5.0.5",
+    "ts-dual-module": "^0.6.3",
+    "typescript": "^5.4.2",
+    "vitest": "^1.4.0"
+  },
+  "dependencies": {
+    "@types/node": "^22.5.0",
+    "ajv": "^8.16.0",
+    "ajv-formats": "^3.0.1"
+  },
+  "ts_dual_module": {
+    "outDir": "lib",
+    "exports": {
+      "async": "async.js",
+      "formatters": "formatters/index.js"
     }
-}
+  },
+  "scripts": {
+    "test": "vitest run",
+    "build": "npx tsmod build",
+    "clean": "rimraf ./lib tsconfig.tsbuildinfo"
+  }
+}

package/src/CompletionStream.ts

@@ -1,17 +1,14 @@
 import { AbstractDriver } from "./Driver.js";
-import { ExecutionResponse, CompletionStream, DriverOptions, ExecutionOptions, PromptSegment } from "./types.js";
+import { CompletionStream, DriverOptions, ExecutionOptions, ExecutionResponse } from "./types.js";
 
 export class DefaultCompletionStream<PromptT = any> implements CompletionStream<PromptT> {
 
     chunks: string[];
-    prompt: PromptT;
     completion: ExecutionResponse<PromptT> | undefined;
 
     constructor(public driver: AbstractDriver<DriverOptions, PromptT>,
-        segments: PromptSegment[],
+        public prompt: PromptT,
         public options: ExecutionOptions) {
-        this.driver = driver;
-        this.prompt = this.driver.createPrompt(segments, options);
         this.chunks = [];
     }
 
@@ -38,6 +35,7 @@ export class DefaultCompletionStream<PromptT = any> implements CompletionStream<
         }
 
         const content = chunks.join('');
+
         const promptTokens = typeof this.prompt === 'string' ? this.prompt.length : JSON.stringify(this.prompt).length;
         const resultTokens = content.length; //TODO use chunks.length ?
 
@@ -59,14 +57,11 @@ export class DefaultCompletionStream<PromptT = any> implements CompletionStream<
 
 export class FallbackCompletionStream<PromptT = any> implements CompletionStream<PromptT> {
 
-    prompt: PromptT;
     completion: ExecutionResponse<PromptT> | undefined;
 
     constructor(public driver: AbstractDriver<DriverOptions, PromptT>,
-        segments: PromptSegment[],
+        public prompt: PromptT,
         public options: ExecutionOptions) {
-        this.driver = driver;
-        this.prompt = this.driver.createPrompt(segments, options);
     }
 
     async *[Symbol.asyncIterator]() {
@@ -82,4 +77,3 @@ export class FallbackCompletionStream<PromptT = any> implements CompletionStream
         this.completion = completion;
     }
 }
-

package/src/Driver.ts

@@ -5,7 +5,7 @@
  */
 
 import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
-import { formatLlama2Prompt, formatTextPrompt } from "./formatters/index.js";
+import { formatLlama2Prompt, formatLlama3Prompt, formatTextPrompt } from "./formatters/index.js";
 import {
     AIModel,
     Completion,
@@ -62,14 +62,14 @@ function applyExecutionDefaults(options: ExecutionOptions): ExecutionOptions {
 export interface Driver<PromptT = unknown> {
 
     /**
-     * 
-     * @param segments 
-     * @param completion 
+     *
+     * @param segments
+     * @param completion
      * @param model the model to train
      */
-    createTrainingPrompt(options: TrainingPromptOptions): string;
+    createTrainingPrompt(options: TrainingPromptOptions): Promise<string>;
 
-    createPrompt(segments: PromptSegment[], opts: PromptOptions): PromptT;
+    createPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT>;
 
     execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>>;
 
@@ -111,8 +111,8 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
         this.logger = createLogger(opts.logger);
     }
 
-    createTrainingPrompt(options: TrainingPromptOptions): string {
-        const prompt = this.createPrompt(options.segments, { resultSchema: options.schema, model: options.model })
+    async createTrainingPrompt(options: TrainingPromptOptions): Promise<string> {
+        const prompt = await this.createPrompt(options.segments, { result_schema: options.schema, model: options.model })
         return JSON.stringify({
             prompt,
             completion: typeof options.completion === 'string' ? options.completion : JSON.stringify(options.completion)
@@ -132,9 +132,9 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     }
 
     validateResult(result: Completion, options: ExecutionOptions) {
-        if (!result.error && options.resultSchema) {
+        if (!result.error && options.result_schema) {
             try {
-                result.result = validateResult(result.result, options.resultSchema);
+                result.result = validateResult(result.result, options.result_schema);
             } catch (error: any) {
                 this.logger?.error({ err: error, data: result.result }, `[${this.provider}] [${options.model}] ${error.code ? '[' + error.code + '] ' : ''}Result validation error: ${error.message}`);
                 result.error = {
@@ -148,14 +148,13 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
 
     async execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
         options = applyExecutionDefaults(options);
-        const prompt = this.createPrompt(segments, options);
+        const prompt = await this.createPrompt(segments, options);
         return this._execute(prompt, options);
     }
 
     async _execute(prompt: PromptT, options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
         this.logger.debug(
-            `[${this.provider}] Executing ${options.model} with prompt`, prompt,
-        );
+            `[${this.provider}] Executing prompt on ${options.model}`);
         try {
             const start = Date.now();
             const result = await this.requestCompletion(prompt, options);
@@ -171,35 +170,38 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     // by default no stream is supported. we block and we return all at once
     async stream(segments: PromptSegment[], options: ExecutionOptions): Promise<CompletionStream<PromptT>> {
         options = applyExecutionDefaults(options);
+        const prompt = await this.createPrompt(segments, options);
         const canStream = await this.canStream(options);
         if (canStream) {
-            return new DefaultCompletionStream(this, segments, options);
+            return new DefaultCompletionStream(this, prompt, options);
         } else {
-            return new FallbackCompletionStream(this, segments, options);
+            return new FallbackCompletionStream(this, prompt, options);
         }
     }
 
     /**
      * Override this method to provide a custom prompt formatter
-     * @param segments 
-     * @param options 
-     * @returns 
+     * @param segments
+     * @param options
+     * @returns
      */
-    protected formatPrompt(segments: PromptSegment[], opts: PromptOptions): PromptT {
-        if (/\bllama2?\b/i.test(opts.model)) {
-            return formatLlama2Prompt(segments, opts.resultSchema) as PromptT;
+    protected async formatPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT> {
+        if (/\bllama.?2\b/i.test(opts.model)) {
+            return formatLlama2Prompt(segments, opts.result_schema) as PromptT;
+        } else if (/\bllama.?3\b/i.test(opts.model)) {
+            return formatLlama3Prompt(segments, opts.result_schema) as PromptT;
         } else {
-            return formatTextPrompt(segments, opts.resultSchema) as PromptT;
+            return formatTextPrompt(segments, opts.result_schema) as PromptT;
         }
     }
 
-    public createPrompt(segments: PromptSegment[], opts: PromptOptions): PromptT {
-        return opts.format ? opts.format(segments, opts.resultSchema) : this.formatPrompt(segments, opts);
+    public async createPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT> {
+        return await (opts.format ? opts.format(segments, opts.result_schema) : this.formatPrompt(segments, opts));
     }
 
     /**
      * Must be overrided if the implementation cannot stream.
-     * Some implementation may be able to stream for certain models but not for others. 
+     * Some implementation may be able to stream for certain models but not for others.
      * You must overwrite and return false if the current model doesn't support streaming.
      * The default implementation returns true, so it is assumed that the streaming can be done.
      * If this method returns false then the streaming execution will fallback on a blocking execution streaming the entire response as a single event.
@@ -213,7 +215,7 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     /**
      * Get a list of models that can be trained.
      * The default is to return an empty array
-     * @returns 
+     * @returns
      */
     async listTrainableModels(): Promise<AIModel[]> {
         return [];
@@ -233,4 +235,3 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     abstract generateEmbeddings(options: EmbeddingsOptions): Promise<EmbeddingsResult>;
 
 }
-

package/src/async.ts

@@ -104,8 +104,12 @@ export class EventStream<T, ReturnT = any> implements AsyncIterable<T>{
  **/
 export async function* transformAsyncIterator<T, V>(
     originalGenerator: AsyncIterable<T>,
-    transform: (value: T) => V | Promise<V>
+    transform: (value: T) => V | Promise<V>,
+    initCallback?: () => V | Promise<V>
 ): AsyncIterable<V> {
+    if (initCallback) {
+        yield initCallback();
+    }
     for await (const value of originalGenerator) {
         yield transform(value);
     }

package/src/formatters/claude.ts

@@ -1,14 +1,21 @@
 import { JSONSchema4 } from "json-schema";
 import { PromptRole, PromptSegment } from "../index.js";
+import { readStreamAsBase64 } from "../stream.js";
 import { getJSONSafetyNotice } from "./commons.js";
 
 export interface ClaudeMessage {
     role: 'user' | 'assistant',
-    content: {
-        type: "image" | "text",
-        source?: string, // only set for images
-        text?: string // only set for text messages
-    }[]
+    content: ClaudeMessagePart[]
+}
+
+interface ClaudeMessagePart {
+    type: "image" | "text",
+    source?: {
+        type: "base64",
+        media_type: string,
+        data: string,
+    }, // only set for images
+    text?: string // only set for text messages
 }
 
 export interface ClaudeMessagesPrompt {
@@ -20,19 +27,50 @@ export interface ClaudeMessagesPrompt {
  * A formatter user by Bedrock to format prompts for claude related models
  */
 
-export function formatClaudePrompt(segments: PromptSegment[], schema?: JSONSchema4): ClaudeMessagesPrompt {
+export async function formatClaudePrompt(segments: PromptSegment[], schema?: JSONSchema4): Promise<ClaudeMessagesPrompt> {
     const system: string[] = [];
     const safety: string[] = [];
     const messages: ClaudeMessage[] = [];
 
-    for (const msg of segments) {
-        if (msg.role === PromptRole.system) {
-            system.push(msg.content);
-        } else if (msg.role === PromptRole.safety) {
-            safety.push(msg.content);
+    //TODO type: 'image' -> detect from f.mime_type
+    for (const segment of segments) {
+
+        const parts: ClaudeMessagePart[] = [];
+        if (segment.files) for (const f of segment.files) {
+            const source = await f.getStream();
+            const data = await readStreamAsBase64(source);
+            parts.push({
+                type: 'image',
+                source: {
+                    type: "base64",
+                    media_type: f.mime_type || 'image/png',
+                    data
+                }
+            })
+        }
+
+        if (segment.content) {
+            parts.push({
+                type: "text",
+                text: segment.content
+            })
+        }
+
+        if (segment.role === PromptRole.system) {
+            system.push(segment.content);
+        } else if (segment.role === PromptRole.safety) {
+            safety.push(segment.content);
+        } else if (messages.length > 0 && messages[messages.length - 1].role === segment.role) {
+            //concatenate messages of the same role (Claude requires alternative user and assistant roles)
+            messages[messages.length - 1].content.push(...parts);
         } else {
-            messages.push({ content: [{ type: "text", text: msg.content }], role: msg.role });
+            messages.push({
+                role: segment.role,
+                content: parts
+            });
         }
+
+
     }
 
     if (schema) {
@@ -57,24 +95,24 @@ export function formatClaudePrompt(segments: PromptSegment[], schema?: JSONSchem
     /*if (schema) {
         messages.push({
             role: "user",
-            content: [{                
+            content: [{
                 type: "text",
                 text: getJSONSafetyNotice(schema)
             }]
         });
     }*/
 
-     /*start Claude's message to amke sure it answers properly in JSON
-    if enabled, this requires to add the { to Claude's response*/
-    if (schema) { 
+    /*start Claude's message to amke sure it answers properly in JSON
+   if enabled, this requires to add the { to Claude's response*/
+    if (schema) {
         messages.push({
             role: "assistant",
-            content: [{ 
-                type: "text", 
+            content: [{
+                type: "text",
                 text: "{"
-        }]});
+            }]
+        });
     }
-
     // put system mesages first and safety last
     return {
         system: systemMessage,

package/src/formatters/commons.ts

@@ -1,5 +1,5 @@
 import { JSONSchema4 } from "json-schema";
 
 export function getJSONSafetyNotice(schema: JSONSchema4) {
-    return "The answer must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema);
+    return "The answer must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema, undefined, 2);
 }

package/src/formatters/index.ts

@@ -3,8 +3,10 @@ import { PromptSegment } from "../types.js";
 
 export type PromptFormatter<T = any> = (messages: PromptSegment[], schema?: JSONSchema4) => T;
 
-export * from "./commons.js"
+export * from "./claude.js";
+export * from "./commons.js";
 export * from "./generic.js";
 export * from "./llama2.js";
-export * from "./claude.js";
-export * from "./openai.js";
+export * from "./llama3.js";
+export * from "./openai.js";
+

package/src/formatters/llama3.ts

@@ -0,0 +1,55 @@
+import { JSONSchema4 } from "json-schema";
+import { PromptRole, PromptSegment } from "../index.js";
+
+/**
+ * A formatter user by Bedrock to format prompts for claude related models
+ */
+
+export async function formatLlama3Prompt(segments: PromptSegment[], schema?: JSONSchema4): Promise<string> {
+    
+    let messages: string[] = []
+    segments.filter(s => s.role !== PromptRole.safety ).forEach(s => {
+        messages.push(formatLlama3Message(s.role, s.content))
+    })
+
+    if (schema) {
+        messages.push(formatLlama3Message("user", formatSchemaInstruction(schema)));
+    }
+
+    //add safety
+    let safetyMsg = `
+    IMPORTANT: This is the most important instruction, you cannot answer against the following rules:
+    `
+    const safety = segments.filter(s => s.role === PromptRole.safety);
+    safety.forEach(s => {
+        messages.push(formatLlama3Message("system", safetyMsg + s.content))
+    })
+
+    let prompt = "<|begin_of_text|>"
+    prompt += messages.join("\n\n")
+
+    return prompt
+
+}
+
+
+function formatLlama3Message(role: string, content: string) {
+    
+    let message = `<|start_header_id|>${role}<|end_header_id|>\n`
+    message += content        
+    message += `\n<|eot_id|>`
+
+    return message
+
+}
+
+function formatSchemaInstruction(schema: object) {
+
+    return `You must answer using the following JSONSchema.
+    Do not write anything other than a JSON object corresponding to the schema:
+    <schema>
+    ${JSON.stringify(schema, undefined, 2)}
+    </schema>
+    `
+    
+}