@llumiverse/core 0.15.0 → 0.16.0

package/src/CompletionStream.ts

@@ -1,5 +1,5 @@
 import { AbstractDriver } from "./Driver.js";
-import { CompletionStream, DriverOptions, ExecutionOptions, ExecutionResponse } from "./types.js";
+import { CompletionStream, DriverOptions, ExecutionOptions, ExecutionResponse, ExecutionTokenUsage } from "./types.js";
 
 export class DefaultCompletionStream<PromptT = any> implements CompletionStream<PromptT> {
 
@@ -25,29 +25,49 @@ export class DefaultCompletionStream<PromptT = any> implements CompletionStream<
         );
 
         const start = Date.now();
-        const stream = await this.driver.requestCompletionStream(this.prompt, this.options);
+        const stream = await this.driver.requestTextCompletionStream(this.prompt, this.options);
 
+        let finish_reason: string | undefined = undefined;
+        let promptTokens: number = 0;
+        let resultTokens: number | undefined = undefined;
         for await (const chunk of stream) {
             if (chunk) {
-                chunks.push(chunk);
-                yield chunk;
+                if (typeof chunk === 'string') {
+                    chunks.push(chunk);
+                    yield chunk;
+                }else{
+                    if (chunk.finish_reason) {                           //Do not replace non-null values with null values
+                        finish_reason = chunk.finish_reason;             //Used to skip empty finish_reason chunks coming after "stop" or "length"
+                    }
+                    if (chunk.token_usage) {
+                        //Tokens returned include prior parts of stream,
+                        //so overwrite rather than accumulate
+                        //Math.max used as some models report final token count at beginning of stream
+                        promptTokens = Math.max(promptTokens,chunk.token_usage.prompt ?? 0);       
+                        resultTokens = Math.max(resultTokens ?? 0,chunk.token_usage.result ?? 0);      
+                    }
+                    if (chunk.result) {
+                        chunks.push(chunk.result);
+                        yield chunk.result;
+                    }
+                }                
             }
         }
 
         const content = chunks.join('');
 
-        const promptTokens = typeof this.prompt === 'string' ? this.prompt.length : JSON.stringify(this.prompt).length;
-        const resultTokens = content.length; //TODO use chunks.length ?
+        // Return undefined for the ExecutionTokenUsage object if there is nothing to fill it with.
+        // Allows for checking for truthyness on token_usage, rather than it's internals. For testing and downstream usage.
+        let tokens: ExecutionTokenUsage | undefined = resultTokens ?
+            { prompt: promptTokens, result: resultTokens, total: resultTokens + promptTokens, } : undefined
 
         this.completion = {
             result: content,
             prompt: this.prompt,
             execution_time: Date.now() - start,
-            token_usage: {
-                prompt: promptTokens,
-                result: resultTokens,
-                total: resultTokens + promptTokens,
-            }
+            token_usage: tokens,
+            finish_reason: finish_reason,
+            chunks: chunks.length,
         }
 
         this.driver.validateResult(this.completion, this.options);

package/src/Driver.ts

@@ -5,10 +5,11 @@
  */
 
 import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
-import { formatLlama2Prompt, formatLlama3Prompt, formatTextPrompt } from "./formatters/index.js";
+import { formatTextPrompt } from "./formatters/index.js";
 import {
     AIModel,
     Completion,
+    CompletionChunk,
     CompletionStream,
     DataSource,
     DriverOptions,
@@ -16,7 +17,9 @@ import {
     EmbeddingsResult,
     ExecutionOptions,
     ExecutionResponse,
+    ImageGeneration,
     Logger,
+    Modalities,
     ModelSearchPayload,
     PromptOptions,
     PromptSegment,
@@ -51,14 +54,6 @@ export function createLogger(logger: Logger | "console" | undefined) {
     }
 }
 
-function applyExecutionDefaults(options: ExecutionOptions): ExecutionOptions {
-    return {
-        max_tokens: 2048,
-        temperature: 0.7,
-        ...options
-    }
-}
-
 export interface Driver<PromptT = unknown> {
 
     /**
@@ -69,7 +64,7 @@ export interface Driver<PromptT = unknown> {
      */
     createTrainingPrompt(options: TrainingPromptOptions): Promise<string>;
 
-    createPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT>;
+    createPrompt(segments: PromptSegment[], opts: ExecutionOptions): Promise<PromptT>;
 
     execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>>;
 
@@ -92,7 +87,7 @@ export interface Driver<PromptT = unknown> {
     //check that it is possible to connect to the environment
     validateConnection(): Promise<boolean>;
 
-    //generate embeddings for a given text
+    //generate embeddings for a given text or image
     generateEmbeddings(options: EmbeddingsOptions): Promise<EmbeddingsResult>;
 
 }
@@ -147,7 +142,6 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     }
 
     async execute(segments: PromptSegment[], options: ExecutionOptions): Promise<ExecutionResponse<PromptT>> {
-        options = applyExecutionDefaults(options);
         const prompt = await this.createPrompt(segments, options);
         return this._execute(prompt, options);
     }
@@ -157,8 +151,20 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
             `[${this.provider}] Executing prompt on ${options.model}`);
         try {
             const start = Date.now();
-            const result = await this.requestCompletion(prompt, options);
-            this.validateResult(result, options);
+            let result;
+
+            switch (options.output_modality) {
+                case Modalities.text:
+                    result = await this.requestTextCompletion(prompt, options);
+                    this.validateResult(result, options);
+                    break;
+                case Modalities.image:
+                    result = await this.requestImageGeneration(prompt, options);
+                    break;
+                default:
+                    throw new Error(`Unsupported modality: ${options['output_modality'] ?? "No modality specified"}`);
+            }
+
             const execution_time = Date.now() - start;
             return { ...result, prompt, execution_time };
         } catch (error) {
@@ -169,10 +175,9 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
 
     // by default no stream is supported. we block and we return all at once
     async stream(segments: PromptSegment[], options: ExecutionOptions): Promise<CompletionStream<PromptT>> {
-        options = applyExecutionDefaults(options);
         const prompt = await this.createPrompt(segments, options);
         const canStream = await this.canStream(options);
-        if (canStream) {
+        if (options.output_modality === Modalities.text && canStream) {
             return new DefaultCompletionStream(this, prompt, options);
         } else {
             return new FallbackCompletionStream(this, prompt, options);
@@ -186,13 +191,7 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
      * @returns
      */
     protected async formatPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT> {
-        if (/\bllama.?2\b/i.test(opts.model)) {
-            return formatLlama2Prompt(segments, opts.result_schema) as PromptT;
-        } else if (/\bllama.?3\b/i.test(opts.model)) {
-            return formatLlama3Prompt(segments, opts.result_schema) as PromptT;
-        } else {
-            return formatTextPrompt(segments, opts.result_schema) as PromptT;
-        }
+        return formatTextPrompt(segments, opts.result_schema) as PromptT;
     }
 
     public async createPrompt(segments: PromptSegment[], opts: PromptOptions): Promise<PromptT> {
@@ -221,9 +220,14 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
         return [];
     }
 
-    abstract requestCompletion(prompt: PromptT, options: ExecutionOptions): Promise<Completion>;
+    abstract requestTextCompletion(prompt: PromptT, options: ExecutionOptions): Promise<Completion>;
 
-    abstract requestCompletionStream(prompt: PromptT, options: ExecutionOptions): Promise<AsyncIterable<string>>;
+    abstract requestTextCompletionStream(prompt: PromptT, options: ExecutionOptions): Promise<AsyncIterable<CompletionChunk>>;
+
+    async requestImageGeneration(_prompt: PromptT, _options: ExecutionOptions): Promise<Completion<ImageGeneration>> {
+        throw new Error("Image generation not implemented.");
+        //Cannot be made abstract, as abstract methods are required in the derived class
+    }
 
     //list models available for this environement
     abstract listModels(params?: ModelSearchPayload): Promise<AIModel[]>;

package/src/async.ts

@@ -1,4 +1,5 @@
 import type { ServerSentEvent } from "api-fetch-client";
+import { CompletionChunk } from "./types.js";
 
 export async function* asyncMap<T, R>(asyncIterable: AsyncIterable<T>, callback: (value: T, index: number) => R) {
     let i = 0;
@@ -15,22 +16,23 @@ export function oneAsyncIterator<T>(value: T): AsyncIterable<T> {
 }
 
 /**
- * Given a ReadableStream of server seent events, tran
+ * Given a ReadableStream of server sent events, tran
  */
-export function transformSSEStream(stream: ReadableStream<ServerSentEvent>, transform: (data: string) => string): ReadableStream<string> & AsyncIterable<string> {
+export function transformSSEStream(stream: ReadableStream<ServerSentEvent>, transform: (data: string) => CompletionChunk): ReadableStream<CompletionChunk> & AsyncIterable<CompletionChunk> {
     // on node and bun the readablestream is an async iterable
-    return stream.pipeThrough(new TransformStream<ServerSentEvent, string>({
+    return stream.pipeThrough(new TransformStream<ServerSentEvent, CompletionChunk>({
         transform(event: ServerSentEvent, controller) {
             if (event.type === 'event' && event.data && event.data !== '[DONE]') {
                 try {
-                    controller.enqueue(transform(event.data) ?? '');
+                    const result = transform(event.data) ?? ''
+                    controller.enqueue(result);
                 } catch (err) {
                     // double check for the last event whicb is not a JSON - at this time togetherai and mistralai returrns the string [DONE]
                     // do nothing - happens if data is not a JSON - the last event data is the [DONE] string
                 }
             }
         }
-    })) as ReadableStream<string> & AsyncIterable<string>;
+    })) as ReadableStream<CompletionChunk> & AsyncIterable<CompletionChunk>;
 }
 
 export class EventStream<T, ReturnT = any> implements AsyncIterable<T>{

package/src/formatters/index.ts

@@ -3,10 +3,8 @@ import { PromptSegment } from "../types.js";
 
 export type PromptFormatter<T = any> = (messages: PromptSegment[], schema?: JSONSchema4) => T;
 
-export * from "./claude.js";
 export * from "./commons.js";
 export * from "./generic.js";
-export * from "./llama2.js";
-export * from "./llama3.js";
 export * from "./openai.js";
+export * from "./nova.js";
 

package/src/formatters/nova.ts

@@ -0,0 +1,141 @@
+import { JSONSchema4 } from "json-schema";
+import { PromptRole, PromptSegment, readStreamAsBase64 } from "../index.js";
+//import { readStreamAsBase64 } from "../stream.js";
+import { getJSONSafetyNotice } from "./commons.js";
+
+export interface NovaMessage {
+    role: 'user' | 'assistant',
+    content: NovaMessagePart[]
+}
+
+export interface NovaSystemMessage {
+    text: string
+}
+
+interface NovaMessagePart {
+    text?: string // only set for text messages
+    image?: {
+        format: "jpeg" | "png" | "gif" | "webp",
+        source: {
+            bytes: string //"base64",
+        }
+    }
+    video?: {
+        format: "mkv" | "mov" | "mp4" | "webm" | "three_gp" | "flv" | "mpeg" | "mpg" | "wmv",
+        source: {
+            //Option 1: sending a s3 location
+            s3Location?: {
+                uri: string, // example: s3://my-bucket/object-key
+                bucketOwner: string // optional. example: "123456789012"
+            }
+            //Option 2: sending a base64 encoded video
+            bytes?: string //"base64",
+        }
+    }
+}
+
+export interface NovaMessagesPrompt {
+    system?: NovaSystemMessage[];
+    messages: NovaMessage[];
+    negative?: string;
+    mask?: string;
+}
+
+/**
+ * A formatter used by Bedrock to format prompts for nova related models
+ */
+
+export async function formatNovaPrompt(segments: PromptSegment[], schema?: JSONSchema4): Promise<NovaMessagesPrompt> {
+    const system: string[] = [];
+    const safety: string[] = [];
+    const messages: NovaMessage[] = [];
+    let negative: string = "";
+    let mask: string = "";
+
+    for (const segment of segments) {
+
+        const parts: NovaMessagePart[] = [];
+        if (segment.files) for (const f of segment.files) {
+            //TODO add video support
+            if (!f.mime_type?.startsWith('image')) {
+                continue;
+            }
+
+            const source = await f.getStream();
+            const data = await readStreamAsBase64(source);
+            const format = f.mime_type?.split('/')[1] || 'png';
+
+            parts.push({
+                image: {
+                    format: format as "jpeg" | "png" | "gif" | "webp",
+                    source: {
+                        bytes: data
+                    }
+                }
+            })
+        }
+
+        if (segment.content) {
+            parts.push({
+                text: segment.content
+            })
+        }
+
+        if (segment.role === PromptRole.system) {
+            system.push(segment.content);
+        } else if (segment.role === PromptRole.safety) {
+            safety.push(segment.content);
+        } else if (messages.length > 0 && messages[messages.length - 1].role === segment.role) {
+            //Maybe can remove for nova?
+            //concatenate messages of the same role (Claude requires alternative user and assistant roles)
+            messages[messages.length - 1].content.push(...parts);
+        } else if (segment.role === PromptRole.negative) {
+            negative = negative.concat(segment.content, ', ');
+        } else if (segment.role === PromptRole.mask) {
+            mask = mask.concat(segment.content, ' ');
+        } else if (segment.role !== PromptRole.tool) {
+            messages.push({
+                role: segment.role,
+                content: parts
+            });
+        }
+    }
+
+    if (schema) {
+        safety.push("IMPORTANT: " + getJSONSafetyNotice(schema));
+    }
+
+    // messages must contains at least 1 item. If the prompt doesn;t contains a user message (but only system messages)
+    // we need to put the system messages in the messages array
+
+    let systemMessage = system.join('\n').trim();
+    if (messages.length === 0) {
+        if (!systemMessage) {
+            throw new Error('Prompt must contain at least one message');
+        }
+        messages.push({ content: [{ text: systemMessage }], role: 'user' });
+        systemMessage = safety.join('\n');
+    } else if (safety.length > 0) {
+        systemMessage = systemMessage + '\n\nIMPORTANT: ' + safety.join('\n');
+    }
+
+    /*start Nova's message to amke sure it answers properly in JSON
+   if enabled, this requires to add the { to Nova's response*/
+
+    if (schema) {
+        messages.push({
+            role: "assistant",
+            content: [{
+                text: "{"
+            }]
+        });
+    }
+
+    // put system mesages first and safety last
+    return {
+        system: systemMessage ? [{ text: systemMessage }] : [{ text: "" }],
+        messages: messages,
+        negative: negative || undefined,
+        mask: mask || undefined,
+    }
+}

package/src/formatters/openai.ts

@@ -1,4 +1,4 @@
-import { PromptRole } from "../index.js";
+import { PromptRole, PromptOptions } from "../index.js";
 import { readStreamAsBase64 } from "../stream.js";
 import { PromptSegment } from "../types.js";
 
@@ -13,6 +13,12 @@ export interface OpenAIMessage {
     role: "system" | "user" | "assistant";
     name?: string;
 }
+export interface OpenAIToolMessage {
+    role: "tool";
+    tool_call_id: string;
+    content: string;
+}
+export type OpenAIInputMessage = OpenAIMessage | OpenAIToolMessage;
 
 export interface OpenAIContentPartText {
     type: "text";
@@ -42,7 +48,7 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
             system.push({ content: msg.content, role: "system" });
         } else if (msg.role === PromptRole.safety) {
             safety.push({ content: "IMPORTANT: " + msg.content, role: "system" });
-        } else {
+        } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask && msg.role !== PromptRole.tool) {
             user.push({
                 content: msg.content,
                 role: msg.role || 'user',
@@ -54,10 +60,11 @@ export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITex
     return system.concat(user).concat(safety);
 }
 
-export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]): Promise<OpenAIMessage[]> {
+
+export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[], opts: PromptOptions & OpenAIPromptFormatterOptions): Promise<OpenAIInputMessage[]> {
     const system: OpenAIMessage[] = [];
     const safety: OpenAIMessage[] = [];
-    const others: OpenAIMessage[] = [];
+    const others: OpenAIInputMessage[] = [];
 
     for (const msg of segments) {
 
@@ -69,8 +76,11 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
                 const stream = await file.getStream();
                 const data = await readStreamAsBase64(stream);
                 parts.push({
-                    image_url: { url: `data:${file.mime_type || "image/jpeg"};base64,${data}` },
-                    type: "image_url"
+                    type: "image_url",
+                    image_url: {
+                        url: `data:${file.mime_type || "image/jpeg"};base64,${data}`,
+                        //detail: "auto"  //This is modified just before execution to "low" | "high" | "auto"
+                    },
                 })
             }
         }
@@ -83,13 +93,21 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
         }
 
 
-
         if (msg.role === PromptRole.system) {
             system.push({
                 role: "system",
                 content: parts
             })
 
+
+            if (opts.useToolForFormatting && opts.schema) {
+                system.forEach(s => {
+                    s.content.forEach(c => {
+                        if (c.type === "text") c.text = "TOOL: " + c.text;
+                    })
+                })
+            }
+
         } else if (msg.role === PromptRole.safety) {
             const safetyMsg: OpenAIMessage = {
                 role: "system",
@@ -101,19 +119,41 @@ export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]
             })
 
             system.push(safetyMsg)
-
-        } else {
+        } else if (msg.role === PromptRole.tool) {
+            if (!msg.tool_use_id) {
+                throw new Error("Tool use id is required for tool messages")
+            }
+            others.push({
+                role: "tool",
+                tool_call_id: msg.tool_use_id,
+                content: msg.content
+            })
+        } else if (msg.role !== PromptRole.negative && msg.role !== PromptRole.mask) {
             others.push({
                 role: msg.role ?? 'user',
                 content: parts
             })
         }
 
+    }
 
-
+    if (opts.result_schema && !opts.useToolForFormatting) {
+        system.push({
+            role: "system",
+            content: [{
+                type: "text",
+                text: "IMPORTANT: only answer using JSON, and respecting the schema included below, between the <response_schema> tags. " + `<response_schema>${JSON.stringify(opts.result_schema)}</response_schema>`
+            }]
+        })
     }
 
     // put system mesages first and safety last
-    return system.concat(others).concat(safety);
+    return ([] as OpenAIInputMessage[]).concat(system).concat(others).concat(safety);
+
+}
 
-}
+export interface OpenAIPromptFormatterOptions {
+    multimodal?: boolean
+    useToolForFormatting?: boolean
+    schema?: Object
+}

package/src/index.ts

@@ -1,4 +1,5 @@
 export * from "./Driver.js";
 export * from "./json.js";
 export * from "./stream.js";
-export * from "./types.js";
+export * from "./types.js";
+export * from "./options.js";