@llumiverse/core 0.12.3 → 0.14.0

package/src/formatters/openai.ts

@@ -1,19 +1,41 @@
 import { PromptRole } from "../index.js";
+import { readStreamAsBase64 } from "../stream.js";
 import { PromptSegment } from "../types.js";
 
+
 export interface OpenAITextMessage {
     content: string;
     role: "system" | "user" | "assistant";
 }
+
+export interface OpenAIMessage {
+    content: (OpenAIContentPartText | OpenAIContentPartImage)[]
+    role: "system" | "user" | "assistant";
+    name?: string;
+}
+
+export interface OpenAIContentPartText {
+    type: "text";
+    text: string
+}
+
+export interface OpenAIContentPartImage {
+    type: "image_url";
+    image_url: {
+        detail?: 'auto' | 'low' | 'high'
+        url: string
+    }
+}
+
 /**
  * OpenAI text only prompts
- * @param segments 
- * @returns 
+ * @param segments
+ * @returns
  */
-export function formatOpenAILikePrompt(segments: PromptSegment[]) {
+export function formatOpenAILikeTextPrompt(segments: PromptSegment[]): OpenAITextMessage[] {
     const system: OpenAITextMessage[] = [];
-    const others: OpenAITextMessage[] = [];
     const safety: OpenAITextMessage[] = [];
+    const user: OpenAITextMessage[] = [];
 
     for (const msg of segments) {
         if (msg.role === PromptRole.system) {
@@ -21,10 +43,75 @@ export function formatOpenAILikePrompt(segments: PromptSegment[]) {
         } else if (msg.role === PromptRole.safety) {
             safety.push({ content: "IMPORTANT: " + msg.content, role: "system" });
         } else {
-            others.push({ content: msg.content, role: "user" });
+            user.push({
+                content: msg.content,
+                role: msg.role || 'user',
+            })
         }
     }
 
     // put system mesages first and safety last
-    return system.concat(others).concat(safety);
+    return system.concat(user).concat(safety);
 }
+
+export async function formatOpenAILikeMultimodalPrompt(segments: PromptSegment[]): Promise<OpenAIMessage[]> {
+    const system: OpenAIMessage[] = [];
+    const safety: OpenAIMessage[] = [];
+    const others: OpenAIMessage[] = [];
+
+    for (const msg of segments) {
+
+        const parts: (OpenAIContentPartImage | OpenAIContentPartText)[] = [];
+
+        //generate the parts based on promptsegment
+        if (msg.files) {
+            for (const file of msg.files) {
+                const stream = await file.getStream();
+                const data = await readStreamAsBase64(stream);
+                parts.push({
+                    image_url: { url: `data:${file.mime_type || "image/jpeg"};base64,${data}` },
+                    type: "image_url"
+                })
+            }
+        } else {
+            parts.push({
+                text: msg.content,
+                type: "text"
+            })
+        }
+
+
+
+        if (msg.role === PromptRole.system) {
+            system.push({
+                role: "system",
+                content: parts
+            })
+
+        } else if (msg.role === PromptRole.safety) {
+            const safetyMsg: OpenAIMessage = {
+                role: "system",
+                content: parts
+            }
+
+            safetyMsg.content.forEach(c => {
+                if (c.type === "text") c.text = "DO NOT IGNORE - IMPORTANT: " + c.text;
+            })
+
+            system.push(safetyMsg)
+
+        } else {
+            others.push({
+                role: msg.role ?? 'user',
+                content: parts
+            })
+        }
+
+
+
+    }
+
+    // put system mesages first and safety last
+    return system.concat(others).concat(safety);
+
+}

package/src/index.ts

@@ -1,3 +1,4 @@
 export * from "./Driver.js";
 export * from "./json.js";
-export * from "./types.js";
+export * from "./stream.js";
+export * from "./types.js";

package/src/resolver.ts

@@ -0,0 +1,39 @@
+/**
+ * Get the property named by "name" of the given object
+ * If an array is idnexed using a string key then a map is done and an array with the content of the properties with that name are returned
+ * Ex: docs.text => will return an array of text properties of the docs array
+ * @param object the obejct
+ * @param name the name of the property.
+ * @returns the property value
+ */
+function _prop(object: any, name: string) {
+    if (object === undefined) {
+        return undefined;
+    }
+    if (Array.isArray(object)) {
+        const index = +name;
+        if (isNaN(index)) {
+            // map array to property
+            return object.map(item => item[name]);
+        } else {
+            return object[index];
+        }
+    } else {
+        return object[name];
+    }
+
+}
+
+export function resolveField(object: any, path: string[]) {
+    let p = object as any;
+    if (!p) return p;
+    if (!path.length) return p;
+    const last = path.length - 1;
+    for (let i = 0; i < last; i++) {
+        p = _prop(p, path[i])
+        if (!p) {
+            return undefined;
+        }
+    }
+    return _prop(p, path[last]);
+}

package/src/stream.ts

@@ -0,0 +1,8 @@
+
+export async function readStreamAsBase64(stream: ReadableStream) {
+    const out: Buffer[] = [];
+    for await (const chunk of stream as any) {
+        out.push(Buffer.from(chunk));
+    }
+    return Buffer.concat(out).toString('base64');
+}

package/src/types.ts

@@ -1,6 +1,6 @@
 import { JSONSchema4 } from "json-schema";
-import { JSONObject } from "./json.js";
 import { PromptFormatter } from "./formatters/index.js";
+import { JSONObject } from "./json.js";
 
 export interface EmbeddingsOptions {
     /**
@@ -12,7 +12,7 @@ export interface EmbeddingsOptions {
      */
     model?: string;
     /**
-     * Additional options for the embeddings generation. Optional. 
+     * Additional options for the embeddings generation. Optional.
      * The supported properties depends on the target implementation.
      */
     [key: string]: any;
@@ -44,7 +44,7 @@ export interface ResultValidationError {
 }
 
 export interface Completion<ResultT = any> {
-    // the driver impl must return the result and optionally the token_usage. the execution time is computed by the extended abstract driver 
+    // the driver impl must return the result and optionally the token_usage. the execution time is computed by the extended abstract driver
     result: ResultT;
     token_usage?: ExecutionTokenUsage;
 
@@ -55,7 +55,7 @@ export interface Completion<ResultT = any> {
 
     /**
      * Set only if a result validation error occured, otherwise if the result is valid the error field is undefined
-     * This can only be set if the resultSchema is set and the reuslt could not be parsed as a json or if the result does not match the schema
+     * This can only be set if the result_schema is set and the reuslt could not be parsed as a json or if the result does not match the schema
      */
     error?: ResultValidationError;
 
@@ -93,18 +93,18 @@ export interface DriverOptions {
 export interface PromptOptions {
     model: string;
     /**
-     * A custom formatter to use for format the final model prompt from the input prompt segments. 
+     * A custom formatter to use for format the final model prompt from the input prompt segments.
      * If no one is specified the driver will choose a formatter compatible with the target model
      */
     format?: PromptFormatter;
-    resultSchema?: JSONSchema4;
+    result_schema?: JSONSchema4;
 }
 export interface ExecutionOptions extends PromptOptions {
     temperature?: number;
     max_tokens?: number;
     stop_sequence?: string | string[];
 
-    /** 
+    /**
      * restricts the selection of tokens to the “k” most likely options, based on their probabilities
      * Lower values make the model more deterministic, more focused. Examples:
      * - 10 - result will be highly controlled anc contextually relevant
@@ -139,7 +139,7 @@ export interface ExecutionOptions extends PromptOptions {
 
     /**
      * If set to true the original response from the target LLM will be included in the response under the original_response field.
-     * This is useful for debugging and for some advanced use cases. 
+     * This is useful for debugging and for some advanced use cases.
      * It is ignored on streaming requests
      */
     include_original_response?: boolean;
@@ -156,6 +156,7 @@ export enum PromptRole {
 export interface PromptSegment {
     role: PromptRole;
     content: string;
+    files?: DataSource[]
 }
 
 export interface ExecutionTokenUsage {
@@ -177,8 +178,9 @@ export interface AIModel<ProviderKeys = string> {
     tags?: string[]; //tags for searching
     owner?: string; //owner of the model
     status?: AIModelStatus; //status of the model
-    canStream?: boolean; //if the model's reponse can be streamed
-    isCustom?: boolean; //if the model is a custom model (a trained model)
+    can_stream?: boolean; //if the model's reponse can be streamed
+    is_custom?: boolean; //if the model is a custom model (a trained model)
+    is_multimodal?: boolean //if the model support files and images
 }
 
 export enum AIModelStatus {
@@ -231,20 +233,21 @@ export enum ModelType {
 
 export interface DataSource {
     name: string;
-    getStream(): ReadableStream<Uint8Array | string>;
+    mime_type?: string;
+    getStream(): Promise<ReadableStream<Uint8Array | string>>;
     getURL(): Promise<string>;
 }
 
 export interface TrainingOptions {
     name: string; // the new model name
-    model: string; // the model to train 
+    model: string; // the model to train
     params?: JSONObject; // the training parameters
 }
 
 export interface TrainingPromptOptions {
     segments: PromptSegment[];
     completion: string | JSONObject;
-    model: string; // the model to train 
+    model: string; // the model to train
     schema?: JSONSchema4; // the resuilt schema f any
 }
 

package/src/validation.ts

@@ -1,7 +1,23 @@
-import { JSONSchema4, validate } from "json-schema";
+import { Ajv } from 'ajv';
+import addFormats from 'ajv-formats';
 import { extractAndParseJSON } from "./json.js";
+import { resolveField } from './resolver.js';
 import { ResultValidationError } from "./types.js";
 
+
+const ajv = new Ajv({
+    coerceTypes: 'array',
+    allowDate: true,
+    strict: false,
+    useDefaults: true,
+    removeAdditional: "failing"
+});
+
+//use ts ignore to avoid error with ESM and ajv-formats
+// @ts-ignore This expression is not callable
+addFormats(ajv)
+
+
 export class ValidationError extends Error implements ResultValidationError {
     constructor(
         public code: 'validation_error' | 'json_error',
@@ -12,8 +28,9 @@ export class ValidationError extends Error implements ResultValidationError {
     }
 }
 
-export function validateResult(data: any, schema: JSONSchema4) {
+export function validateResult(data: any, schema: Object) {
     let json;
+
     if (typeof data === "string") {
         try {
             json = extractAndParseJSON(data);
@@ -23,11 +40,35 @@ export function validateResult(data: any, schema: JSONSchema4) {
     } else {
         json = data;
     }
-    const validation = validate(json, schema);
-    if (!validation.valid) {
-        throw new ValidationError(
-            "validation_error",
-            validation.errors.map(e => e.message).join(",\n"))
+
+    const validate = ajv.compile(schema);
+    const valid = validate(json);
+
+    if (!valid && validate.errors) {
+        let errors = [];    
+
+        for (const e of validate.errors) {
+            const path = e.instancePath.split("/").slice(1);
+            const value = resolveField(json, path);
+            const schemaPath = e.schemaPath.split("/").slice(1);
+            const schemaFieldFormat = resolveField(schema, schemaPath);
+            const schemaField = resolveField(schema, schemaPath.slice(0, -3));
+
+            //ignore date if empty or null
+            if (!value 
+                && ["date", "date-time"].includes(schemaFieldFormat)
+                && !schemaField?.required?.includes(path[path.length - 1])) {
+                continue;
+            } else {
+                errors.push(e);
+            }
+        }
+
+        //console.log("Errors", errors)
+        if (errors.length > 0) {
+            const errorsMessage = errors.map(e => `${e.instancePath}: ${e.message}\n${JSON.stringify(e.params)}`).join(",\n\n");
+            throw new ValidationError("validation_error", errorsMessage)
+        }
     }
 
     return json;