@llumiverse/core 0.8.0

package/src/async.ts

@@ -0,0 +1,120 @@
+
+export async function* asyncMap<T, R>(asyncIterable: AsyncIterable<T>, callback: (value: T, index: number) => R) {
+    let i = 0;
+    for await (const val of asyncIterable)
+        yield callback(val, i++);
+}
+
+export function oneAsyncIterator<T>(value: T): AsyncIterable<T> {
+    return {
+        async *[Symbol.asyncIterator]() {
+            yield value
+        }
+    }
+}
+
+
+export class EventStream<T, ReturnT = any> implements AsyncIterable<T>{
+
+    private queue: T[] = [];
+    private pending?: {
+        resolve: (result: IteratorResult<T, ReturnT | undefined>) => void,
+        reject: (err: any) => void
+    };
+    private done = false;
+
+
+    push(event: T) {
+        if (this.done) {
+            throw new Error('Cannot push to a closed stream');
+        }
+        if (this.pending) {
+            this.pending.resolve({ value: event });
+            this.pending = undefined;
+        } else {
+            this.queue.push(event);
+        }
+    }
+
+    /**
+     * Close the stream. This means the stream cannot be feeded anymore.
+     * But the consumer can still consume the remaining events.
+     */
+    close(value?: ReturnT) {
+        this.done = true;
+        if (this.pending) {
+            this.pending.resolve({ done: true, value });
+            this.pending = undefined;
+        }
+    }
+
+    [Symbol.asyncIterator](): AsyncIterator<T, ReturnT | undefined> {
+        const self = this;
+        return {
+            next(): Promise<IteratorResult<T, ReturnT | undefined>> {
+                const next = self.queue.shift();
+                if (next !== undefined) {
+                    return Promise.resolve({ value: next });
+                } else if (self.done) {
+                    return Promise.resolve({ done: true, value: undefined as ReturnT });
+                } else {
+                    return new Promise<IteratorResult<T, ReturnT | undefined>>((resolve, reject) => {
+                        self.pending = { resolve, reject };
+                    });
+                }
+            },
+            async return(value?: ReturnT | Promise<ReturnT>): Promise<IteratorResult<T, ReturnT>> {
+                self.done = true;
+                self.queue = [];
+                if (value === undefined) {
+                    return { done: true, value: undefined as ReturnT };
+                }
+                const _value = await value;
+                return { done: true, value: _value };
+            }
+        }
+    }
+}
+
+
+
+/**
+ * Transform an async iterator by applying a function to each value.
+ * @param originalGenerator
+ * @param transform
+ **/
+export async function* transformAsyncIterator<T, V>(
+    originalGenerator: AsyncIterable<T>,
+    transform: (value: T) => V | Promise<V>
+): AsyncIterable<V> {
+    for await (const value of originalGenerator) {
+        yield transform(value);
+    }
+}
+
+//TODO move in a test file
+// const max = 10; let cnt = 0;
+// function feedStream(stream: EventStream<string>) {
+//     setTimeout(() => {
+//         cnt++;
+//         console.log('push: ', cnt, max);
+//         stream.push('event ' + cnt);
+//         if (cnt < max) {
+//             console.log('next: ', cnt, max);
+//             setTimeout(() => feedStream(stream), 1000);
+//         } else {
+//             console.log('end of stream');
+//             stream.close();
+//         }
+//     }, 1000);
+// }
+
+// const stream = new EventStream<string>();
+// feedStream(stream);
+
+// for await (const chunk of stream) {
+//     console.log('++++chunk:', chunk);
+// }
+
+
+

package/src/formatters.ts

@@ -0,0 +1,147 @@
+import { JSONSchema4 } from "json-schema";
+import OpenAI from "openai";
+import {
+    PromptFormats,
+    PromptRole,
+    PromptSegment,
+} from "./types.js";
+
+export function inferFormatterFromModelName(modelName: string): PromptFormats {
+    const name = modelName.toLowerCase();
+    if (name.includes("llama")) {
+        return PromptFormats.llama2;
+    } else if (name.includes("gpt")) {
+        return PromptFormats.openai;
+    } else if (name.includes("claude")) {
+        return PromptFormats.claude;
+    } else {
+        return PromptFormats.genericTextLLM;
+    }
+}
+
+export const PromptFormatters: Record<
+    PromptFormats,
+    (messages: PromptSegment[], schema?: JSONSchema4) => any
+> = {
+    openai: openAI,
+    llama2: llama2,
+    claude: claude,
+    genericTextLLM: genericColonSeparator,
+};
+
+function openAI(segments: PromptSegment[]) {
+    const system: OpenAI.Chat.ChatCompletionMessageParam[] = [];
+    const others: OpenAI.Chat.ChatCompletionMessageParam[] = [];
+    const safety: OpenAI.Chat.ChatCompletionMessageParam[] = [];
+
+    for (const msg of segments) {
+        if (msg.role === PromptRole.system) {
+            system.push({ content: msg.content, role: "system" });
+        } else if (msg.role === PromptRole.safety) {
+            safety.push({ content: msg.content, role: "system" });
+        } else {
+            others.push({ content: msg.content, role: "user" });
+        }
+    }
+
+    // put system mesages first and safety last
+    return system.concat(others).concat(safety);
+}
+
+function llama2(messages: PromptSegment[], schema?: JSONSchema4) {
+    const BOS = "<s>";
+    const EOS = "</s>";
+    const INST = "[INST]";
+    const INST_END = "[/INST]";
+    const SYS = "<<SYS>>\n";
+    const SYS_END = "\n<</SYS>>";
+
+    const promptMessages = [BOS];
+    const specialTokens = [BOS, EOS, INST, INST_END, SYS, SYS_END];
+
+    for (const m of messages) {
+        if (m.role === PromptRole.user) {
+            if (specialTokens.includes(m.content)) {
+                throw new Error(
+                    `Cannot use special token ${m.content.trim()} in user message`
+                );
+            }
+            promptMessages.push(`${INST} ${m.content.trim()} ${INST_END}`);
+        }
+        if (m.role === PromptRole.assistant) {
+            promptMessages.push(`${m.content.trim()}`);
+        }
+        if (m.role === PromptRole.system) {
+            promptMessages.push(`${SYS}${m.content.trim()}${SYS_END}`);
+        }
+    }
+
+    for (const m of messages ?? []) {
+        if (m.role === PromptRole.safety) {
+            promptMessages.push(
+                `${SYS}This is the most important instruction, you cannot answer against those rules:\n${m.content.trim()}${SYS_END}}`
+            );
+        }
+    }
+
+    if (schema) {
+        promptMessages.push(formatSchemaInstruction(schema));
+    }
+
+    promptMessages.push(EOS);
+
+    return promptMessages.join("\n\n");
+}
+
+function genericColonSeparator(
+    messages: PromptSegment[],
+    schema?: JSONSchema4,
+    labels: {
+        user: string;
+        assistant: string;
+        system: string;
+    } = { user: "User", assistant: "Assistant", system: "System" }
+) {
+    const promptMessages = [];
+    for (const m of messages) {
+        if (m.role === PromptRole.user) {
+            promptMessages.push(`${labels?.user}: ${m.content.trim()}`);
+        }
+        if (m.role === PromptRole.assistant) {
+            promptMessages.push(`${labels.assistant}: ${m.content.trim()}`);
+        }
+        if (m.role === PromptRole.system) {
+            promptMessages.push(`${labels.system}: ${m.content.trim()}`);
+        }
+    }
+
+    if (schema) {
+        promptMessages.push(`${labels.system}: You must answer using the following JSONSchema:
+        ---
+        ${JSON.stringify(schema)}
+        ---`);
+    }
+
+    return promptMessages.join("\n\n");
+}
+
+function claude(messages: PromptSegment[], schema?: JSONSchema4) {
+    const prompt = genericColonSeparator(messages, schema, {
+        user: "\nHuman",
+        assistant: "\nAssistant",
+        system: "\nHuman",
+    });
+
+    return "\n\n" + prompt + "\n\nAssistant:";
+}
+
+function formatSchemaInstruction(schema: JSONSchema4) {
+    const schema_instruction = `<<SYS>>You must answer using the following JSONSchema.
+Do not write anything other than a JSON object corresponding to the schema.
+<schema>
+${JSON.stringify(schema)}
+</schema>
+<</SYS>>`;
+
+    return schema_instruction;
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from "./Driver.js";
+export * from "./formatters.js";
+export * from "./json.js";
+export * from "./types.js";

package/src/json.ts

@@ -0,0 +1,17 @@
+
+function extractJsonFromText(text: string): string {
+    const start = text.indexOf("{");
+    const end = text.lastIndexOf("}");
+    return text.substring(start, end + 1);
+}
+
+//TODO LAX parse JSON
+export function parseJSON(text: string): Json {
+    return JSON.parse(extractJsonFromText(text));
+}
+
+export type JsonPrimative = string | number | boolean | null;
+export type JsonArray = Json[];
+export type JsonObject = { [key: string]: Json };
+export type JsonComposite = JsonArray | JsonObject;
+export type Json = JsonPrimative | JsonComposite;

package/src/types.ts

@@ -0,0 +1,187 @@
+import { JSONSchema4 } from "json-schema";
+import { Readable } from "stream";
+import { JsonObject } from "./json.js";
+
+export interface ResultValidationError {
+    code: 'validation_error' | 'json_error';
+    message: string;
+    data?: string;
+}
+
+export interface Completion<ResultT = any> {
+    // the driver impl must return the result and optionally the token_usage. the execution time is computed by the extended abstract driver 
+    result: ResultT;
+    token_usage?: ExecutionTokenUsage;
+    execution_time?: number;
+    /**
+     * Set only if a result validation error occured, otherwise if the result is valid the error field is undefined
+     * This can only be set if the resultSchema is set and the reuslt could not be parsed as a json or if the result does not match the schema
+     */
+    error?: ResultValidationError;
+}
+
+export interface ExecutionResponse<PromptT = any> extends Completion {
+    prompt: PromptT;
+}
+
+
+export interface CompletionStream<PromptT = any> extends AsyncIterable<string> {
+    completion: ExecutionResponse<PromptT> | undefined;
+}
+
+export interface Logger {
+    debug: (...obj: any[]) => void;
+    info: (...obj: any[]) => void;
+    warn: (...obj: any[]) => void;
+    error: (...obj: any[]) => void;
+}
+
+export interface DriverOptions {
+    logger?: Logger | false;
+}
+
+export interface PromptOptions {
+    model: string;
+    format?: PromptFormats;
+    resultSchema?: JSONSchema4;
+}
+export interface ExecutionOptions extends PromptOptions {
+    temperature?: number;
+    max_tokens?: number;
+}
+
+// ============== Prompts ===============
+export enum PromptRole {
+    safety = "safety",
+    system = "system",
+    user = "user",
+    assistant = "assistant",
+}
+
+export interface PromptSegment {
+    role: PromptRole;
+    content: string;
+}
+
+export interface ExecutionTokenUsage {
+    prompt?: number;
+    result?: number;
+    total?: number;
+}
+
+
+// ============== AI MODEL ==============
+
+export interface AIModel<ProviderKeys = string> {
+    id: string; //id of the model known by the provider
+    name: string; //human readable name
+    provider: ProviderKeys; //provider name
+    description?: string;
+    version?: string; //if any version is specified
+    type?: ModelType; //type of the model
+    tags?: string[]; //tags for searching
+    owner?: string; //owner of the model
+    status?: AIModelStatus; //status of the model
+    canStream?: boolean; //if the model's reponse can be streamed
+    isCustom?: boolean; //if the model is a custom model (a trained model)
+}
+
+export enum AIModelStatus {
+    Available = "available",
+    Pending = "pending",
+    Stopped = "stopped",
+    Unavailable = "unavailable",
+    Unknown = "unknown"
+}
+
+/**
+ * payload to list available models for an enviroment
+ * @param environmentId id of the environment
+ * @param query text to search for in model name/description
+ * @param type type of the model
+ * @param tags tags for searching
+ */
+export interface ModelSearchPayload {
+    text: string;
+    type?: ModelType;
+    tags?: string[];
+    owner?: string;
+}
+
+
+export enum ModelType {
+    Classifier = "classifier",
+    Regressor = "regressor",
+    Clustering = "clustering",
+    AnomalyDetection = "anomaly-detection",
+    TimeSeries = "time-series",
+    Text = "text",
+    Image = "image",
+    Audio = "audio",
+    Video = "video",
+    Embedding = "embedding",
+    Chat = "chat",
+    Code = "code",
+    NLP = "nlp",
+    MultiModal = "multi-modal",
+    Test = "test",
+    Other = "other",
+    Unknown = "unknown"
+}
+
+// ============== Built-in formats and drivers =====================
+//TODO
+
+export enum PromptFormats {
+    openai = "openai",
+    llama2 = "llama2",
+    claude = "claude",
+    genericTextLLM = "genericTextLLM",
+}
+
+export enum BuiltinProviders {
+    openai = 'openai',
+    huggingface_ie = 'huggingface_ie',
+    replicate = 'replicate',
+    bedrock = 'bedrock',
+    vertexai = 'vertexai',
+    togetherai = 'togetherai',
+    //virtual = 'virtual',    
+    //cohere = 'cohere',
+}
+
+// ============== training =====================
+
+
+export interface DataSource {
+    name: string;
+    getStream(): Readable;
+    getURL(): Promise<string>;
+}
+
+export interface TrainingOptions {
+    name: string; // the new model name
+    model: string; // the model to train 
+    params?: JsonObject; // the training parameters
+}
+
+export interface TrainingPromptOptions {
+    segments: PromptSegment[];
+    completion: string | JsonObject;
+    model: string; // the model to train 
+    schema?: JSONSchema4; // the resuilt schema f any
+}
+
+export enum TrainingJobStatus {
+    running = "running",
+    succeeded = "succeeded",
+    failed = "failed",
+    cancelled = "cancelled",
+}
+
+export interface TrainingJob {
+    id: string; // id of the training job
+    status: TrainingJobStatus; // status of the training job - depends on the implementation
+    details?: string;
+    model?: string; // the name of the fine tuned model which is created
+}

package/src/validation.ts

@@ -0,0 +1,34 @@
+import { JSONSchema4, validate } from "json-schema";
+import { parseJSON } from "./json.js";
+import { ResultValidationError } from "./types.js";
+
+export class ValidationError extends Error implements ResultValidationError {
+    constructor(
+        public code: 'validation_error' | 'json_error',
+        message: string
+    ) {
+        super(message)
+        this.name = 'ValidationError'
+    }
+}
+
+export function validateResult(data: any, schema: JSONSchema4) {
+    let json;
+    if (typeof data === "string") {
+        try {
+            json = parseJSON(data);
+        } catch (error: any) {
+            throw new ValidationError("json_error", error.message)
+        }
+    } else {
+        json = data;
+    }
+    const validation = validate(json, schema);
+    if (!validation.valid) {
+        throw new ValidationError(
+            "validation_error",
+            validation.errors.map(e => e.message).join(",\n"))
+    }
+
+    return json;
+}