@llumiverse/core 0.10.0 → 0.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@llumiverse/core",
-    "version": "0.10.0",
+    "version": "0.12.0",
     "type": "module",
     "description": "Provide an universal API to LLMs. Support for existing LLMs can be added by writing a driver.",
     "files": [
@@ -32,6 +32,9 @@
         "*": {
             "async": [
                 "./lib/types/async.d.ts"
+            ],
+            "formatters": [
+                "./lib/types/formatters/index.d.ts"
             ]
         }
     },
@@ -47,6 +50,12 @@
             "default": "./lib/esm/async.js",
             "import": "./lib/esm/async.js",
             "require": "./lib/cjs/async.js"
+        },
+        "./formatters": {
+            "types": "./lib/types/formatters/index.d.ts",
+            "default": "./lib/esm/formatters/index.js",
+            "import": "./lib/esm/formatters/index.js",
+            "require": "./lib/cjs/formatters/index.js"
         }
     },
     "scripts": {
@@ -66,8 +75,8 @@
         "@types/json-schema": "^7.0.15",
         "api-fetch-client": "^0.8.6",
         "ts-dual-module": "^0.6.2",
-        "typescript": "^5.3.3",
-        "vitest": "^1.2.2"
+        "typescript": "^5.4.2",
+        "vitest": "^1.4.0"
     },
     "dependencies": {
         "json-schema": "^0.4.0"
@@ -75,7 +84,8 @@
     "ts_dual_module": {
         "outDir": "lib",
         "exports": {
-            "async": "async.js"
+            "async": "async.js",
+            "formatters": "formatters/index.js"
         }
     }
-}
+}

package/src/CompletionStream.ts

@@ -60,7 +60,6 @@ export class DefaultCompletionStream<PromptT = any> implements CompletionStream<
 export class FallbackCompletionStream<PromptT = any> implements CompletionStream<PromptT> {
 
     prompt: PromptT;
-    chunks: string[];
     completion: ExecutionResponse<PromptT> | undefined;
 
     constructor(public driver: AbstractDriver<DriverOptions, PromptT>,
@@ -68,22 +67,16 @@ export class FallbackCompletionStream<PromptT = any> implements CompletionStream
         public options: ExecutionOptions) {
         this.driver = driver;
         this.prompt = this.driver.createPrompt(segments, options);
-        this.chunks = [];
     }
 
     async *[Symbol.asyncIterator]() {
         // reset state
         this.completion = undefined;
-        if (this.chunks.length > 0) {
-            this.chunks = [];
-        }
         this.driver.logger.debug(
             `[${this.driver.provider}] Streaming is not supported, falling back to blocking execution`
         );
         const completion = await this.driver._execute(this.prompt, this.options);
-
-        const content = completion.result === 'string' ? completion.result : JSON.stringify(completion.result);
-        this.chunks.push(content);
+        const content = typeof completion.result === 'string' ? completion.result : JSON.stringify(completion.result);
         yield content;
 
         this.completion = completion;

package/src/Driver.ts

@@ -5,7 +5,7 @@
  */
 
 import { DefaultCompletionStream, FallbackCompletionStream } from "./CompletionStream.js";
-import { PromptFormatters } from "./formatters/index.js";
+import { formatLlama2Prompt, formatTextPrompt } from "./formatters/index.js";
 import {
     AIModel,
     Completion,
@@ -18,7 +18,6 @@ import {
     ExecutionResponse,
     Logger,
     ModelSearchPayload,
-    PromptFormats,
     PromptOptions,
     PromptSegment,
     TrainingJob,
@@ -54,7 +53,7 @@ export function createLogger(logger: Logger | "console" | undefined) {
 
 function applyExecutionDefaults(options: ExecutionOptions): ExecutionOptions {
     return {
-        max_tokens: 1024,
+        max_tokens: 2048,
         temperature: 0.7,
         ...options
     }
@@ -106,7 +105,6 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
     logger: Logger;
 
     abstract provider: string; // the provider name
-    abstract defaultFormat: PromptFormats;
 
     constructor(opts: OptionsT) {
         this.options = opts;
@@ -181,11 +179,22 @@ export abstract class AbstractDriver<OptionsT extends DriverOptions = DriverOpti
         }
     }
 
+    /**
+     * Override this method to provide a custom prompt formatter
+     * @param segments 
+     * @param options 
+     * @returns 
+     */
+    protected formatPrompt(segments: PromptSegment[], opts: PromptOptions): PromptT {
+        if (/\bllama2?\b/i.test(opts.model)) {
+            return formatLlama2Prompt(segments, opts.resultSchema) as PromptT;
+        } else {
+            return formatTextPrompt(segments, opts.resultSchema) as PromptT;
+        }
+    }
+
     public createPrompt(segments: PromptSegment[], opts: PromptOptions): PromptT {
-        return PromptFormatters[opts.format || this.defaultFormat](
-            segments,
-            opts.resultSchema
-        );
+        return opts.format ? opts.format(segments, opts.resultSchema) : this.formatPrompt(segments, opts);
     }
 
     /**

package/src/formatters/claude.ts

@@ -1,5 +1,6 @@
 import { JSONSchema4 } from "json-schema";
 import { PromptRole, PromptSegment } from "../index.js";
+import { getJSONSafetyNotice } from "./commons.js";
 
 export interface ClaudeMessage {
     role: 'user' | 'assistant',
@@ -19,7 +20,7 @@ export interface ClaudeMessagesPrompt {
  * A formatter user by Bedrock to format prompts for claude related models
  */
 
-export function claudeMessages(segments: PromptSegment[], schema?: JSONSchema4): ClaudeMessagesPrompt {
+export function formatClaudePrompt(segments: PromptSegment[], schema?: JSONSchema4): ClaudeMessagesPrompt {
     const system: string[] = [];
     const safety: string[] = [];
     const messages: ClaudeMessage[] = [];
@@ -35,13 +36,9 @@ export function claudeMessages(segments: PromptSegment[], schema?: JSONSchema4):
     }
 
     if (schema) {
-        safety.push(`You must answer using the following JSONSchema:
----
-${JSON.stringify(schema)}
----`);
+        safety.push(getJSONSafetyNotice(schema));
     }
 
-
     // messages must contains at least 1 item. If the prompt doesn;t contains a user message (but only system messages)
     // we need to put the system messages in the messages array
 

package/src/formatters/commons.ts

@@ -0,0 +1,5 @@
+import { JSONSchema4 } from "json-schema";
+
+export function getJSONSafetyNotice(schema: JSONSchema4) {
+    return "The answer must be a JSON object using the following JSON Schema:\n" + JSON.stringify(schema);
+}

package/src/formatters/generic.ts

@@ -1,34 +1,66 @@
 import { JSONSchema4 } from "json-schema";
-import { PromptRole, PromptSegment } from "../index.js";
+import { PromptRole, PromptSegment } from "../types.js";
+import { getJSONSafetyNotice } from "./commons.js";
 
-export function genericColonSeparator(
-    messages: PromptSegment[],
-    schema?: JSONSchema4,
-    labels: {
-        user: string;
-        assistant: string;
-        system: string;
-    } = { user: "User", assistant: "Assistant", system: "System" }
-) {
-    const promptMessages = [];
-    for (const m of messages) {
-        if (m.role === PromptRole.user) {
-            promptMessages.push(`${labels?.user}: ${m.content.trim()}`);
-        }
-        if (m.role === PromptRole.assistant) {
-            promptMessages.push(`${labels.assistant}: ${m.content.trim()}`);
+interface Labels {
+    user: string,
+    system: string,
+    assistant: string,
+    safety: string,
+    instruction: string
+}
+
+export function createTextPromptFormatter(labels: Labels = {
+    user: "USER",
+    system: "CONTEXT",
+    assistant: "ASSISTANT",
+    safety: "IMPORTANT",
+    instruction: "INSTRUCTION"
+}) {
+    return function genericTextPrompt(segments: PromptSegment[], schema?: JSONSchema4): string {
+        const isChat = segments.find(m => m.role === PromptRole.assistant);
+        const context: string[] = [];
+        const content: string[] = [];
+        const safety: string[] = [];
+        for (const segment of segments) {
+            switch (segment.role) {
+                case PromptRole.user:
+                    if (isChat) {
+                        content.push(`${labels.user}: ${segment.content}`);
+                    } else {
+                        content.push(segment.content);
+                    }
+                    break;
+                case PromptRole.assistant:
+                    content.push(`${labels.assistant}: ${segment.content}`);
+                    break;
+                case PromptRole.system:
+                    context.push(segment.content);
+                    break;
+                case PromptRole.safety:
+                    safety.push(segment.content);
+                    break;
+            }
         }
-        if (m.role === PromptRole.system) {
-            promptMessages.push(`${labels.system}: ${m.content.trim()}`);
+
+        if (schema) {
+            safety.push(getJSONSafetyNotice(schema));
         }
-    }
 
-    if (schema) {
-        promptMessages.push(`${labels.system}: You must answer using the following JSONSchema:
-        ---
-        ${JSON.stringify(schema)}
-        ---`);
+        const out = [];
+        if (context.length > 0) {
+            out.push(`${labels.system}: ${context.join('\n')}`);
+        }
+        if (content.length > 0) {
+            const prefix = context.length > 0 && !isChat ? `${labels.instruction}: ` : '';
+            out.push(prefix + content.join('\n'));
+        }
+        if (safety.length > 0) {
+            out.push(`${labels.safety}: ${safety.join('\n')}`);
+        }
+        return out.join('\n');
     }
-
-    return promptMessages.join("\n\n");
 }
+
+const formatTextPrompt = createTextPromptFormatter();
+export { formatTextPrompt };

package/src/formatters/index.ts

@@ -1,33 +1,10 @@
 import { JSONSchema4 } from "json-schema";
-import { genericColonSeparator } from "./generic.js";
-import { llama2 } from "./llama2.js";
-import { openAI } from "./openai.js";
-import {
-    PromptFormats,
-    PromptSegment
-} from "../types.js";
-import { claudeMessages } from "./claude.js";
+import { PromptSegment } from "../types.js";
 
-export function inferFormatterFromModelName(modelName: string): PromptFormats {
-    const name = modelName.toLowerCase();
-    if (name.includes("llama")) {
-        return PromptFormats.llama2;
-    } else if (name.includes("gpt")) {
-        return PromptFormats.openai;
-    } else if (name.includes("claude")) {
-        return PromptFormats.claude;
-    } else {
-        return PromptFormats.genericTextLLM;
-    }
-}
-
-export const PromptFormatters: Record<
-    PromptFormats,
-    (messages: PromptSegment[], schema?: JSONSchema4) => any
-> = {
-    openai: openAI,
-    llama2: llama2,
-    claude: claudeMessages,
-    genericTextLLM: genericColonSeparator,
-};
+export type PromptFormatter<T = any> = (messages: PromptSegment[], schema?: JSONSchema4) => T;
 
+export * from "./commons.js"
+export * from "./generic.js";
+export * from "./llama2.js";
+export * from "./claude.js";
+export * from "./openai.js";

package/src/formatters/llama2.ts

@@ -1,7 +1,7 @@
 import { JSONSchema4 } from "json-schema";
 import { PromptRole, PromptSegment } from "../index.js";
 
-export function llama2(messages: PromptSegment[], schema?: JSONSchema4) {
+export function formatLlama2Prompt(messages: PromptSegment[], schema?: JSONSchema4) {
     const BOS = "<s>";
     const EOS = "</s>";
     const INST = "[INST]";

package/src/formatters/openai.ts

@@ -1,17 +1,25 @@
 import { PromptRole } from "../index.js";
 import { PromptSegment } from "../types.js";
-import OpenAI from "openai";
 
-export function openAI(segments: PromptSegment[]) {
-    const system: OpenAI.Chat.ChatCompletionMessageParam[] = [];
-    const others: OpenAI.Chat.ChatCompletionMessageParam[] = [];
-    const safety: OpenAI.Chat.ChatCompletionMessageParam[] = [];
+export interface OpenAITextMessage {
+    content: string;
+    role: "system" | "user" | "assistant";
+}
+/**
+ * OpenAI text only prompts
+ * @param segments 
+ * @returns 
+ */
+export function formatOpenAILikePrompt(segments: PromptSegment[]) {
+    const system: OpenAITextMessage[] = [];
+    const others: OpenAITextMessage[] = [];
+    const safety: OpenAITextMessage[] = [];
 
     for (const msg of segments) {
         if (msg.role === PromptRole.system) {
             system.push({ content: msg.content, role: "system" });
         } else if (msg.role === PromptRole.safety) {
-            safety.push({ content: msg.content, role: "system" });
+            safety.push({ content: "IMPORTANT: " + msg.content, role: "system" });
         } else {
             others.push({ content: msg.content, role: "user" });
         }

package/src/index.ts

@@ -1,4 +1,3 @@
 export * from "./Driver.js";
-export * from "./formatters/index.js";
 export * from "./json.js";
 export * from "./types.js";

package/src/json.ts

@@ -6,15 +6,15 @@ function extractJsonFromText(text: string): string {
     return text.replace(/\\n/g, "");
 }
 
-export function extractAndParseJSON(text: string): Json {
+export function extractAndParseJSON(text: string): JSONValue {
     return parseJSON(extractJsonFromText(text));
 }
 
-export type JsonPrimative = string | number | boolean | null;
-export type JsonArray = Json[];
-export type JsonObject = { [key: string]: Json };
-export type JsonComposite = JsonArray | JsonObject;
-export type Json = JsonPrimative | JsonComposite;
+export type JSONPrimitive = string | number | boolean | null;
+export type JSONArray = JSONValue[];
+export type JSONObject = { [key: string]: JSONValue };
+export type JSONComposite = JSONArray | JSONObject;
+export type JSONValue = JSONPrimitive | JSONComposite;
 
 
 
@@ -184,7 +184,7 @@ export class JsonParser {
 }
 
 
-export function parseJSON(text: string): Json {
+export function parseJSON(text: string): JSONValue {
     text = text.trim();
     try {
         return JSON.parse(text);

package/src/types.ts

@@ -1,6 +1,6 @@
 import { JSONSchema4 } from "json-schema";
-import { Readable } from "stream";
-import { JsonObject } from "./json.js";
+import { JSONObject } from "./json.js";
+import { PromptFormatter } from "./formatters/index.js";
 
 export interface EmbeddingsOptions {
     /**
@@ -48,11 +48,22 @@ export interface Completion<ResultT = any> {
     result: ResultT;
     token_usage?: ExecutionTokenUsage;
 
+    /**
+     * The finish reason as reported by the model: stop | length or other model specific values
+     */
+    finish_reason?: "stop" | "length" | string;
+
     /**
      * Set only if a result validation error occured, otherwise if the result is valid the error field is undefined
      * This can only be set if the resultSchema is set and the reuslt could not be parsed as a json or if the result does not match the schema
      */
     error?: ResultValidationError;
+
+    /**
+     * The original response. Only included if the option include_original_response is set to true and the request is made using execute. Not supported when streaming.
+     */
+    original_response?: Record<string, any>;
+
 }
 
 export interface ExecutionResponse<PromptT = any> extends Completion {
@@ -81,12 +92,57 @@ export interface DriverOptions {
 
 export interface PromptOptions {
     model: string;
-    format?: PromptFormats;
+    /**
+     * A custom formatter to use for format the final model prompt from the input prompt segments. 
+     * If no one is specified the driver will choose a formatter compatible with the target model
+     */
+    format?: PromptFormatter;
     resultSchema?: JSONSchema4;
 }
 export interface ExecutionOptions extends PromptOptions {
     temperature?: number;
     max_tokens?: number;
+    stop_sequence?: string | string[];
+
+    /** 
+     * restricts the selection of tokens to the “k” most likely options, based on their probabilities
+     * Lower values make the model more deterministic, more focused. Examples:
+     * - 10 - result will be highly controlled anc contextually relevant
+     * - 50 - result will be more creative but maintaining a balance between control and creativity
+     * - 100 - will lead to more creative and less predictable outputs
+     * It will be ignored on OpenAI since it does not support it
+     */
+    top_k?: number;
+
+    /**
+     * An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
+     * Either use temperature or top_p, not both
+     */
+    top_p?: number;
+
+    /**
+     * Only supported for OpenAI. Look at OpenAI documentation for more detailsx
+     */
+    top_logprobs?: number;
+
+    /**
+     * Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.
+     * Ignored for models which doesn;t support it
+     */
+    presence_penalty?: number;
+
+    /**
+     * Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.
+     * Ignored for models which doesn;t support it
+     */
+    frequency_penalty?: number;
+
+    /**
+     * If set to true the original response from the target LLM will be included in the response under the original_response field.
+     * This is useful for debugging and for some advanced use cases. 
+     * It is ignored on streaming requests
+     */
+    include_original_response?: boolean;
 }
 
 // ============== Prompts ===============
@@ -171,13 +227,6 @@ export enum ModelType {
 // ============== Built-in formats and drivers =====================
 //TODO
 
-export enum PromptFormats {
-    openai = "openai",
-    llama2 = "llama2",
-    claude = "claude",
-    genericTextLLM = "genericTextLLM",
-}
-
 export enum BuiltinProviders {
     openai = 'openai',
     huggingface_ie = 'huggingface_ie',
@@ -186,6 +235,7 @@ export enum BuiltinProviders {
     vertexai = 'vertexai',
     togetherai = 'togetherai',
     mistralai = 'mistralai',
+    groq = 'groq',
     //virtual = 'virtual',    
     //cohere = 'cohere',
 }
@@ -193,21 +243,22 @@ export enum BuiltinProviders {
 // ============== training =====================
 
 
+
 export interface DataSource {
     name: string;
-    getStream(): Readable;
+    getStream(): ReadableStream<Uint8Array | string>;
     getURL(): Promise<string>;
 }
 
 export interface TrainingOptions {
     name: string; // the new model name
     model: string; // the model to train 
-    params?: JsonObject; // the training parameters
+    params?: JSONObject; // the training parameters
 }
 
 export interface TrainingPromptOptions {
     segments: PromptSegment[];
-    completion: string | JsonObject;
+    completion: string | JSONObject;
     model: string; // the model to train 
     schema?: JSONSchema4; // the resuilt schema f any
 }