@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/ai/baml_client/type_builder.js

@@ -0,0 +1,96 @@
+/*************************************************************************************************
+
+Welcome to Baml! To use this generated code, please run one of the following:
+
+$ npm install @boundaryml/baml
+$ yarn add @boundaryml/baml
+$ pnpm add @boundaryml/baml
+
+*************************************************************************************************/
+import { TypeBuilder as _TypeBuilder } from '@boundaryml/baml/type_builder';
+import { DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME } from "./globals";
+export default class TypeBuilder {
+    tb;
+    AgentContext;
+    BugFailureClassification;
+    ConnectorInstructions;
+    ExtractedData;
+    MisalignmentClassification;
+    MultiMediaMessage;
+    PartialRecipe;
+    QueryResponse;
+    constructor() {
+        this.tb = new _TypeBuilder({
+            classes: new Set([
+                "AgentContext", "BugFailureClassification", "ConnectorInstructions", "ExtractedData", "MisalignmentClassification", "MultiMediaMessage", "PartialRecipe", "QueryResponse",
+            ]),
+            enums: new Set([]),
+            runtime: DO_NOT_USE_DIRECTLY_UNLESS_YOU_KNOW_WHAT_YOURE_DOING_RUNTIME
+        });
+        this.AgentContext = this.tb.classViewer("AgentContext", [
+            "instructions", "observationContent", "connectorInstructions",
+        ]);
+        this.BugFailureClassification = this.tb.classViewer("BugFailureClassification", [
+            "reasoning", "classification", "title", "expectedResult", "actualResult", "severity",
+        ]);
+        this.ConnectorInstructions = this.tb.classViewer("ConnectorInstructions", [
+            "connectorId", "instructions",
+        ]);
+        this.ExtractedData = this.tb.classBuilder("ExtractedData", []);
+        this.MisalignmentClassification = this.tb.classViewer("MisalignmentClassification", [
+            "reasoning", "classification", "fault", "message",
+        ]);
+        this.MultiMediaMessage = this.tb.classViewer("MultiMediaMessage", [
+            "role", "cacheControl", "content",
+        ]);
+        this.PartialRecipe = this.tb.classBuilder("PartialRecipe", [
+            "reasoning",
+        ]);
+        this.QueryResponse = this.tb.classBuilder("QueryResponse", []);
+    }
+    __tb() {
+        return this.tb._tb();
+    }
+    string() {
+        return this.tb.string();
+    }
+    literalString(value) {
+        return this.tb.literalString(value);
+    }
+    literalInt(value) {
+        return this.tb.literalInt(value);
+    }
+    literalBool(value) {
+        return this.tb.literalBool(value);
+    }
+    int() {
+        return this.tb.int();
+    }
+    float() {
+        return this.tb.float();
+    }
+    bool() {
+        return this.tb.bool();
+    }
+    list(type) {
+        return this.tb.list(type);
+    }
+    null() {
+        return this.tb.null();
+    }
+    map(key, value) {
+        return this.tb.map(key, value);
+    }
+    union(types) {
+        return this.tb.union(types);
+    }
+    addClass(name) {
+        return this.tb.addClass(name);
+    }
+    addEnum(name) {
+        return this.tb.addEnum(name);
+    }
+    addBaml(baml) {
+        this.tb.addBaml(baml);
+    }
+}

package/dist/ai/baml_client/types.d.ts

@@ -0,0 +1,69 @@
+/*************************************************************************************************
+
+Welcome to Baml! To use this generated code, please run one of the following:
+
+$ npm install @boundaryml/baml
+$ yarn add @boundaryml/baml
+$ pnpm add @boundaryml/baml
+
+*************************************************************************************************/
+import type { Image } from "@boundaryml/baml";
+/**
+ * Recursively partial type that can be null.
+ *
+ * @deprecated Use types from the `partial_types` namespace instead, which provides type-safe partial implementations
+ * @template T The type to make recursively partial.
+ */
+export type RecursivePartialNull<T> = T extends object ? {
+    [P in keyof T]?: RecursivePartialNull<T[P]>;
+} : T | null;
+export interface Checked<T, CheckName extends string = string> {
+    value: T;
+    checks: Record<CheckName, Check>;
+}
+export interface Check {
+    name: string;
+    expr: string;
+    status: "succeeded" | "failed";
+}
+export declare function all_succeeded<CheckName extends string>(checks: Record<CheckName, Check>): boolean;
+export declare function get_checks<CheckName extends string>(checks: Record<CheckName, Check>): Check[];
+export interface AgentContext {
+    instructions?: string | null;
+    observationContent: MultiMediaMessage[];
+    connectorInstructions: ConnectorInstructions[];
+}
+export interface BugFailureClassification {
+    reasoning: string;
+    classification: "bug";
+    title: string;
+    expectedResult: string;
+    actualResult: string;
+    severity: "critical" | "high" | "medium" | "low";
+}
+export interface ConnectorInstructions {
+    connectorId: string;
+    instructions: string;
+}
+export interface ExtractedData {
+    [key: string]: any;
+}
+export interface MisalignmentClassification {
+    reasoning: string;
+    classification: "misalignment";
+    fault: "test" | "agent";
+    message: string;
+}
+export interface MultiMediaMessage {
+    role: "user" | "assistant";
+    cacheControl: boolean;
+    content: (Image | string)[];
+}
+export interface PartialRecipe {
+    reasoning: string;
+    [key: string]: any;
+}
+export interface QueryResponse {
+    [key: string]: any;
+}
+export type MultiMediaContentPart = Image | string;

package/dist/ai/baml_client/types.js

@@ -0,0 +1,15 @@
+/*************************************************************************************************
+
+Welcome to Baml! To use this generated code, please run one of the following:
+
+$ npm install @boundaryml/baml
+$ yarn add @boundaryml/baml
+$ pnpm add @boundaryml/baml
+
+*************************************************************************************************/
+export function all_succeeded(checks) {
+    return get_checks(checks).every(check => check.status === "succeeded");
+}
+export function get_checks(checks) {
+    return Object.values(checks);
+}

package/dist/ai/claudeCode.d.ts

@@ -0,0 +1 @@
+export declare function completeClaudeCodeAuthFlow(): Promise<string>;

package/dist/ai/claudeCode.js

@@ -0,0 +1,186 @@
+import { promises as fs } from 'fs';
+import { homedir } from 'os';
+import { join, dirname } from 'path';
+import crypto from 'crypto';
+import { bold, cyanBright } from 'ansis';
+import { exec } from 'node:child_process';
+// Simple cross-platform function to open URLs
+function openUrl(url) {
+    return new Promise((resolve, reject) => {
+        let command;
+        switch (process.platform) {
+            case 'darwin':
+                command = `open "${url}"`;
+                break;
+            case 'win32':
+                command = `start "${url}"`;
+                break;
+            default:
+                command = `xdg-open "${url}"`;
+                break;
+        }
+        exec(command, (error) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+// Constants
+const CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e';
+const CREDS_PATH = join(homedir(), '.magnitude', 'credentials', 'claudeCode.json');
+// 1. Generate PKCE pair
+function generatePKCE() {
+    const verifier = crypto.randomBytes(32).toString('base64url');
+    const challenge = crypto
+        .createHash('sha256')
+        .update(verifier)
+        .digest('base64url');
+    return { verifier, challenge };
+}
+// 2. Get OAuth authorization URL
+function getAuthorizationURL(pkce) {
+    const url = new URL('https://claude.ai/oauth/authorize');
+    url.searchParams.set('code', 'true');
+    url.searchParams.set('client_id', CLIENT_ID);
+    url.searchParams.set('response_type', 'code');
+    url.searchParams.set('redirect_uri', 'https://console.anthropic.com/oauth/code/callback');
+    url.searchParams.set('scope', 'org:create_api_key user:profile user:inference');
+    url.searchParams.set('code_challenge', pkce.challenge);
+    url.searchParams.set('code_challenge_method', 'S256');
+    url.searchParams.set('state', pkce.verifier);
+    return url.toString();
+}
+// 3. Exchange authorization code for tokens
+async function exchangeCodeForTokens(code, verifier) {
+    const [authCode, state] = code.split('#');
+    const response = await fetch('https://console.anthropic.com/v1/oauth/token', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            code: authCode,
+            state: state,
+            grant_type: 'authorization_code',
+            client_id: CLIENT_ID,
+            redirect_uri: 'https://console.anthropic.com/oauth/code/callback',
+            code_verifier: verifier,
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`Token exchange failed: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return {
+        access_token: data.access_token,
+        refresh_token: data.refresh_token,
+        expires_at: Date.now() + (data.expires_in * 1000),
+    };
+}
+// 4. Refresh access token
+async function refreshAccessToken(refreshToken) {
+    const response = await fetch('https://console.anthropic.com/v1/oauth/token', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            grant_type: 'refresh_token',
+            refresh_token: refreshToken,
+            client_id: CLIENT_ID,
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`Token refresh failed: ${response.statusText}`);
+    }
+    const data = await response.json();
+    return {
+        access_token: data.access_token,
+        refresh_token: data.refresh_token,
+        expires_at: Date.now() + (data.expires_in * 1000),
+    };
+}
+// 5. Save credentials
+async function saveCredentials(creds) {
+    await fs.mkdir(dirname(CREDS_PATH), { recursive: true });
+    await fs.writeFile(CREDS_PATH, JSON.stringify(creds, null, 2));
+    await fs.chmod(CREDS_PATH, 0o600); // Read/write for owner only
+}
+// 6. Load credentials
+async function loadCredentials() {
+    try {
+        const data = await fs.readFile(CREDS_PATH, 'utf-8');
+        return JSON.parse(data);
+    }
+    catch {
+        return null;
+    }
+}
+// 7. Get valid access token (refresh if needed)
+async function getValidAccessToken() {
+    const creds = await loadCredentials();
+    if (!creds)
+        return null;
+    // If token is still valid, return it
+    if (creds.expires_at > Date.now() + 60000) { // 1 minute buffer
+        return creds.access_token;
+    }
+    // Otherwise, refresh it
+    try {
+        const newCreds = await refreshAccessToken(creds.refresh_token);
+        await saveCredentials(newCreds);
+        return newCreds.access_token;
+    }
+    catch {
+        return null;
+    }
+}
+export async function completeClaudeCodeAuthFlow() {
+    // Try to get existing valid token
+    const existingToken = await getValidAccessToken();
+    if (existingToken)
+        return existingToken;
+    // Otherwise, go through auth flow
+    const pkce = generatePKCE();
+    const authUrl = getAuthorizationURL(pkce);
+    console.log(bold `Claude Code Pro/Max access token missing or expired.`);
+    //console.log(cyanBright`Accounts with Max plan can be used for API access.`)
+    console.log(cyanBright `Opening browser for authentication...`);
+    try {
+        await openUrl(authUrl);
+    }
+    catch (err) {
+        console.log('Could not open browser automatically');
+    }
+    //console.log('Open this URL in your browser:');
+    console.log(bold `\nIf browser did not open, visit:`);
+    console.log(authUrl);
+    console.log(bold `\nPaste the authorization code here:`);
+    const code = await new Promise((resolve) => {
+        process.stdin.once('data', (data) => {
+            resolve(data.toString().trim());
+        });
+    });
+    const creds = await exchangeCodeForTokens(code, pkce.verifier);
+    await saveCredentials(creds);
+    console.log(bold `\nCredentials saved!`);
+    return creds.access_token;
+}
+/*
+// Usage
+const response = await fetch('https://api.anthropic.com/v1/messages', {
+  method: 'POST',
+  headers: {
+    'Content-Type': 'application/json',
+    'Authorization': `Bearer ${accessToken}`,
+    'anthropic-beta': 'oauth-2025-04-20',
+    // make sure to exclude x-api-key
+  },
+  body: JSON.stringify({
+    model: 'claude-3-5-sonnet-20241022',
+    messages: [{ role: 'user', content: 'Hello!' }],
+    max_tokens: 100,
+  }),
+});
+
+*/

package/dist/ai/modelHarness.d.ts

@@ -0,0 +1,39 @@
+import { AgentContext } from "@/ai/baml_client";
+import { Action } from "@/actions/types";
+import { LLMClient, ModelUsage } from "@/ai/types";
+import { ActionDefinition } from "@/actions";
+import { Schema, z } from 'zod';
+import { Image } from '@/memory/image';
+import EventEmitter from "eventemitter3";
+import { MultiMediaContentPart } from "@/memory/rendering";
+interface ModelHarnessOptions {
+    llm: LLMClient;
+}
+export interface ModelHarnessEvents {
+    'tokensUsed': (usage: ModelUsage) => {};
+}
+export declare class ModelHarness {
+    /**
+     * Strong reasoning agent for high level strategy and planning.
+     */
+    readonly events: EventEmitter<ModelHarnessEvents>;
+    private options;
+    private collector;
+    private cr;
+    private baml;
+    private logger;
+    private prevTotalInputTokens;
+    private prevTotalOutputTokens;
+    constructor(options: ModelHarnessOptions);
+    setup(): Promise<void>;
+    describeModel(): string;
+    private _reportUsage;
+    partialAct<T>(context: AgentContext, // Changed to ModularMemoryContext
+    task: string, data: MultiMediaContentPart[], actionVocabulary: ActionDefinition<T>[]): Promise<{
+        reasoning: string;
+        actions: Action[];
+    }>;
+    extract<T extends Schema>(instructions: string, schema: T, screenshot: Image, domContent: string): Promise<z.infer<T>>;
+    query<T extends Schema>(context: AgentContext, query: string, schema: T): Promise<z.infer<T>>;
+}
+export {};

package/dist/ai/modelHarness.js

@@ -0,0 +1,189 @@
+import { convertToBamlClientOptions } from "./util";
+// Import ModularMemoryContext instead of old MemoryContext
+import { b } from "@/ai/baml_client";
+import { Collector, ClientRegistry } from "@boundaryml/baml";
+import logger from "@/logger";
+import TypeBuilder from "./baml_client/type_builder";
+import { z } from 'zod';
+import { convertActionDefinitionsToBaml, convertZodToBaml } from "@/actions/util";
+import EventEmitter from "eventemitter3";
+export class ModelHarness {
+    /**
+     * Strong reasoning agent for high level strategy and planning.
+     */
+    events = new EventEmitter();
+    options;
+    collector;
+    cr;
+    baml;
+    logger;
+    prevTotalInputTokens = 0;
+    prevTotalOutputTokens = 0;
+    constructor(options) {
+        this.options = {
+            llm: options.llm,
+            //promptCaching: options.promptCaching ?? false
+        };
+        this.logger = logger.child({ name: 'llm' });
+    }
+    async setup() {
+        // Must be called after constructor
+        this.collector = new Collector("macro");
+        this.cr = new ClientRegistry();
+        let bamlClientOptions = await convertToBamlClientOptions(this.options.llm);
+        this.cr.addLlmClient('Magnus', this.options.llm.provider === 'claude-code' ? 'anthropic' : this.options.llm.provider, bamlClientOptions, 'DefaultRetryPolicy');
+        this.cr.setPrimary('Magnus');
+        this.baml = b.withOptions({ collector: this.collector, clientRegistry: this.cr });
+    }
+    describeModel() {
+        return `${this.options.llm.provider}:${'model' in this.options.llm.options ? this.options.llm.options.model : 'unknown'}`;
+    }
+    _reportUsage() {
+        // console.log('this.collector.last', this.collector.last)
+        // if (this.collector.last) console.log("calls:", this.collector.last.calls)//console.log("Response: ", this.collector.last.calls[-1].httpResponse);
+        //console.log('last call:', this.collector.last?.calls.at(-1)?.httpResponse?.body.json());
+        // Get tokens used since last call to reportUsage
+        //console.log(this.collector.usage);
+        let inputTokens = 0;
+        let outputTokens = 0;
+        let cacheWriteInputTokens = 0;
+        let cacheReadInputTokens = 0;
+        if (this.options.llm.provider === 'anthropic' || this.options.llm.provider === 'claude-code') {
+            const usage = this.collector.last?.calls.at(-1)?.httpResponse?.body.json().usage;
+            //console.log("Usage from Anthropic:", usage);
+            if (!usage) {
+                // Sometimes apparently this happens? Happened once after extract for example
+                logger.warn("No usage returned from Anthropic provider, cached cost may be inaccurate");
+                inputTokens = (this.collector.usage.inputTokens ?? 0) - this.prevTotalInputTokens;
+                outputTokens = (this.collector.usage.outputTokens ?? 0) - this.prevTotalOutputTokens;
+            }
+            else {
+                inputTokens = usage.input_tokens;
+                outputTokens = usage.output_tokens;
+                cacheWriteInputTokens = usage.cache_creation_input_tokens;
+                cacheReadInputTokens = usage.cache_read_input_tokens;
+            }
+        }
+        else {
+            inputTokens = (this.collector.usage.inputTokens ?? 0) - this.prevTotalInputTokens;
+            outputTokens = (this.collector.usage.outputTokens ?? 0) - this.prevTotalOutputTokens;
+        }
+        const model = this.options.llm.options.model ?? 'unknown';
+        // Get cost if known
+        const knownCostMap = {
+            // TODO: track cached savings on Gemini
+            'gemini-2.5-pro': { inputTokens: 1.25, outputTokens: 10.0 },
+            'gemini-2.5-flash': { inputTokens: 0.30, outputTokens: 2.50 },
+            'gemini-2.5-flash-lite': { inputTokens: 0.10, outputTokens: 0.40 },
+            'claude-3.5-sonnet': { inputTokens: 3.00, outputTokens: 15.00, cacheWriteInputTokens: 3.75, cacheReadInputTokens: 0.30 },
+            'claude-3.7-sonnet': { inputTokens: 3.00, outputTokens: 15.00, cacheWriteInputTokens: 3.75, cacheReadInputTokens: 0.30 },
+            'claude-sonnet-4': { inputTokens: 3.00, outputTokens: 15.00, cacheWriteInputTokens: 3.75, cacheReadInputTokens: 0.30 },
+            'claude-haiku-4-5': { inputTokens: 1.00, outputTokens: 5.00, cacheWriteInputTokens: 1.25, cacheReadInputTokens: 0.10 },
+            'claude-opus-4': { inputTokens: 15.00, outputTokens: 75.00, cacheWriteInputTokens: 18.75, cacheReadInputTokens: 1.50 },
+            'gpt-4.1': { inputTokens: 2.00, outputTokens: 8.00 },
+            'gpt-4.1-mini': { inputTokens: 0.40, outputTokens: 1.60 },
+            'gpt-4.1-nano': { inputTokens: 0.10, outputTokens: 0.40 },
+            'gpt-4o': { inputTokens: 3.75, outputTokens: 15.00 },
+            // Assuming Nebius prices, may be higher
+            'qwen2.5-vl-72b': { inputTokens: 0.25, outputTokens: 0.75 }
+        };
+        let inputTokenCost;
+        let outputTokenCost;
+        let cacheWriteInputTokenCost;
+        let cacheReadInputTokenCost;
+        for (const [name, costs] of Object.entries(knownCostMap)) {
+            if (model.includes(name)) {
+                inputTokenCost = costs.inputTokens / 1_000_000;
+                outputTokenCost = costs.outputTokens / 1_000_000;
+                cacheReadInputTokenCost = costs.cacheReadInputTokens ? costs.cacheReadInputTokens / 1_000_000 : undefined;
+                cacheWriteInputTokenCost = costs.cacheWriteInputTokens ? costs.cacheWriteInputTokens / 1_000_000 : undefined;
+            }
+        }
+        // console.log("cacheWriteInputTokenCost:", cacheWriteInputTokenCost);
+        // console.log("cacheWriteInputTokens:", cacheWriteInputTokens);
+        const usage = {
+            llm: {
+                provider: this.options.llm.provider,
+                model: model
+            }, //this.options.llm,
+            inputTokens: inputTokens,
+            outputTokens: outputTokens,
+            ...(cacheWriteInputTokens ? { cacheWriteInputTokens } : {}),
+            ...(cacheReadInputTokens ? { cacheReadInputTokens } : {}),
+            ...(inputTokenCost ? {
+                inputCost: inputTokens * inputTokenCost +
+                    (cacheWriteInputTokenCost ? cacheWriteInputTokenCost * cacheWriteInputTokens : 0.0) +
+                    (cacheReadInputTokenCost ? cacheReadInputTokenCost * cacheReadInputTokens : 0.0)
+            } : {}),
+            ...(outputTokenCost ? { outputCost: outputTokens * outputTokenCost } : {}),
+            // ...(cacheWriteInputTokenCost ? { : inputTokens * inputTokenCost } : {}),
+            // ...(cacheReadInputTokenCost ? { outputCost: outputTokens * outputTokenCost } : {})
+        };
+        this.events.emit('tokensUsed', usage);
+        //console.log("Usage:", usage);
+        this.prevTotalInputTokens += inputTokens;
+        this.prevTotalOutputTokens += outputTokens;
+    }
+    async partialAct(context, // Changed to ModularMemoryContext
+    task, data, actionVocabulary) {
+        const tb = new TypeBuilder();
+        tb.PartialRecipe.addProperty('actions', tb.list(convertActionDefinitionsToBaml(tb, actionVocabulary))).description('Always provide at least one action');
+        const start = Date.now();
+        // Assuming this.baml.CreatePartialRecipe is now typed to accept ModularMemoryContext
+        // after BAML generation picked up changes in planner.baml
+        const response = await this.baml.CreatePartialRecipe(context, task, data, this.options.llm.provider === 'claude-code', { tb });
+        this.logger.trace(`createPartialRecipe took ${Date.now() - start}ms`);
+        // BAML does not carry over action type to @@dynamic of PartialRecipe, so forced cast necssary
+        //return response as unknown as { actions: z.infer<ActionDefinition<T>['schema']>[] };//, finished: boolean };
+        this._reportUsage();
+        return {
+            reasoning: response.reasoning, //(response.observations ? response.observations + " " : "") + response.meta_reasoning + " " + response.reasoning,
+            actions: response.actions // as z.infer<ActionDefinition<T>['schema']>[]
+        };
+    }
+    async extract(instructions, schema, screenshot, domContent) {
+        const tb = new TypeBuilder();
+        if (schema instanceof z.ZodObject) {
+            // populate ExtractedData with schema KVs instead of wrapping in data key unnecessarily
+            for (const [key, fieldSchema] of Object.entries(schema.shape)) {
+                tb.ExtractedData.addProperty(key, convertZodToBaml(tb, fieldSchema));
+            }
+        }
+        else {
+            // for array or primitive have to wrap data key
+            tb.ExtractedData.addProperty('data', convertZodToBaml(tb, schema));
+        }
+        // } else if (schema instanceof z.ZodArray) {
+        // }
+        const resp = await this.baml.ExtractData(instructions, await screenshot.toBaml(), domContent, this.options.llm.provider === 'claude-code', { tb });
+        this._reportUsage();
+        if (schema instanceof z.ZodObject) {
+            return resp;
+        }
+        else {
+            return resp.data;
+        }
+    }
+    // ^ extract could prob be a subset of query w trimmed mem
+    async query(context, query, schema) {
+        const tb = new TypeBuilder();
+        if (schema instanceof z.ZodObject) {
+            // populate ExtractedData with schema KVs instead of wrapping in data key unnecessarily
+            for (const [key, fieldSchema] of Object.entries(schema.shape)) {
+                tb.QueryResponse.addProperty(key, convertZodToBaml(tb, fieldSchema));
+            }
+        }
+        else {
+            // for array or primitive have to wrap data key
+            tb.QueryResponse.addProperty('data', convertZodToBaml(tb, schema));
+        }
+        const resp = await this.baml.QueryMemory(context, query, this.options.llm.provider === 'claude-code', { tb });
+        this._reportUsage();
+        if (schema instanceof z.ZodObject) {
+            return resp;
+        }
+        else {
+            return resp.data;
+        }
+    }
+}

package/dist/ai/multiModelHarness.d.ts

@@ -0,0 +1,27 @@
+import { MultiMediaContentPart } from "@/memory/rendering";
+import { ModelHarnessEvents } from "./modelHarness";
+import { LLMClient } from "./types";
+import { ActionDefinition } from "@/actions";
+import { AgentContext } from "./baml_client";
+import { Action } from "@/actions/types";
+import { Image } from '@/memory/image';
+import EventEmitter from "eventemitter3";
+import z from "zod";
+export declare class MultiModelHarness {
+    /**
+     * Delegates model responsibilites to different LLMs and consolidates their usage
+     */
+    private roles;
+    private uniqueModels;
+    readonly events: EventEmitter<ModelHarnessEvents>;
+    constructor(clients: LLMClient[]);
+    setup(): Promise<void>;
+    describe(): string;
+    partialAct<T>(context: AgentContext, task: string, data: MultiMediaContentPart[], actionVocabulary: ActionDefinition<T>[]): Promise<{
+        reasoning: string;
+        actions: Action[];
+    }>;
+    extract<T extends z.Schema>(instructions: string, schema: T, screenshot: Image, domContent: string): Promise<z.infer<T>>;
+    query<T extends z.Schema>(context: AgentContext, query: string, schema: T): Promise<z.infer<T>>;
+    get numUniqueModels(): number;
+}

package/dist/ai/multiModelHarness.js

@@ -0,0 +1,52 @@
+import { ModelHarness } from "./modelHarness";
+import { allBrowserAgentRoles } from "./types";
+import EventEmitter from "eventemitter3";
+export class MultiModelHarness {
+    /**
+     * Delegates model responsibilites to different LLMs and consolidates their usage
+     */
+    // Roles may reference the same harness
+    roles = {};
+    uniqueModels = [];
+    events = new EventEmitter();
+    constructor(clients) {
+        // Sort by specificity (from least specific to most specific)
+        const sortedClients = clients.toSorted((a, b) => (b.roles ? b.roles.length : 9999) - (a.roles ? a.roles.length : 9999));
+        for (const client of sortedClients) {
+            const harness = new ModelHarness({ llm: client });
+            this.uniqueModels.push(harness);
+            if (client.roles) {
+                for (const role of client.roles) {
+                    this.roles[role] = harness;
+                }
+            }
+            else {
+                for (const role of allBrowserAgentRoles) {
+                    this.roles[role] = harness;
+                }
+            }
+            // Forward token usage events upward
+            harness.events.on('tokensUsed', (usage) => { this.events.emit('tokensUsed', usage); }, this);
+        }
+    }
+    async setup() {
+        await Promise.all(this.uniqueModels.map(model => model.setup()));
+    }
+    describe() {
+        // for now - describe least specific model
+        return this.uniqueModels[0].describeModel();
+    }
+    // TODO: generalize responsibility delegation
+    async partialAct(context, task, data, actionVocabulary) {
+        return await this.roles['act'].partialAct(context, task, data, actionVocabulary);
+    }
+    async extract(instructions, schema, screenshot, domContent) {
+        return await this.roles['extract'].extract(instructions, schema, screenshot, domContent);
+    }
+    async query(context, query, schema) {
+        return await this.roles['query'].query(context, query, schema);
+    }
+    get numUniqueModels() {
+        return this.uniqueModels.length;
+    }
+}