@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/ai/types.d.ts

@@ -0,0 +1,90 @@
+export type BrowserAgentRole = 'act' | 'extract' | 'query';
+export declare const allBrowserAgentRoles: BrowserAgentRole[];
+export type LLMClient = (AnthropicClient | ClaudeCodeClient | BedrockClient | GoogleAIClient | GoogleVertexClient | OpenAIClient | OpenAIGenericClient | AzureOpenAIClient) & {
+    roles?: BrowserAgentRole[];
+};
+export interface AnthropicClient {
+    provider: 'anthropic';
+    options: {
+        model: string;
+        apiKey?: string;
+        temperature?: number;
+        promptCaching?: boolean;
+    };
+}
+export interface ClaudeCodeClient {
+    provider: 'claude-code';
+    options: {
+        model: string;
+        temperature?: number;
+        promptCaching?: boolean;
+    };
+}
+export interface BedrockClient {
+    provider: 'aws-bedrock';
+    options: {
+        model: string;
+        temperature?: number;
+    };
+}
+export interface GoogleAIClient {
+    provider: 'google-ai';
+    options: {
+        model: string;
+        apiKey?: string;
+        temperature?: number;
+        baseUrl?: string;
+    };
+}
+export interface GoogleVertexClient {
+    provider: 'vertex-ai';
+    options: {
+        model: string;
+        location?: string;
+        baseUrl?: string;
+        projectId?: string;
+        credentials?: string | object;
+        anthropicVersion?: string;
+        temperature?: number;
+    };
+}
+export interface OpenAIClient {
+    provider: 'openai';
+    options: {
+        model: string;
+        apiKey?: string;
+        temperature?: number;
+    };
+}
+export interface AzureOpenAIClient {
+    provider: 'azure-openai';
+    options: {
+        resourceName: string;
+        deploymentId: string;
+        apiVersion: string;
+        apiKey: string;
+    };
+}
+export interface OpenAIGenericClient {
+    provider: 'openai-generic';
+    options: {
+        model: string;
+        baseUrl: string;
+        apiKey?: string;
+        temperature?: number;
+        headers?: Record<string, string>;
+    };
+}
+export interface LLMClientIdentifier {
+    provider: string;
+    model: string;
+}
+export interface ModelUsage {
+    llm: LLMClientIdentifier;
+    inputTokens: number;
+    outputTokens: number;
+    cacheWriteInputTokens?: number;
+    cacheReadInputTokens?: number;
+    inputCost?: number;
+    outputCost?: number;
+}

package/dist/ai/types.js

@@ -0,0 +1,5 @@
+// export interface CheckResult {
+//     passed: boolean,
+//     confidence: number
+// }
+export const allBrowserAgentRoles = ['act', 'extract', 'query'];

package/dist/ai/util.d.ts

@@ -0,0 +1,13 @@
+import { type LLMClient } from '@/ai/types';
+import { AgentOptions } from "@/agent";
+import { BrowserConnectorOptions } from "@/connectors/browserConnector";
+export declare function convertToBamlClientOptions(client: LLMClient): Promise<Record<string, any>>;
+export declare function tryDeriveUIGroundedClient(): LLMClient | null;
+export declare function isClaude(llm: LLMClient): boolean;
+export declare function buildDefaultBrowserAgentOptions({ agentOptions, browserOptions }: {
+    agentOptions: AgentOptions;
+    browserOptions: BrowserConnectorOptions;
+}): {
+    agentOptions: AgentOptions;
+    browserOptions: BrowserConnectorOptions;
+};

package/dist/ai/util.js

@@ -0,0 +1,165 @@
+import { completeClaudeCodeAuthFlow } from './claudeCode';
+function cleanNestedObject(obj) {
+    // Remove null/undefined key values entirely
+    return Object.fromEntries(Object.entries(obj)
+        // Filter out null/undefined values
+        .filter(([_, value]) => value !== null && value !== undefined)
+        // Process nested objects recursively
+        .map(([key, value]) => [
+        key,
+        typeof value === 'object' ? cleanNestedObject(value) : value
+    ]));
+}
+export async function convertToBamlClientOptions(client) {
+    // extract options compatible with https://docs.boundaryml.com/ref/llm-client-providers/overview
+    // Default to temperature 0.0
+    // Some client options (e.g. azure) do not have a temperature setting
+    const temp = 'temperature' in client.options ?
+        (client.options.temperature ?? 0.0) : 0.0;
+    let options;
+    if (client.provider === 'claude-code') {
+        // Special case - oauth with claude code max anthropic account
+        const oauthToken = await completeClaudeCodeAuthFlow();
+        options = {
+            model: client.options.model,
+            temperature: temp,
+            headers: {
+                'Authorization': `Bearer ${oauthToken}`,
+                'anthropic-beta': 'oauth-2025-04-20' + (client.options.promptCaching ? ',prompt-caching-2024-07-31' : ''),
+                // Overrides this header from being automatically derived from ANTHROPIC_API_KEY
+                'X-API-Key': ''
+            },
+            ...(client.options.promptCaching ? { allowed_role_metadata: "all" } : {}),
+        };
+    }
+    else if (client.provider === 'anthropic') {
+        options = {
+            api_key: client.options.apiKey,
+            model: client.options.model,
+            temperature: temp,
+            ...(client.options.promptCaching ? {
+                allowed_role_metadata: "all",
+                headers: { 'anthropic-beta': 'prompt-caching-2024-07-31' }
+            } : {}),
+        };
+    }
+    else if (client.provider === 'aws-bedrock') {
+        options = {
+            model_id: client.options.model,
+            inference_configuration: {
+                temperature: temp,
+            }
+        };
+    }
+    else if (client.provider === 'google-ai') {
+        options = {
+            base_url: client.options.baseUrl,
+            model: client.options.model,
+            api_key: client.options.apiKey,
+            generationConfig: {
+                temperature: temp,
+                //thinking_budget: 0
+            }
+        };
+    }
+    else if (client.provider === 'vertex-ai') {
+        options = {
+            location: client.options.location,
+            base_url: client.options.baseUrl,
+            project_id: client.options.projectId,
+            credentials: client.options.credentials,
+            anthropic_version: client.options.anthropicVersion,
+            model: client.options.model,
+            // generationConfig: {
+            //     temperature: temp,
+            // }
+        };
+    }
+    else if (client.provider === 'openai') {
+        options = {
+            api_key: client.options.apiKey,
+            model: client.options.model,
+            temperature: temp,
+        };
+    }
+    else if (client.provider === 'openai-generic') {
+        options = {
+            base_url: client.options.baseUrl,
+            api_key: client.options.apiKey,
+            model: client.options.model,
+            temperature: temp,
+            headers: {
+                "HTTP-Referer": "https://magnitude.run",
+                "X-Title": "Magnitude",
+                ...client.options.headers
+            }
+        };
+    }
+    else if (client.provider === 'azure-openai') {
+        options = {
+            resource_name: client.options.resourceName,
+            deployment_id: client.options.deploymentId,
+            api_version: client.options.apiVersion,
+            api_key: client.options.apiKey
+        };
+    }
+    else {
+        throw new Error(`Invalid provider: ${client.provider}`);
+    }
+    return cleanNestedObject(options);
+}
+export function tryDeriveUIGroundedClient() {
+    if (process.env.ANTHROPIC_API_KEY) {
+        return {
+            provider: 'anthropic',
+            options: {
+                // TODO: do more testing on best claude model for visuals
+                // model: 'claude-3-5-sonnet-20240620', // <- definitely not, pre computer use
+                // model: 'claude-3-5-sonnet-20241022', // <- not great on rescaling res
+                //model: 'claude-3-7-sonnet-latest', // <- underplans
+                // model: 'claude-sonnet-4-20250514', // <- underplans, also supposedly worse at visual reasoning
+                model: 'claude-haiku-4-5-20251001', // <- fast, cost-effective, good performance
+                apiKey: process.env.ANTHROPIC_API_KEY
+            }
+        };
+    }
+    else {
+        return null;
+    }
+}
+export function isClaude(llm) {
+    if ('model' in llm.options) { //if (llm.provider === 'anthropic' || llm.provider === 'aws-bedrock' || llm.provider === 'vertex-ai') {
+        const model = llm.options.model;
+        if (model.includes('claude'))
+            return true;
+    }
+    return false;
+}
+const DEFAULT_BROWSER_AGENT_TEMP = 0.2;
+export function buildDefaultBrowserAgentOptions({ agentOptions, browserOptions }) {
+    /**
+     * Given any provided options for agent or browser connector, fill out additional key fields using environment,
+     * or any model-specific constraints (e.g. Claude needing 1024x768 virtual screen space)
+     */
+    const envLlm = tryDeriveUIGroundedClient();
+    let llms = agentOptions.llm ? (Array.isArray(agentOptions.llm) ? agentOptions.llm : [agentOptions.llm]) : (envLlm ? [envLlm] : []);
+    if (llms.length == 0) {
+        throw new Error("No LLM configured or available from environment. Set environment variable ANTHROPIC_API_KEY and try again. See https://docs.magnitude.run/customizing/llm-configuration for details");
+    }
+    // Set reasonable temp if not provided
+    let virtualScreenDimensions = null;
+    for (const llm of llms) {
+        let llmOptions = { temperature: DEFAULT_BROWSER_AGENT_TEMP, ...(llm?.options ?? {}) };
+        //let modifiedLlm = {...llm, options: llmOptions as any }
+        llm.options = llmOptions;
+        if (isClaude(llm)) {
+            // Claude only really works on 1024x768 screenshots
+            // if any model is claude, use virtual screen dimensions
+            virtualScreenDimensions = { width: 1024, height: 768 };
+        }
+    }
+    return {
+        agentOptions: { ...agentOptions, llm: llms },
+        browserOptions: { ...browserOptions, virtualScreenDimensions: virtualScreenDimensions ?? undefined }
+    };
+}

package/dist/cli.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+'use strict';
+

package/dist/common/actions.js

@@ -0,0 +1,14 @@
+"use strict";
+// import { ClickWebAction, NavigateWebAction, ScrollWebAction, SwitchTabWebAction, TypeWebAction } from "@/web/types";
+// import { ClickIntent, TypeIntent, ScrollIntent, SwitchTabIntent, Action } from "@/actions/types";
+// //'load' | 'click' | 'hover' | 'type' | 'scroll' | 'wait' | 'back';
+// // move somewhere else, to action types, why is this here
+// export type ActionDescriptor = Action & { screenshot: string };
+// // export type ActionDescriptor = NavigateActionDescriptor | ClickActionDescriptor | TypeActionDescriptor | ScrollActionDescriptor | SwitchTabDescriptor;
+// // export type ActionVariant = ActionDescriptor['variant'];
+// // export type NavigateActionDescriptor = NavigateWebAction & { screenshot: string };
+// // export type ClickActionDescriptor = ClickIntent & ClickWebAction & { screenshot: string };
+// // export type TypeActionDescriptor = TypeIntent & TypeWebAction & { screenshot: string };
+// // export type ScrollActionDescriptor = ScrollIntent & ScrollWebAction & { screenshot: string };
+// // export type SwitchTabDescriptor = SwitchTabIntent & SwitchTabWebAction & { screenshot: string };
+// //const a: ClickActionDescriptor = { variant: 'click', x: 1, y: 2, target: ''}

package/dist/common/events.d.ts

@@ -0,0 +1,15 @@
+import { Action } from "@/actions/types";
+import { ActOptions } from "@/agent";
+import { ModelUsage } from "@/ai/types";
+export interface AgentEvents {
+    'start': () => void;
+    'stop': () => void;
+    'thought': (thought: string) => void;
+    'actStarted': (task: string, options: ActOptions) => void;
+    'actDone': (task: string, options: ActOptions) => void;
+    'actionStarted': (action: Action) => void;
+    'actionDone': (action: Action) => void;
+    'pause': () => void;
+    'resume': () => void;
+    'tokensUsed': (usage: ModelUsage) => void;
+}

package/dist/common/events.js

@@ -0,0 +1,3 @@
+// Common interfaces that can be used anywhere
+// Goal is not to expose internals but provide all necessary info in events
+export {};

package/dist/common/failure.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Represents any reason why a test case could have failed, for example:
+ * - Step could not be completed
+ * - Check did not pass
+ * - Could not navigate to starting URL
+ * - Time or action based timeout
+ * - Operation cancelled by signal
+ * - ...
+ */
+export type FailureDescriptor = BugDetectedFailure | MisalignmentFailure | NetworkFailure | BrowserFailure | RateLimitFailure | ApiKeyFailure | UnknownFailure | CancelledFailure;
+export type BugSeverity = 'critical' | 'high' | 'medium' | 'low';
+/**
+ * Step and check failures are classified into one of:
+ * BugDetectedFailure: seems to be something wrong with the application itself
+ * MisalignmentFailure: seems to be a discrepency between the test case and the interface
+ *    OR the agent did not properly recognize the relationship between test case and interface
+ */
+export interface BugDetectedFailure {
+    variant: 'bug';
+    title: string;
+    expectedResult: string;
+    actualResult: string;
+    severity: BugSeverity;
+}
+export interface MisalignmentFailure {
+    /**
+     * Major misalignment: when a step/check fails due to:
+     * 1. Poorly written step/check that is completely unrelated to the interface
+     * 2. Or interface has changed so much that step/check no longer applicable
+     * 3. Planner did not do good enough job adjusting recipe for minor misalignment
+     * Misalignment could be due to a poorly written test case OR bad agent behavior.
+     */
+    variant: 'misalignment';
+    message: string;
+}
+export interface NetworkFailure {
+    /**
+     * For example, failure to connect to starting URL, or any other network errors
+     * that would completely prevent the test from executing.
+     */
+    variant: 'network';
+    message: string;
+}
+export interface BrowserFailure {
+    /**
+     * E.g. something goes wrong with playwright interactions, any DOM manipulation, etc.
+     */
+    variant: 'browser';
+    message: string;
+}
+export interface RateLimitFailure {
+    variant: 'rate_limit';
+    message: string;
+}
+export interface ApiKeyFailure {
+    variant: 'api_key';
+    message: string;
+}
+export interface UnknownFailure {
+    variant: 'unknown';
+    message: string;
+}
+export interface CancelledFailure {
+    /**
+     * Operation was cancelled, typically by an AbortSignal from a controlling process (e.g., test runner pool).
+     */
+    variant: 'cancelled';
+}

package/dist/common/failure.js

@@ -0,0 +1 @@
+export {};

package/dist/common/index.d.ts

@@ -0,0 +1,3 @@
+export * from './events';
+export * from './failure';
+export * from './util';

package/dist/common/index.js

@@ -0,0 +1,3 @@
+export * from './events';
+export * from './failure';
+export * from './util';

package/dist/common/retry.d.ts

@@ -0,0 +1,16 @@
+type RetryOptions = {
+    retries?: number;
+    delay?: number;
+    maxDelay?: number;
+    exponential?: boolean | number;
+    throwOnExhaustion?: boolean;
+    retryIf?: (error: Error) => boolean;
+    onRetry?: (error: Error, attempt: number) => void;
+};
+export declare function retry<T>(fn: () => Promise<T>, options: RetryOptions & {
+    throwOnExhaustion: false;
+}): Promise<T | null>;
+export declare function retry<T>(fn: () => Promise<T>, options?: RetryOptions & {
+    throwOnExhaustion?: true;
+}): Promise<T>;
+export {};

package/dist/common/retry.js

@@ -0,0 +1,30 @@
+export async function retry(fn, options = {}) {
+    const { retries = 3, delay = 0, maxDelay = Infinity, exponential = false, throwOnExhaustion = true, // Now defaults to true
+    retryIf = () => true, onRetry } = options;
+    const multiplier = exponential === true ? 2 : exponential || 1;
+    for (let attempt = 0; attempt <= retries; attempt++) {
+        try {
+            return await fn();
+        }
+        catch (error) {
+            if (!(error instanceof Error)) {
+                throw new Error(`Non-Error thrown: ${String(error)}`);
+            }
+            if (attempt === retries || !retryIf(error)) {
+                if (throwOnExhaustion) {
+                    throw error;
+                }
+                return null;
+            }
+            if (onRetry) {
+                onRetry(error, attempt + 1);
+            }
+            if (delay > 0) {
+                const currentDelay = Math.min(delay * Math.pow(multiplier, attempt), maxDelay);
+                await new Promise(resolve => setTimeout(resolve, currentDelay));
+            }
+        }
+    }
+    // Unreachable
+    return null;
+}

package/dist/common/util.d.ts

@@ -0,0 +1,23 @@
+export type RetryMode = {
+    mode: 'retry_on_partial_message';
+    errorSubstrings: string[];
+} | {
+    mode: 'retry_all';
+};
+export type RetryParams = {
+    retryLimit: number;
+    delayMs: number;
+    showWarnOnRetry: boolean;
+};
+export type RetryOptions = RetryMode & RetryParams;
+export declare function retryOnErrorIsSuccess<T>(fnToRetry: () => Promise<T>, retryOptions: RetryMode & Partial<RetryParams>): Promise<boolean>;
+export declare function retryOnError<T>(fnToRetry: () => Promise<T>, retryOptions: RetryMode & Partial<RetryParams>): Promise<T>;
+/**
+ * Performs a deep comparison between two values to determine if they are equivalent.
+ *
+ * @param a The first value to compare.
+ * @param b The second value to compare.
+ * @param cache A WeakMap to handle circular references. Should not be provided by the caller.
+ * @returns `true` if the values are deeply equal, `false` otherwise.
+ */
+export declare function deepEquals(a: any, b: any, cache?: WeakMap<object, WeakSet<object>>): boolean;

package/dist/common/util.js

@@ -0,0 +1,132 @@
+import logger from "@/logger";
+export async function retryOnErrorIsSuccess(fnToRetry, retryOptions) {
+    try {
+        await retryOnError(fnToRetry, retryOptions);
+        return true;
+    }
+    catch (error) {
+        return false;
+    }
+}
+export async function retryOnError(fnToRetry, retryOptions) {
+    let lastError;
+    const options = {
+        ...retryOptions,
+        retryLimit: retryOptions.retryLimit ?? 3,
+        delayMs: retryOptions.delayMs ?? 200,
+        showWarnOnRetry: retryOptions.showWarnOnRetry ?? false,
+        //throw: retryOptions.throw ?? true
+    };
+    if (options.retryLimit < 0) {
+        options.retryLimit = 0;
+    }
+    for (let attempt = 0; attempt <= options.retryLimit; attempt++) {
+        try {
+            return await fnToRetry();
+        }
+        catch (error) {
+            lastError = error;
+            const errorMessage = String(error?.message ?? error);
+            if (options.mode === 'retry_all') {
+                if (options.showWarnOnRetry) {
+                    logger.warn(`Retrying on: ${errorMessage}`);
+                }
+            }
+            else if (options.mode === 'retry_on_partial_message') {
+                const includesSubstring = options.errorSubstrings.some((substring) => errorMessage.includes(substring));
+                if (includesSubstring) {
+                    if (options.showWarnOnRetry) {
+                        logger.warn(`Retrying on: ${errorMessage}`);
+                    }
+                }
+                else {
+                    // Error message does NOT contain the target substring. This error is not retryable.
+                    throw lastError; // Throw this current error immediately.
+                }
+            }
+        }
+        await new Promise((resolve) => setTimeout(resolve, options.delayMs));
+    }
+    throw lastError;
+    //if (options.throw) throw lastError;
+}
+/**
+ * Performs a deep comparison between two values to determine if they are equivalent.
+ *
+ * @param a The first value to compare.
+ * @param b The second value to compare.
+ * @param cache A WeakMap to handle circular references. Should not be provided by the caller.
+ * @returns `true` if the values are deeply equal, `false` otherwise.
+ */
+export function deepEquals(a, b, cache = new WeakMap()) {
+    // 1. Strict equality check (handles primitives, same object references, null, undefined)
+    if (a === b) {
+        return true;
+    }
+    // 2. If a or b is null or not an object, and they weren't ===, they are not equal.
+    //    (This covers cases like comparing a primitive to an object, or one null and one object)
+    if (a == null || typeof a !== 'object' || b == null || typeof b !== 'object') {
+        return false;
+    }
+    // 3. Handle circular references:
+    //    If we've already started comparing 'a' and 'b' in the current path,
+    //    assume they are equal to break the cycle.
+    if (cache.has(a) && cache.get(a).has(b)) {
+        return true;
+    }
+    // Add to cache before recursive calls
+    if (!cache.has(a)) {
+        cache.set(a, new WeakSet());
+    }
+    cache.get(a).add(b);
+    // Symmetrically for b -> a to catch cycles like a.c = b; b.c = a;
+    if (!cache.has(b)) {
+        cache.set(b, new WeakSet());
+    }
+    cache.get(b).add(a);
+    // 4. Handle Dates
+    if (a instanceof Date && b instanceof Date) {
+        return a.getTime() === b.getTime();
+    }
+    // 5. Handle Regular Expressions
+    if (a instanceof RegExp && b instanceof RegExp) {
+        return a.source === b.source && a.flags === b.flags;
+    }
+    // 6. Handle Arrays
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length) {
+            return false;
+        }
+        for (let i = 0; i < a.length; i++) {
+            if (!deepEquals(a[i], b[i], cache)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    // 7. Handle plain Objects
+    //    At this point, 'a' and 'b' are objects but not arrays, dates, or regexps.
+    //    If one is an array and the other an object, they are not equal.
+    if (Array.isArray(a) || Array.isArray(b)) {
+        return false;
+    }
+    const keysA = Object.keys(a);
+    const keysB = Object.keys(b);
+    if (keysA.length !== keysB.length) {
+        return false;
+    }
+    for (const key of keysA) {
+        // Check if key exists in B. `Object.keys` only returns own enumerable properties.
+        // `hasOwnProperty` is a more robust check if b might have non-enumerable properties from its prototype with the same name.
+        // However, since we compare keysA.length and keysB.length, if key is in keysA but not in keysB,
+        // lengths would mismatch or the loop over keysB (if we did that) would find a key in B not in A.
+        // The most crucial part is `Object.prototype.hasOwnProperty.call(b, key)`
+        if (!Object.prototype.hasOwnProperty.call(b, key)) {
+            return false;
+        }
+        if (!deepEquals(a[key], b[key], cache)) {
+            return false;
+        }
+    }
+    return true;
+}

package/dist/connectors/browserConnector.d.ts

@@ -0,0 +1,40 @@
+import { AgentConnector } from ".";
+import { WebHarness } from "@/web/harness";
+import { ActionDefinition } from '@/actions';
+import { BrowserOptions } from "@/web/browserProvider";
+import { TabState } from '@/web/tabs';
+import { Observation } from "@/memory/observation";
+import { Image } from "@/memory/image";
+import { ActionVisualizerOptions } from "@/web/visualizer";
+export interface BrowserConnectorOptions {
+    browser?: BrowserOptions;
+    url?: string;
+    virtualScreenDimensions?: {
+        width: number;
+        height: number;
+    };
+    minScreenshots?: number;
+    visuals?: ActionVisualizerOptions;
+}
+export interface BrowserConnectorStateData {
+    screenshot: Image;
+    tabs: TabState;
+}
+export declare class BrowserConnector implements AgentConnector {
+    readonly id: string;
+    private harness;
+    private options;
+    private browser?;
+    private context;
+    private logger;
+    constructor(options?: BrowserConnectorOptions);
+    onStart(): Promise<void>;
+    onStop(): Promise<void>;
+    getActionSpace(): ActionDefinition<any>[];
+    getHarness(): WebHarness;
+    private captureCurrentState;
+    transformScreenshot(screenshot: Image): Promise<Image>;
+    getLastScreenshot(): Promise<Image>;
+    collectObservations(): Promise<Observation[]>;
+    getInstructions(): Promise<void | string>;
+}