@ddwang/magnitude-core 0.3.1-ddwang.1

package/dist/actions/webActions.d.ts

@@ -0,0 +1,79 @@
+import { ActionDefinition } from ".";
+export declare const clickCoordAction: ActionDefinition<{
+    x: number;
+    y: number;
+}>;
+export declare const mouseDoubleClickAction: ActionDefinition<{
+    x: number;
+    y: number;
+}>;
+export declare const mouseRightClickAction: ActionDefinition<{
+    x: number;
+    y: number;
+}>;
+export declare const mouseDragAction: ActionDefinition<{
+    from: {
+        x: number;
+        y: number;
+    };
+    to: {
+        x: number;
+        y: number;
+    };
+}>;
+export declare const typeAction: ActionDefinition<{
+    content: string;
+}>;
+export declare const keyboardEnterAction: ActionDefinition<any>;
+export declare const keyboardTabAction: ActionDefinition<any>;
+export declare const keyboardBackspaceAction: ActionDefinition<any>;
+export declare const keyboardSelectAllAction: ActionDefinition<any>;
+export declare const scrollCoordAction: ActionDefinition<{
+    x: number;
+    y: number;
+    deltaX: number;
+    deltaY: number;
+}>;
+export declare const switchTabAction: ActionDefinition<{
+    index: number;
+}>;
+export declare const newTabAction: ActionDefinition<{}>;
+export declare const navigateAction: ActionDefinition<{
+    url: string;
+}>;
+export declare const goBackAction: ActionDefinition<{}>;
+export declare const waitAction: ActionDefinition<{
+    seconds: number;
+}>;
+export declare const webActions: readonly [ActionDefinition<{
+    x: number;
+    y: number;
+}>, ActionDefinition<{
+    x: number;
+    y: number;
+}>, ActionDefinition<{
+    x: number;
+    y: number;
+}>, ActionDefinition<{
+    x: number;
+    y: number;
+    deltaX: number;
+    deltaY: number;
+}>, ActionDefinition<{
+    from: {
+        x: number;
+        y: number;
+    };
+    to: {
+        x: number;
+        y: number;
+    };
+}>, ActionDefinition<{}>, ActionDefinition<{
+    index: number;
+}>, ActionDefinition<{
+    url: string;
+}>, ActionDefinition<{
+    content: string;
+}>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<{
+    seconds: number;
+}>];

package/dist/actions/webActions.js

@@ -0,0 +1,190 @@
+import { createAction } from ".";
+import { z } from "zod";
+import { BrowserConnector } from "@/connectors/browserConnector"; // Changed from WebInteractionFacet
+export const clickCoordAction = createAction({
+    name: 'mouse:click',
+    description: "Click something",
+    schema: z.object({
+        x: z.number().int(),
+        y: z.number().int(),
+    }),
+    resolver: async ({ input: { x, y }, agent }) => {
+        const web = agent.require(BrowserConnector);
+        const harness = web.getHarness();
+        await harness.click({ x, y });
+    },
+    render: ({ x, y }) => `⊙ click (${x}, ${y})`
+});
+export const mouseDoubleClickAction = createAction({
+    name: 'mouse:double_click',
+    schema: z.object({
+        x: z.number().int(),
+        y: z.number().int(),
+    }),
+    resolver: async ({ input: { x, y }, agent }) => {
+        const web = agent.require(BrowserConnector);
+        const harness = web.getHarness();
+        await harness.doubleClick({ x, y });
+    },
+    render: ({ x, y }) => `⊙ double click (${x}, ${y})`
+});
+export const mouseRightClickAction = createAction({
+    name: 'mouse:right_click',
+    schema: z.object({
+        x: z.number().int(),
+        y: z.number().int(),
+    }),
+    resolver: async ({ input: { x, y }, agent }) => {
+        await agent.require(BrowserConnector).getHarness().rightClick({ x, y });
+    },
+    render: ({ x, y }) => `⊙ right click (${x}, ${y})`
+});
+export const mouseDragAction = createAction({
+    name: 'mouse:drag',
+    description: "Click and hold mouse in one location and release in another",
+    schema: z.object({
+        from: z.object({ x: z.number().int(), y: z.number().int() }),
+        to: z.object({ x: z.number().int(), y: z.number().int() })
+    }),
+    resolver: async ({ input: { from, to }, agent }) => {
+        const web = agent.require(BrowserConnector);
+        const harness = web.getHarness();
+        await harness.drag({ x1: from.x, y1: from.y, x2: to.x, y2: to.y });
+    },
+    render: ({ from, to }) => `⤡ drag (${from.x}, ${from.y}) -> (${to.x}, ${to.y})`
+});
+export const typeAction = createAction({
+    name: 'keyboard:type',
+    description: "Make sure to click where you need to type first", // make sure you click into it first
+    schema: z.object({
+        content: z.string().describe("Content to type"),
+    }),
+    resolver: async ({ input: { content }, agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.type({ content });
+    },
+    render: ({ content }) => `⌨︎ type "${content}"`
+});
+export const keyboardEnterAction = createAction({
+    name: 'keyboard:enter',
+    resolver: async ({ agent }) => {
+        await agent.require(BrowserConnector).getHarness().enter();
+    },
+    render: () => `⏎ press enter`
+});
+export const keyboardTabAction = createAction({
+    name: 'keyboard:tab',
+    resolver: async ({ agent }) => {
+        await agent.require(BrowserConnector).getHarness().tab();
+    },
+    render: () => `⇥ press tab`
+});
+export const keyboardBackspaceAction = createAction({
+    name: 'keyboard:backspace',
+    resolver: async ({ agent }) => {
+        await agent.require(BrowserConnector).getHarness().backspace();
+    },
+    render: () => `⌫ press backspace`
+});
+export const keyboardSelectAllAction = createAction({
+    name: 'keyboard:select_all',
+    description: "Select all content in the active text area (CTRL+A)",
+    resolver: async ({ input: { content }, agent }) => {
+        await agent.require(BrowserConnector).getHarness().selectAll();
+    },
+    render: () => `⬚ select all`
+});
+export const scrollCoordAction = createAction({
+    name: 'mouse:scroll',
+    description: "Hover mouse over target and scroll",
+    schema: z.object({
+        x: z.number().int(),
+        y: z.number().int(),
+        deltaX: z.number().int().describe("Pixels to scroll horizontally"),
+        deltaY: z.number().int().describe("Pixels to scroll vertically"),
+    }),
+    resolver: async ({ input: { x, y, deltaX, deltaY }, agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.scroll({ x, y, deltaX, deltaY });
+    },
+    render: ({ x, y, deltaX, deltaY }) => `↕ scroll (${deltaX}px, ${deltaY}px)`
+});
+// Grounding agnostic
+export const switchTabAction = createAction({
+    name: 'browser:tab:switch',
+    description: "Switch to a tab that is already open",
+    schema: z.object({
+        index: z.number().int().describe("Index of tab to switch to"),
+    }),
+    resolver: async ({ input: { index }, agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.switchTab({ index });
+    },
+    render: ({ index }) => `⧉ switch to tab ${index}`
+});
+export const newTabAction = createAction({
+    name: 'browser:tab:new',
+    description: "Open and switch to a new tab",
+    schema: z.object({}),
+    resolver: async ({ agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.newTab();
+    },
+    render: () => `⊞ open new tab`
+});
+export const navigateAction = createAction({
+    name: 'browser:nav',
+    description: "Navigate to a URL directly",
+    schema: z.object({
+        url: z.string().describe('URL to navigate to'),
+    }),
+    resolver: async ({ input: { url }, agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.navigate(url);
+    },
+    render: ({ url }) => `⛓︎ navigate to ${url}`
+});
+export const goBackAction = createAction({
+    name: 'browser:nav:back',
+    description: "Go back",
+    schema: z.object({}),
+    resolver: async ({ agent }) => {
+        const webConnector = agent.require(BrowserConnector);
+        const harness = webConnector.getHarness();
+        await harness.goBack();
+    },
+    render: () => `← navigate back`
+});
+// gets overused currently if we include this
+export const waitAction = createAction({
+    name: 'wait',
+    description: "Actions include smart waiting automatically - so only use this when a significant additional wait is clearly required.",
+    schema: z.object({
+        seconds: z.number()
+    }),
+    resolver: async ({ input: { seconds }, agent }) => {
+        await new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+    },
+    render: ({ seconds }) => `◴ wait for ${seconds}s`
+});
+export const webActions = [
+    clickCoordAction,
+    mouseDoubleClickAction,
+    mouseRightClickAction,
+    scrollCoordAction,
+    mouseDragAction,
+    newTabAction,
+    switchTabAction,
+    navigateAction,
+    typeAction,
+    keyboardEnterAction,
+    keyboardTabAction,
+    keyboardBackspaceAction,
+    keyboardSelectAllAction,
+    waitAction,
+];

package/dist/agent/browserAgent.d.ts

@@ -0,0 +1,29 @@
+import { BrowserContext, Page } from "playwright";
+import { Agent, AgentOptions } from ".";
+import { BrowserConnectorOptions } from "@/connectors/browserConnector";
+import { Schema, ZodSchema } from "zod";
+import z from "zod";
+import EventEmitter from "eventemitter3";
+export declare function startBrowserAgent(options?: AgentOptions & BrowserConnectorOptions & {
+    narrate?: boolean;
+}): Promise<BrowserAgent>;
+type ExtractedOutput = string | number | boolean | bigint | Date | null | undefined | {
+    [key: string]: ExtractedOutput;
+} | ExtractedOutput[];
+export interface BrowserAgentEvents {
+    'nav': (url: string) => void;
+    'extractStarted': (instructions: string, schema: ZodSchema) => void;
+    'extractDone': (instructions: string, data: ExtractedOutput) => void;
+}
+export declare class BrowserAgent extends Agent {
+    readonly browserAgentEvents: EventEmitter<BrowserAgentEvents>;
+    constructor({ agentOptions, browserOptions }: {
+        agentOptions?: Partial<AgentOptions>;
+        browserOptions?: BrowserConnectorOptions;
+    });
+    get page(): Page;
+    get context(): BrowserContext;
+    nav(url: string): Promise<void>;
+    extract<T extends Schema>(instructions: string, schema: T): Promise<z.infer<T>>;
+}
+export {};

package/dist/agent/browserAgent.js

@@ -0,0 +1,122 @@
+import { Agent } from ".";
+import { BrowserConnector } from "@/connectors/browserConnector";
+import { buildDefaultBrowserAgentOptions } from "@/ai/util";
+import { narrateBrowserAgent } from "./narrator";
+import { partitionHtml, serializeToMarkdown } from 'magnitude-extract';
+import EventEmitter from "eventemitter3";
+import { retry } from "@/common/retry";
+// export interface StartAgentWithWebOptions {
+//     agentBaseOptions?: Partial<AgentOptions>;
+//     webConnectorOptions?: BrowserConnectorOptions;
+// }
+const DEFAULT_BROWSER_AGENT_TEMP = 0.2;
+// Helper function to start a web agent
+export async function startBrowserAgent(options //StartAgentWithWebOptions = {}
+) {
+    //console.log("sba options:", options);
+    const { agentOptions, browserOptions } = buildDefaultBrowserAgentOptions({ agentOptions: options ?? {}, browserOptions: options ?? {} });
+    const agent = new BrowserAgent({
+        agentOptions: agentOptions,
+        browserOptions: browserOptions,
+    });
+    if (options?.narrate || process.env.MAGNITUDE_NARRATE) {
+        narrateBrowserAgent(agent);
+        //agent.events.on('actionStarted', (action: any) => { console.log(action) })
+    }
+    //console.log('starting agent')
+    await agent.start();
+    //console.log('agent started');
+    return agent;
+}
+async function getFullPageContent(page) {
+    // 1. Get all iframe element handles
+    const iframeHandles = await page.locator('iframe').elementHandles();
+    // 2. Iterate through each iframe handle
+    for (const iframeHandle of iframeHandles) {
+        // 3. Get the Frame object for the iframe
+        const frame = await iframeHandle.contentFrame();
+        if (frame) {
+            // 4. Get the HTML content of the iframe
+            const iframeContent = await frame.content();
+            // 5. Use evaluate to replace the iframe element with its content.
+            // We pass the content as an argument to avoid issues with string escaping.
+            await iframeHandle.evaluate((iframeNode, { content }) => {
+                // Create a new div element to hold the iframe's content
+                const div = document.createElement('div');
+                // Use DOMParser to handle Trusted Types restrictions
+                const parser = new DOMParser();
+                const doc = parser.parseFromString(content, 'text/html');
+                // Move all body children to the div
+                while (doc.body.firstChild) {
+                    div.appendChild(doc.body.firstChild);
+                }
+                // Also preserve any head elements that might be important (styles, etc)
+                const headElements = doc.head.querySelectorAll('style, link[rel="stylesheet"]');
+                headElements.forEach(el => div.appendChild(el.cloneNode(true)));
+                // Add a data-attribute to mark that this was an expanded iframe
+                div.dataset.expandedFromIframe = 'true';
+                div.dataset.iframeSrc = iframeNode.getAttribute('src') || '';
+                // Replace the iframeNode with the new div
+                iframeNode.parentNode?.replaceChild(div, iframeNode);
+            }, { content: iframeContent });
+        }
+    }
+    // 6. Return the final, modified page content
+    return page.content();
+}
+export class BrowserAgent extends Agent {
+    browserAgentEvents = new EventEmitter();
+    constructor({ agentOptions, browserOptions }) {
+        //console.log("agent options:", agent);
+        //console.log("browser options:", browserOptions);
+        super({
+            ...agentOptions,
+            connectors: [new BrowserConnector(browserOptions || {}), ...(agentOptions?.connectors ?? [])]
+        });
+    }
+    get page() {
+        return this.require(BrowserConnector).getHarness().page;
+    }
+    get context() {
+        return this.require(BrowserConnector).getHarness().context;
+    }
+    async nav(url) {
+        this.browserAgentEvents.emit('nav', url);
+        await this.require(BrowserConnector).getHarness().navigate(url);
+    }
+    async extract(instructions, schema) {
+        this.browserAgentEvents.emit('extractStarted', instructions, schema);
+        //const htmlContent = await this.page.content();
+        const htmlContent = await retry(async () => await getFullPageContent(this.page), { retries: 5, delay: 200, exponential: true });
+        // const accessibilityTree = await this.page.accessibility.snapshot({ interestingOnly: true });
+        // const pageRepr = renderMinimalAccessibilityTree(accessibilityTree);
+        const partitionOptions = {
+            extractImages: true,
+            extractForms: true,
+            extractLinks: true,
+            skipNavigation: false, // NAVIGATION SKIPPING IS BROKEN
+            minTextLength: 3,
+            includeOriginalHtml: false,
+            includeMetadata: true
+        };
+        // Process HTML
+        const result = partitionHtml(htmlContent, partitionOptions);
+        // Configure markdown serializer options
+        const markdownOptions = {
+            includeMetadata: false,
+            includePageNumbers: true,
+            includeElementIds: false,
+            includeCoordinates: false,
+            preserveHierarchy: true,
+            escapeSpecialChars: true,
+            includeFormFields: true,
+            includeImageMetadata: true
+        };
+        // Convert to markdown
+        const markdown = serializeToMarkdown(result, markdownOptions);
+        const screenshot = await this.require(BrowserConnector).getHarness().screenshot();
+        const data = await this.models.extract(instructions, schema, screenshot, markdown);
+        this.browserAgentEvents.emit('extractDone', instructions, data);
+        return data;
+    }
+}

package/dist/agent/errors.d.ts

@@ -0,0 +1,8 @@
+export interface AgentErrorOptions {
+    variant?: string;
+    adaptable?: boolean;
+}
+export declare class AgentError extends Error {
+    readonly options: Required<AgentErrorOptions>;
+    constructor(message: string, options?: AgentErrorOptions);
+}

package/dist/agent/errors.js

@@ -0,0 +1,68 @@
+// export class AgentError extends Error {
+//     public readonly failure: FailureDescriptor;
+//     public readonly options: Required<AgentErrorOptions>;
+//     constructor(failure: FailureDescriptor, options: AgentErrorOptions = {}) {
+//         super(`${failure.variant}: ${JSON.stringify(failure.variant, null, 4)}`);
+//         this.options = {
+//             adaptable: options.adaptable ?? false
+//         };
+//         this.failure = failure;
+//     }
+// }
+export class AgentError extends Error {
+    //public readonly failure: FailureDescriptor;
+    options;
+    constructor(message, options = {}) {
+        // super(`${failure.variant}: ${JSON.stringify(failure.variant, null, 4)}`);
+        super(message);
+        this.options = {
+            variant: options.variant ?? 'unknown',
+            adaptable: options.adaptable ?? false,
+        };
+        // if (options.variant !== 'unknown') {
+        //     super(`${failure.variant}: ${JSON.stringify(failure.variant, null, 4)}`);
+        // }
+        //this.options = options;
+        //this.failure = failure;
+    }
+}
+// // Agent will only throw these types of errors
+// import { ActionIngredient } from "./recipe/types";
+// import { WebAction } from "./web/types";
+// export class TestCaseError extends Error {};
+// // Base class for wrapping errors
+// class WrapperError extends TestCaseError {
+//     public readonly originalError: Error;
+//     constructor(message: string, originalError: Error) {
+//         super(`${message} - ${originalError.message}`);
+//         this.name = this.constructor.name;
+//         this.originalError = originalError;
+//         // Preserve the stack trace
+//         if (Error.captureStackTrace) {
+//             Error.captureStackTrace(this, this.constructor);
+//         }
+//     }
+// }
+// export class NavigationError extends WrapperError {
+//     public readonly url: string;
+//     constructor(url: string, originalError: Error) {
+//         super(`Failed to navigate to URL: ${url}`, originalError);
+//         this.url = url;
+//     }
+// }
+// export class ActionExecutionError extends WrapperError {
+//     public readonly action: WebAction;
+//     public readonly statusCode?: number;
+//     constructor(action: WebAction, originalError: Error) {
+//         super(`Browser encountered error while executing action: ${action}`, originalError);
+//         this.action = action;
+//     }
+// }
+// export class ActionConversionError extends WrapperError {
+//     public readonly action: ActionIngredient;
+//     public readonly statusCode?: number;
+//     constructor(action: ActionIngredient, originalError: Error) {
+//         super(`Micro failed to convert action (does the target line up with the screenshot?): ${action}`, originalError);
+//         this.action = action;
+//     }
+// }

package/dist/agent/index.d.ts

@@ -0,0 +1,53 @@
+import EventEmitter from "eventemitter3";
+import z from "zod";
+import { Action } from "@/actions/types";
+import { AgentEvents } from "@/common/events";
+import { AgentConnector } from '@/connectors';
+import { RenderableContent } from '@/memory/observation';
+import { LLMClient } from "@/ai/types";
+import { AgentMemory } from "@/memory";
+import { ActionDefinition } from "@/actions";
+import { MultiModelHarness } from '@/ai/multiModelHarness';
+export interface AgentOptions {
+    llm?: LLMClient | LLMClient[];
+    connectors?: AgentConnector[];
+    actions?: ActionDefinition<any>[];
+    prompt?: string | null;
+    telemetry?: boolean;
+}
+export interface ActOptions {
+    prompt?: string;
+    data?: RenderableContent;
+    memory?: AgentMemory;
+}
+export declare class Agent {
+    private options;
+    private connectors;
+    private actions;
+    private memoryOptions;
+    readonly models: MultiModelHarness;
+    readonly events: EventEmitter<AgentEvents>;
+    private doneActing;
+    private _paused;
+    private _pauseResolve;
+    protected latestTaskMemory: AgentMemory;
+    constructor(baseConfig?: Partial<AgentOptions>);
+    getConnector<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C | undefined;
+    require<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C;
+    start(): Promise<void>;
+    identifyAction(action: Action): ActionDefinition<any>;
+    exec(action: Action, memory?: AgentMemory): Promise<void>;
+    protected _recordConnectorObservations(memory: AgentMemory): Promise<void>;
+    get memory(): AgentMemory;
+    act(taskOrSteps: string | string[], options?: ActOptions): Promise<void>;
+    _traceAct(task: string, memory: AgentMemory, options?: ActOptions): Promise<void>;
+    private _buildContext;
+    _act(description: string, memory: AgentMemory, options?: ActOptions): Promise<void>;
+    query<T extends z.Schema>(query: string, schema: T): Promise<z.infer<T>>;
+    queueDone(): Promise<void>;
+    private _waitIfPaused;
+    pause(): void;
+    resume(): void;
+    get paused(): boolean;
+    stop(): Promise<void>;
+}