@ddwang/magnitude-core 0.3.1-ddwang.1 → 0.3.1-ddwang.2

package/dist/actions/desktopActions.js

@@ -1,4 +1,5 @@
 import { createAction } from ".";
+import { operationSleep } from '@/common/operation';
 import { z } from "zod";
 import { DesktopConnector } from "@/connectors/desktopConnector";
 // Mouse actions
@@ -136,7 +137,7 @@ export const desktopWaitAction = createAction({
         seconds: z.number().describe("Seconds to wait"),
     }),
     resolver: async ({ input: { seconds }, agent }) => {
-        await new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+        await operationSleep(seconds * 1000);
     },
     render: ({ seconds }) => `◴ wait ${seconds}s`
 });

package/dist/actions/index.d.ts

@@ -1,24 +1,25 @@
-import { Agent } from "@/agent";
+import type { Agent } from "@/agent";
+import type { AgentMemory } from '@/memory/agentMemory';
 import { RenderableContent } from "@/memory/observation";
 import { z, Schema, ZodTypeAny } from "zod";
+import type { OperationOptions } from '@/common/operation';
+export type ActionContext<T> = {
+    input: T;
+    agent: Agent;
+    memory?: AgentMemory;
+} & OperationOptions;
 export interface ActionDefinition<T> {
     name: string;
     description?: string;
     schema: Schema<T>;
-    resolver: ({ input, agent }: {
-        input: T;
-        agent: Agent;
-    }) => Promise<void | RenderableContent>;
+    resolver: (context: ActionContext<T>) => Promise<void | RenderableContent>;
     render: (action: T) => string;
 }
 export declare function createAction<S extends ZodTypeAny>(action: {
     name: string;
     description?: string;
     schema?: S;
-    resolver: ({ input, agent }: {
-        input: z.infer<S>;
-        agent: Agent;
-    }) => Promise<void | RenderableContent>;
+    resolver: (context: ActionContext<z.infer<S>>) => Promise<void | RenderableContent>;
     render?: (action: z.infer<S>) => string;
 }): ActionDefinition<z.infer<S>>;
 export type ActionPayload<A extends ActionDefinition<any>> = {

package/dist/actions/memoryActions.d.ts

@@ -0,0 +1,9 @@
+export declare const memoryActions: (import(".").ActionDefinition<{
+    key: string;
+    text: string;
+    sources: number[];
+    operation: "add" | "correct";
+    expected_text: string | null;
+}> | import(".").ActionDefinition<{
+    key: string;
+}>)[];

package/dist/actions/memoryActions.js

@@ -0,0 +1,34 @@
+import { createAction } from '.';
+import { z } from 'zod';
+import { NOTEBOOK_LIMITS, noteUpdateSchema } from '@/memory/notebook';
+export const memoryActions = [
+    createAction({
+        name: 'memory:note',
+        description: 'Add a new record or explicitly correct one existing record before leaving facts needed later. Adds never overwrite. Corrections must match the targeted note\'s current text. Cite supporting observations. This consumes one action without interacting with the browser.',
+        schema: noteUpdateSchema,
+        resolver: async ({ input, agent, memory }) => {
+            try {
+                const { operation, expected_text, ...note } = input;
+                if (operation === 'add' ? expected_text !== null : expected_text === null) {
+                    throw new Error('Use expected_text null for add; for correct, copy the targeted note\'s exact current text.');
+                }
+                (memory ?? agent.memory).remember(note, expected_text ?? undefined);
+                return { saved: input.key };
+            }
+            catch (error) {
+                return { saved: false, error: String(error), instruction: 'No notes changed. Review the error and notebook. New records need new keys; corrections need an exact current-text match. Preserve existing facts when addressing source or capacity errors.' };
+            }
+        },
+        render: input => `note: ${input.key}`,
+    }),
+    createAction({
+        name: 'memory:forget',
+        description: 'Remove an obsolete task note after consolidating any facts still needed. This does not delete the captured observation history.',
+        schema: z.object({ key: z.string().min(1).max(NOTEBOOK_LIMITS.key) }),
+        resolver: async ({ input, agent, memory }) => {
+            (memory ?? agent.memory).forget(input.key);
+            return { forgotten: input.key };
+        },
+        render: input => `forget note: ${input.key}`,
+    }),
+];

package/dist/actions/types.d.ts

@@ -7,12 +7,16 @@ export interface Action {
     variant: string;
     [key: string]: any;
 }
-export type ActionIntent = ClickIntent | TypeIntent | ScrollIntent | SwitchTabIntent;
+export type ActionIntent = ClickIntent | HoverIntent | TypeIntent | ScrollIntent | SwitchTabIntent;
 export type Intent = ActionIntent | CheckIntent;
 export interface ClickIntent {
     variant: 'click';
     target: string;
 }
+export interface HoverIntent {
+    variant: 'hover';
+    target: string;
+}
 export interface TypeIntent {
     variant: 'type';
     target: string;

package/dist/actions/util.d.ts

@@ -1,7 +1,7 @@
 import { ActionDefinition } from ".";
 import TypeBuilder from "@/ai/baml_client/type_builder";
 import { FieldType } from "@boundaryml/baml/native";
-import { Schema } from "zod";
+import { Schema, ZodSchema } from "zod";
 export interface JsonSchema {
     type?: string;
     title?: string;
@@ -18,3 +18,4 @@ export interface JsonSchema {
 }
 export declare function convertZodToBaml(tb: TypeBuilder, schema: Schema): FieldType;
 export declare function convertActionDefinitionsToBaml<T>(tb: TypeBuilder, actionVocabulary: ActionDefinition<T>[]): FieldType;
+export declare function actionInputSchema<T>({ name, description, schema }: ActionDefinition<T>): ZodSchema<any>;

package/dist/actions/util.js

@@ -268,29 +268,12 @@ export function convertActionDefinitionsToBaml(tb, actionVocabulary) {
     /**
      * Convert action definitions to BAML TypeBuilder union representing any one of the actions
      */
-    const actionTypes = [];
-    for (const { name, description, schema } of actionVocabulary) {
-        // Need to augment the schema a bit:
-        // (1) if object, unpack and include variant as literal { variant: name, ...others }
-        // (2) if primitive, add as name input { variant: name, input: <primitive> }
-        let wrapperSchema;
-        const baseWrapperSchema = z.object({
-            variant: description ? z.literal(name).describe(description) : z.literal(name)
-        });
-        if (schema instanceof ZodObject) {
-            // Merge the variant field with the existing object schema
-            // The original schema's fields will be at the top level alongside 'variant'.
-            wrapperSchema = baseWrapperSchema.merge(schema);
-        }
-        else {
-            wrapperSchema = baseWrapperSchema.extend({
-                input: schema,
-            });
-        }
-        //if (description) wrapperSchema = wrapperSchema.describe(description);
-        actionTypes.push(convertZodToBaml(tb, wrapperSchema));
-    }
-    return tb.union(actionTypes);
+    return tb.union(actionVocabulary.map(action => convertZodToBaml(tb, actionInputSchema(action))));
+}
+// One wire schema for BAML's prompt and provider-enforced structured output.
+export function actionInputSchema({ name, description, schema }) {
+    const wrapper = z.object({ variant: description ? z.literal(name).describe(description) : z.literal(name) });
+    return schema instanceof ZodObject ? wrapper.merge(schema) : wrapper.extend({ input: schema });
 }
 //const tb = new TypeBuilder();
 //console.log(convertZodToBaml(tb, z.string()))

package/dist/actions/webActions.d.ts

@@ -7,6 +7,10 @@ export declare const mouseDoubleClickAction: ActionDefinition<{
     x: number;
     y: number;
 }>;
+export declare const mouseHoverAction: ActionDefinition<{
+    x: number;
+    y: number;
+}>;
 export declare const mouseRightClickAction: ActionDefinition<{
     x: number;
     y: number;
@@ -28,6 +32,7 @@ export declare const keyboardEnterAction: ActionDefinition<any>;
 export declare const keyboardTabAction: ActionDefinition<any>;
 export declare const keyboardBackspaceAction: ActionDefinition<any>;
 export declare const keyboardSelectAllAction: ActionDefinition<any>;
+export declare const keyboardEscapeAction: ActionDefinition<{}>;
 export declare const scrollCoordAction: ActionDefinition<{
     x: number;
     y: number;
@@ -54,6 +59,9 @@ export declare const webActions: readonly [ActionDefinition<{
 }>, ActionDefinition<{
     x: number;
     y: number;
+}>, ActionDefinition<{
+    x: number;
+    y: number;
 }>, ActionDefinition<{
     x: number;
     y: number;
@@ -74,6 +82,6 @@ export declare const webActions: readonly [ActionDefinition<{
     url: string;
 }>, ActionDefinition<{
     content: string;
-}>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<{
+}>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<any>, ActionDefinition<{}>, ActionDefinition<any>, ActionDefinition<{
     seconds: number;
 }>];

package/dist/actions/webActions.js

@@ -28,6 +28,19 @@ export const mouseDoubleClickAction = createAction({
     },
     render: ({ x, y }) => `⊙ double click (${x}, ${y})`
 });
+export const mouseHoverAction = createAction({
+    name: 'mouse:hover',
+    description: "Hover over an element to reveal tooltips, dropdown menus, or hidden content",
+    schema: z.object({
+        x: z.number().int(),
+        y: z.number().int(),
+    }),
+    resolver: async ({ input: { x, y }, agent }) => {
+        const web = agent.require(BrowserConnector);
+        await web.getHarness().hover({ x, y });
+    },
+    render: ({ x, y }) => `◎ hover (${x}, ${y})`
+});
 export const mouseRightClickAction = createAction({
     name: 'mouse:right_click',
     schema: z.object({
@@ -95,14 +108,23 @@ export const keyboardSelectAllAction = createAction({
     },
     render: () => `⬚ select all`
 });
+export const keyboardEscapeAction = createAction({
+    name: 'keyboard:escape',
+    description: 'Press Escape to dismiss a dismissible dialog, menu, or focused interaction. This does not bypass a subscription or sign-in requirement.',
+    schema: z.object({}),
+    resolver: async ({ agent }) => {
+        await agent.require(BrowserConnector).getHarness().escape();
+    },
+    render: () => '⎋ press Escape'
+});
 export const scrollCoordAction = createAction({
     name: 'mouse:scroll',
-    description: "Hover mouse over target and scroll",
+    description: "Hover over the area to scroll. Distances are in pixels, not wheel ticks. To browse a full page, use about 500-600 pixels per scroll. Reserve small distances for fine adjustments or small scrollable areas.",
     schema: z.object({
         x: z.number().int(),
         y: z.number().int(),
-        deltaX: z.number().int().describe("Pixels to scroll horizontally"),
-        deltaY: z.number().int().describe("Pixels to scroll vertically"),
+        deltaX: z.number().int().describe("Horizontal distance in pixels: positive scrolls right, negative scrolls left. Use 0 for vertical-only scrolling."),
+        deltaY: z.number().int().describe("Vertical distance in pixels: 600 scrolls down, -600 scrolls up. Use about 500-600 to browse a full page; 5 moves only 5 pixels. Use smaller distances for fine adjustments and 0 for horizontal-only scrolling."),
     }),
     resolver: async ({ input: { x, y, deltaX, deltaY }, agent }) => {
         const webConnector = agent.require(BrowserConnector);
@@ -165,10 +187,10 @@ export const waitAction = createAction({
     name: 'wait',
     description: "Actions include smart waiting automatically - so only use this when a significant additional wait is clearly required.",
     schema: z.object({
-        seconds: z.number()
+        seconds: z.number().finite().nonnegative()
     }),
     resolver: async ({ input: { seconds }, agent }) => {
-        await new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+        await agent.require(BrowserConnector).wait(seconds * 1000);
     },
     render: ({ seconds }) => `◴ wait for ${seconds}s`
 });
@@ -176,6 +198,7 @@ export const webActions = [
     clickCoordAction,
     mouseDoubleClickAction,
     mouseRightClickAction,
+    mouseHoverAction,
     scrollCoordAction,
     mouseDragAction,
     newTabAction,
@@ -185,6 +208,7 @@ export const webActions = [
     keyboardEnterAction,
     keyboardTabAction,
     keyboardBackspaceAction,
+    keyboardEscapeAction,
     keyboardSelectAllAction,
     waitAction,
 ];

package/dist/actions/webActions.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/actions/webActions.test.js

@@ -0,0 +1,43 @@
+import { expect, mock, test } from 'bun:test';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+import { keyboardEscapeAction, scrollCoordAction, waitAction, webActions } from './webActions';
+test('Escape is exposed and forwards the Escape command', async () => {
+    const escape = mock(async () => { });
+    const agent = { require: () => ({ getHarness: () => ({ escape }) }) };
+    expect(webActions).toContain(keyboardEscapeAction);
+    await keyboardEscapeAction.resolver({ agent, input: {} });
+    expect(escape).toHaveBeenCalledTimes(1);
+});
+test('wait validates its duration and uses browser cooldown handling', async () => {
+    const wait = mock(async () => { });
+    const agent = { require: () => ({ wait }) };
+    await waitAction.resolver({ agent, input: { seconds: 2 } });
+    expect(wait).toHaveBeenCalledWith(2000);
+    expect(waitAction.schema.safeParse({ seconds: -1 }).success).toBe(false);
+    expect(waitAction.schema.safeParse({ seconds: Infinity }).success).toBe(false);
+});
+test('scroll scale and precision guidance are included in the action schema', () => {
+    const prompt = `${scrollCoordAction.description} ${JSON.stringify(zodToJsonSchema(scrollCoordAction.schema))}`;
+    expect(prompt).toContain('not wheel ticks');
+    expect(prompt).toContain('500-600');
+    expect(prompt).toContain('600 scrolls down, -600 scrolls up');
+    expect(prompt).toContain('positive scrolls right, negative scrolls left');
+    expect(prompt).toContain('fine adjustments');
+});
+for (const deltas of [
+    { deltaX: 0, deltaY: 5 },
+    { deltaX: 0, deltaY: -5 },
+    { deltaX: 0, deltaY: 600 },
+    { deltaX: 0, deltaY: -600 },
+    { deltaX: 600, deltaY: 0 },
+    { deltaX: -5, deltaY: 0 },
+]) {
+    test(`scroll forwards pixel distances unchanged: ${deltas.deltaX}, ${deltas.deltaY}`, async () => {
+        const scroll = mock(async () => { });
+        const agent = { require: () => ({ getHarness: () => ({ scroll }) }) };
+        const input = scrollCoordAction.schema.parse({ x: 120, y: 250, ...deltas });
+        await scrollCoordAction.resolver({ agent, input });
+        expect(scroll).toHaveBeenCalledTimes(1);
+        expect(scroll).toHaveBeenCalledWith({ x: 120, y: 250, ...deltas });
+    });
+}

package/dist/agent/browserAgent.d.ts

@@ -4,6 +4,7 @@ import { BrowserConnectorOptions } from "@/connectors/browserConnector";
 import { Schema, ZodSchema } from "zod";
 import z from "zod";
 import EventEmitter from "eventemitter3";
+import { type OperationOptions } from '@/common/operation';
 export declare function startBrowserAgent(options?: AgentOptions & BrowserConnectorOptions & {
     narrate?: boolean;
 }): Promise<BrowserAgent>;
@@ -23,7 +24,8 @@ export declare class BrowserAgent extends Agent {
     });
     get page(): Page;
     get context(): BrowserContext;
-    nav(url: string): Promise<void>;
-    extract<T extends Schema>(instructions: string, schema: T): Promise<z.infer<T>>;
+    nav(url: string, options?: OperationOptions): Promise<void>;
+    extract<T extends Schema>(instructions: string, schema: T, options?: OperationOptions): Promise<z.infer<T>>;
+    private _extract;
 }
 export {};

package/dist/agent/browserAgent.js

@@ -5,6 +5,7 @@ import { narrateBrowserAgent } from "./narrator";
 import { partitionHtml, serializeToMarkdown } from 'magnitude-extract';
 import EventEmitter from "eventemitter3";
 import { retry } from "@/common/retry";
+import { checkOperation } from '@/common/operation';
 // export interface StartAgentWithWebOptions {
 //     agentBaseOptions?: Partial<AgentOptions>;
 //     webConnectorOptions?: BrowserConnectorOptions;
@@ -29,15 +30,19 @@ export async function startBrowserAgent(options //StartAgentWithWebOptions = {}
     return agent;
 }
 async function getFullPageContent(page) {
+    checkOperation();
     // 1. Get all iframe element handles
     const iframeHandles = await page.locator('iframe').elementHandles();
     // 2. Iterate through each iframe handle
     for (const iframeHandle of iframeHandles) {
+        checkOperation();
         // 3. Get the Frame object for the iframe
         const frame = await iframeHandle.contentFrame();
+        checkOperation();
         if (frame) {
             // 4. Get the HTML content of the iframe
             const iframeContent = await frame.content();
+            checkOperation();
             // 5. Use evaluate to replace the iframe element with its content.
             // We pass the content as an argument to avoid issues with string escaping.
             await iframeHandle.evaluate((iframeNode, { content }) => {
@@ -62,6 +67,7 @@ async function getFullPageContent(page) {
         }
     }
     // 6. Return the final, modified page content
+    checkOperation();
     return page.content();
 }
 export class BrowserAgent extends Agent {
@@ -80,11 +86,17 @@ export class BrowserAgent extends Agent {
     get context() {
         return this.require(BrowserConnector).getHarness().context;
     }
-    async nav(url) {
-        this.browserAgentEvents.emit('nav', url);
-        await this.require(BrowserConnector).getHarness().navigate(url);
+    async nav(url, options = {}) {
+        return this.runOperation(options, async () => {
+            this.browserAgentEvents.emit('nav', url);
+            checkOperation();
+            await this.require(BrowserConnector).getHarness().navigate(url);
+        });
+    }
+    async extract(instructions, schema, options = {}) {
+        return this.runOperation(options, () => this._extract(instructions, schema));
     }
-    async extract(instructions, schema) {
+    async _extract(instructions, schema) {
         this.browserAgentEvents.emit('extractStarted', instructions, schema);
         //const htmlContent = await this.page.content();
         const htmlContent = await retry(async () => await getFullPageContent(this.page), { retries: 5, delay: 200, exponential: true });
@@ -115,7 +127,9 @@ export class BrowserAgent extends Agent {
         // Convert to markdown
         const markdown = serializeToMarkdown(result, markdownOptions);
         const screenshot = await this.require(BrowserConnector).getHarness().screenshot();
+        checkOperation();
         const data = await this.models.extract(instructions, schema, screenshot, markdown);
+        checkOperation();
         this.browserAgentEvents.emit('extractDone', instructions, data);
         return data;
     }

package/dist/agent/errors.d.ts

@@ -6,3 +6,17 @@ export declare class AgentError extends Error {
     readonly options: Required<AgentErrorOptions>;
     constructor(message: string, options?: AgentErrorOptions);
 }
+export declare class ActionLimitError extends AgentError {
+    readonly limit: number;
+    constructor(limit: number);
+}
+export declare class OperationCancelledError extends AgentError {
+    constructor(reason?: unknown);
+}
+export declare class OperationDeadlineError extends AgentError {
+    readonly deadline: number;
+    constructor(deadline: number);
+}
+export declare class AgentBusyError extends AgentError {
+    constructor();
+}

package/dist/agent/errors.js

@@ -26,6 +26,35 @@ export class AgentError extends Error {
         //this.failure = failure;
     }
 }
+export class ActionLimitError extends AgentError {
+    limit;
+    constructor(limit) {
+        super(`Task did not finish within ${limit} actions`, { variant: 'action_limit' });
+        this.limit = limit;
+        this.name = 'ActionLimitError';
+    }
+}
+export class OperationCancelledError extends AgentError {
+    constructor(reason) {
+        super('Operation cancelled', { variant: 'cancelled' });
+        this.name = 'OperationCancelledError';
+        this.cause = reason;
+    }
+}
+export class OperationDeadlineError extends AgentError {
+    deadline;
+    constructor(deadline) {
+        super('Operation deadline exceeded', { variant: 'deadline' });
+        this.deadline = deadline;
+        this.name = 'OperationDeadlineError';
+    }
+}
+export class AgentBusyError extends AgentError {
+    constructor() {
+        super('Agent has an active operation; await whenIdle() before reusing it', { variant: 'busy' });
+        this.name = 'AgentBusyError';
+    }
+}
 // // Agent will only throw these types of errors
 // import { ActionIngredient } from "./recipe/types";
 // import { WebAction } from "./web/types";

package/dist/agent/index.d.ts

@@ -5,7 +5,8 @@ import { AgentEvents } from "@/common/events";
 import { AgentConnector } from '@/connectors';
 import { RenderableContent } from '@/memory/observation';
 import { LLMClient } from "@/ai/types";
-import { AgentMemory } from "@/memory";
+import { type OperationOptions } from '@/common/operation';
+import { AgentMemory, MemoryRenderOptions } from "@/memory";
 import { ActionDefinition } from "@/actions";
 import { MultiModelHarness } from '@/ai/multiModelHarness';
 export interface AgentOptions {
@@ -14,8 +15,9 @@ export interface AgentOptions {
     actions?: ActionDefinition<any>[];
     prompt?: string | null;
     telemetry?: boolean;
+    maxActions?: number;
 }
-export interface ActOptions {
+export interface ActOptions extends OperationOptions {
     prompt?: string;
     data?: RenderableContent;
     memory?: AgentMemory;
@@ -30,20 +32,29 @@ export declare class Agent {
     private doneActing;
     private _paused;
     private _pauseResolve;
+    private activeOperation?;
+    private idle;
+    private stopped;
     protected latestTaskMemory: AgentMemory;
     constructor(baseConfig?: Partial<AgentOptions>);
     getConnector<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C | undefined;
     require<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C;
     start(): Promise<void>;
     identifyAction(action: Action): ActionDefinition<any>;
-    exec(action: Action, memory?: AgentMemory): Promise<void>;
+    /** True until underlying work settles, including after a cancelled caller returns. */
+    get busy(): boolean;
+    whenIdle(): Promise<void>;
+    protected runOperation<T>(options: OperationOptions, fn: () => Promise<T>): Promise<T>;
+    exec(action: Action, memory?: AgentMemory, options?: OperationOptions): Promise<unknown>;
+    private _exec;
     protected _recordConnectorObservations(memory: AgentMemory): Promise<void>;
     get memory(): AgentMemory;
     act(taskOrSteps: string | string[], options?: ActOptions): Promise<void>;
-    _traceAct(task: string, memory: AgentMemory, options?: ActOptions): Promise<void>;
+    private _runAct;
+    private _traceAct;
     private _buildContext;
-    _act(description: string, memory: AgentMemory, options?: ActOptions): Promise<void>;
-    query<T extends z.Schema>(query: string, schema: T): Promise<z.infer<T>>;
+    private _act;
+    query<T extends z.Schema>(query: string, schema: T, options?: MemoryRenderOptions & OperationOptions): Promise<z.infer<T>>;
     queueDone(): Promise<void>;
     private _waitIfPaused;
     pause(): void;