@ddwang/magnitude-core 0.3.1-ddwang.2 → 0.3.1-ddwang.21

package/dist/index.d.cts

@@ -1,8 +1,8 @@
 import EventEmitter from 'eventemitter3';
-import z$1, { z, Schema, ZodTypeAny, ZodSchema } from 'zod';
+import z$1, { z, Schema, ZodSchema } from 'zod';
 export { z } from 'zod';
 import { Image as Image$1 } from '@boundaryml/baml';
-import sharp, { Sharp } from 'sharp';
+import { Sharp, FormatEnum } from 'sharp';
 import { BrowserContext, Page, PageScreenshotOptions, Browser, BrowserContextOptions, LaunchOptions } from 'playwright';
 import { PostHog } from 'posthog-node';
 import pino from 'pino';
@@ -83,6 +83,15 @@ interface CheckIntent {
 
 type BrowserAgentRole = 'act' | 'extract' | 'query';
 declare const allBrowserAgentRoles: BrowserAgentRole[];
+/** Optional request budget for one model call, used for act() planning and for extract(). */
+interface TimeoutPolicy {
+    /** Includes transport retries and format correction within one invocation. */
+    timeoutMs: number;
+    /** Retries after timeout, once the previous invocation settles. Default: 1. */
+    maxRetries?: number;
+}
+/** The published name for the planner's policy, kept for existing imports. */
+type PlannerOptions = TimeoutPolicy;
 type LLMClient = (AnthropicClient | ClaudeCodeClient | BedrockClient | GoogleAIClient | GoogleVertexClient | OpenAIClient | BasetenClient | OpenAIGenericClient | AzureOpenAIClient) & {
     roles?: BrowserAgentRole[];
 };
@@ -201,7 +210,100 @@ interface ModelUsage {
     outputCost?: number;
 }
 
+interface OperationOptions {
+    signal?: AbortSignal;
+    /** Absolute Unix time in milliseconds, shared by every step, retry, and wait. */
+    deadline?: number;
+}
+type OperationKind = 'act' | 'query' | 'extract' | 'nav' | 'exec';
+type OperationPhase = 'preparing' | 'context' | 'observations' | 'model' | 'action' | 'screenshot' | 'stability' | 'cooldown' | 'retry' | 'paused' | 'cursor';
+interface OperationTiming {
+    count: number;
+    totalMs: number;
+}
+interface ProviderAttemptDiagnostics {
+    operationId: string;
+    attempt: number;
+    provider: string;
+    model: string;
+    startedAt: number;
+    elapsedMs: number | null;
+    httpStatus: number | null;
+    requestId: string | null;
+    /** Local planner invocation, when this HTTP attempt belongs to act() planning. */
+    plannerCallId?: string;
+    /** HTTP outcome, not successful parsing or completion of the agent's task. */
+    outcome: 'succeeded' | 'failed' | 'unknown';
+    /** Reported input tokens, including cached input, or null when the response has no usage. */
+    promptTokens: number | null;
+    /** Reported input tokens read from the provider's cache, or null when the response has no usage. */
+    cachedPromptTokens: number | null;
+    /** Reported output tokens, including reasoning, or null when the response has no usage. */
+    outputTokens: number | null;
+}
+interface PlannerCallDiagnostics {
+    id: string;
+    /** One-based timeout attempt within a planning step. */
+    attempt: number;
+    provider: string;
+    model: string;
+    startedAt: number;
+    endedAt?: number;
+    elapsedMs: number;
+    timeoutMs?: number;
+    status: 'running' | 'draining' | 'finished';
+    outcome?: 'succeeded' | 'failed' | 'timeout' | 'cancelled' | 'deadline';
+}
+interface BrowserClickDiagnostics {
+    operationId: string;
+    actionIndex: number | null;
+    x: number;
+    y: number;
+    button: 'left' | 'right' | 'middle';
+    clickCount: number;
+    screenshot: {
+        width: number;
+        height: number;
+    } | null;
+    viewport: {
+        width: number;
+        height: number;
+    } | null;
+    hit: {
+        tag: string | null;
+        role: string | null;
+    } | null;
+}
+interface OperationDiagnostics {
+    id: string;
+    kind: OperationKind;
+    status: 'running' | 'draining' | 'finished';
+    outcome?: 'succeeded' | 'failed' | 'cancelled' | 'deadline';
+    phase: OperationPhase;
+    startedAt: number;
+    elapsedMs: number;
+    cancellationToDrainMs?: number;
+    cancellationToIdleMs?: number;
+    lastAction?: {
+        index: number;
+        name: string;
+        state: 'pending' | 'started' | 'completed' | 'failed';
+    };
+    /** Inclusive totals: nested and concurrent phases overlap and must not be summed. */
+    timings: Partial<Record<OperationPhase, OperationTiming>>;
+    /** Latest 100 SDK attempts, available after each model invocation settles. */
+    providerAttempts?: ProviderAttemptDiagnostics[];
+    providerAttemptsTruncated?: boolean;
+    /** Latest 100 planner invocations, including the currently running request. */
+    plannerCalls?: PlannerCallDiagnostics[];
+    plannerCallsTruncated?: boolean;
+    /** Pre-dispatch hit evidence for the latest submitted click, not proof of success. */
+    lastClick?: BrowserClickDiagnostics;
+}
+
 interface AgentEvents {
+    /** Payload-free snapshots. Existing action/thought events retain their original payloads. */
+    'operation': (diagnostics: OperationDiagnostics) => void;
     'start': () => void;
     'stop': () => void;
     'thought': (thought: string) => void;
@@ -266,7 +368,7 @@ declare class Image {
     private encoded?;
     constructor(img: Sharp);
     static fromBase64(base64: string): Image;
-    getFormat(): Promise<keyof sharp.FormatEnum>;
+    getFormat(): Promise<keyof FormatEnum>;
     private encode;
     /**
      * Convert the image to a JSON representation
@@ -295,6 +397,8 @@ interface ObservationRetentionOptions {
     type: string;
     limit?: number;
     dedupe?: boolean;
+    /** Render only the latest of this type after cached history; overrides limit and dedupe. Full audit retains every entry. */
+    current?: boolean;
 }
 type ObservationSource = `connector:${string}` | `action:taken:${string}` | `action:result:${string}` | `thought`;
 declare class Observation {
@@ -322,37 +426,19 @@ declare class Observation {
 declare const noteSchema: z.ZodObject<{
     key: z.ZodString;
     text: z.ZodString;
-    sources: z.ZodArray<z.ZodNumber, "many">;
-}, "strict", z.ZodTypeAny, {
-    key: string;
-    text: string;
-    sources: number[];
-}, {
-    key: string;
-    text: string;
-    sources: number[];
-}>;
+    sources: z.ZodArray<z.ZodNumber>;
+}, z.core.$strict>;
 type NoteInput = z.infer<typeof noteSchema>;
 declare const noteUpdateSchema: z.ZodObject<{
     key: z.ZodString;
     text: z.ZodString;
-    sources: z.ZodArray<z.ZodNumber, "many">;
-} & {
-    operation: z.ZodEnum<["add", "correct"]>;
+    sources: z.ZodArray<z.ZodNumber>;
+    operation: z.ZodEnum<{
+        add: "add";
+        correct: "correct";
+    }>;
     expected_text: z.ZodNullable<z.ZodString>;
-}, "strict", z.ZodTypeAny, {
-    key: string;
-    text: string;
-    sources: number[];
-    operation: "add" | "correct";
-    expected_text: string | null;
-}, {
-    key: string;
-    text: string;
-    sources: number[];
-    operation: "add" | "correct";
-    expected_text: string | null;
-}>;
+}, z.core.$strict>;
 type NoteUpdate = z.infer<typeof noteUpdateSchema>;
 
 interface SerializedAgentMemory {
@@ -384,6 +470,8 @@ declare class AgentMemory {
     private cacheControlIndices;
     constructor(options?: AgentMemoryOptions);
     get instructions(): string | null;
+    /** Apply current instructions and provider caching policy without replacing task evidence. */
+    configure(options: Pick<AgentMemoryOptions, 'instructions' | 'promptCaching'>): void;
     render(options?: MemoryRenderOptions): Promise<MultiMediaMessage[]>;
     simpleRender(): Promise<(Image$1 | string)[]>;
     private observationPrefix;
@@ -395,15 +483,10 @@ declare class AgentMemory {
     recordObservation(obs: Observation): void;
     getLastThoughtMessage(): string | null;
     toJSON(): Promise<SerializedAgentMemory>;
+    /** Replace saved state atomically; runtime caching and thought limits stay with this instance. */
     loadJSON(data: SerializedAgentMemory): Promise<void>;
 }
 
-interface OperationOptions {
-    signal?: AbortSignal;
-    /** Absolute Unix time in milliseconds, shared by every step, retry, and wait. */
-    deadline?: number;
-}
-
 type ActionContext<T> = {
     input: T;
     agent: Agent;
@@ -416,13 +499,14 @@ interface ActionDefinition<T> {
     resolver: (context: ActionContext<T>) => Promise<void | RenderableContent>;
     render: (action: T) => string;
 }
-declare function createAction<S extends ZodTypeAny>(action: {
-    name: string;
-    description?: string;
-    schema?: S;
-    resolver: (context: ActionContext<z.infer<S>>) => Promise<void | RenderableContent>;
-    render?: (action: z.infer<S>) => string;
-}): ActionDefinition<z.infer<S>>;
+type ActionOptions<T> = Omit<ActionDefinition<T>, 'render'> & {
+    render?: ActionDefinition<T>['render'];
+};
+type EmptyActionOptions = Omit<ActionOptions<Record<string, never>>, 'schema'> & {
+    schema?: undefined;
+};
+declare function createAction<T>(action: ActionOptions<T>): ActionDefinition<T>;
+declare function createAction(action: EmptyActionOptions): ActionDefinition<Record<string, never>>;
 type ActionPayload<A extends ActionDefinition<any>> = {
     name: A['name'];
 } & z.infer<A['schema']>;
@@ -433,6 +517,8 @@ interface AgentConnector {
     onStop?(): Promise<void>;
     beforeAction?(action: Action, options?: OperationOptions): Promise<void>;
     onTaskStart?(options?: OperationOptions): void;
+    /** Called before each planner request, after the observations it will see are recorded. */
+    beforePlan?(options?: OperationOptions): Promise<void>;
     getActionSpace?(): ActionDefinition<any>[];
     collectObservations?(options?: OperationOptions): Promise<Observation[]>;
     getInstructions?(options?: OperationOptions): Promise<void | string>;
@@ -457,7 +543,10 @@ declare class MultiModelHarness {
     private roles;
     private uniqueModels;
     readonly events: EventEmitter<ModelHarnessEvents>;
-    constructor(clients: LLMClient[]);
+    constructor(clients: LLMClient[], policies?: {
+        planner?: TimeoutPolicy;
+        extract?: TimeoutPolicy;
+    });
     setup(): Promise<void>;
     describe(): string;
     partialAct<T>(context: AgentContext, task: string, data: MultiMediaContentPart[], actionVocabulary: ActionDefinition<T>[]): Promise<PlannerResponse>;
@@ -473,6 +562,10 @@ interface AgentOptions {
     prompt?: string | null;
     telemetry?: boolean;
     maxActions?: number;
+    /** Omitted preserves existing planner retry behavior without a per-call timeout. */
+    planner?: TimeoutPolicy;
+    /** Omitted runs extract() without a per-call timeout. */
+    extract?: TimeoutPolicy;
 }
 interface ActOptions extends OperationOptions {
     prompt?: string;
@@ -490,18 +583,30 @@ declare class Agent {
     private _paused;
     private _pauseResolve;
     private activeOperation?;
+    private latestOperation?;
     private idle;
-    private stopped;
+    private resolveIdle?;
+    private pendingWork;
+    private lifecycleTail;
+    private lifecycleRequest?;
+    private lifecycleState;
+    private telemetryStarted;
     protected latestTaskMemory: AgentMemory;
     constructor(baseConfig?: Partial<AgentOptions>);
     getConnector<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C | undefined;
     require<C extends AgentConnector>(connectorClass: new (...args: any[]) => C): C;
     start(): Promise<void>;
+    get lifecycle(): 'new' | 'starting' | 'ready' | 'stopping' | 'stopped';
+    private beginWork;
+    private endWork;
+    private scheduleLifecycle;
     identifyAction(action: Action): ActionDefinition<any>;
     /** True until underlying work settles, including after a cancelled caller returns. */
     get busy(): boolean;
     whenIdle(): Promise<void>;
-    protected runOperation<T>(options: OperationOptions, fn: () => Promise<T>): Promise<T>;
+    /** A payload-free snapshot of the active or most recent operation. */
+    get operation(): OperationDiagnostics | undefined;
+    protected runOperation<T>(options: OperationOptions, fn: () => Promise<T>, kind?: OperationKind): Promise<T>;
     exec(action: Action, memory?: AgentMemory, options?: OperationOptions): Promise<unknown>;
     private _exec;
     protected _recordConnectorObservations(memory: AgentMemory): Promise<void>;
@@ -518,10 +623,13 @@ declare class Agent {
     resume(): void;
     get paused(): boolean;
     stop(): Promise<void>;
+    private stopConnectors;
 }
 
 interface ActionVisualizerOptions {
     showCursor?: boolean;
+    /** Set false to skip the 300 ms movement animation. Defaults to true. */
+    animateCursor?: boolean;
     showHoverCircle?: boolean;
     showClickRipple?: boolean;
     showDragLine?: boolean;
@@ -557,6 +665,8 @@ interface WebHarnessOptions {
     };
     visuals?: ActionVisualizerOptions;
     switchTabsOnActivity?: boolean;
+    /** Refuse to type into password and one-time-code fields; the host owns authentication. */
+    hostOnlyAuthentication?: boolean;
 }
 interface WebHarnessEvents {
     'activePageChanged': (page: Page) => Promise<void>;
@@ -572,6 +682,7 @@ declare class WebHarness {
     readonly visualizer: ActionVisualizer;
     private transformer;
     private tabs;
+    private lastScreenshot?;
     readonly events: EventEmitter<WebHarnessEvents>;
     constructor(context: BrowserContext, options?: WebHarnessOptions);
     setActivePage(page: Page): Promise<void>;
@@ -594,7 +705,13 @@ declare class WebHarness {
     }, options?: {
         transform: boolean;
     }): Promise<void>;
+    /** Recheck an observed target after hover effects, before submitting any click. */
+    clickGrounded(resolve: () => Promise<{
+        x: number;
+        y: number;
+    } | null>): Promise<boolean>;
     private _click;
+    private dispatchClick;
     hover({ x, y }: {
         x: number;
         y: number;
@@ -621,6 +738,8 @@ declare class WebHarness {
     }, options?: {
         transform: boolean;
     }): Promise<void>;
+    /** Stops the operation when the focused field, in any frame, takes a password or one-time code. */
+    private refuseCredentialField;
     type({ content }: {
         content: string;
     }): Promise<void>;
@@ -686,6 +805,7 @@ interface BrowserBlock {
 }
 declare class BrowserBlockedError extends Error {
     readonly block: BrowserBlock;
+    readonly operation?: OperationDiagnostics;
     constructor(block: BrowserBlock);
 }
 interface HttpDiagnostic {
@@ -697,6 +817,7 @@ interface HttpDiagnostic {
 }
 interface RecoveryOptions {
     maxRateLimitWaitMs?: number;
+    /** Unchanged attempts before stopping; known-state cycles allow twice this many attempts. Default: 6. */
     repeatedActionLimit?: number;
     /** Opt in to heuristic loop warnings and automatic no-progress termination. */
     noProgress?: boolean;
@@ -707,16 +828,18 @@ declare class BrowserRecovery {
     waitUntil?: number;
     private rateWaitMs;
     private blockedActions;
-    private recent;
     private recentStates;
     private repetitions;
+    private knownStateAttempts;
     private lastFingerprint?;
     readonly maxRateLimitWaitMs: number;
     readonly repeatedActionLimit: number;
     readonly noProgress: boolean;
     constructor(options?: RecoveryOptions);
     reset(): void;
-    observe(fingerprint: string, action: Action | undefined, block: BrowserBlock | undefined, now?: number): void;
+    /** New non-page evidence, such as a browser download lifecycle transition. */
+    recordProgress(): void;
+    observe(fingerprint: string | null, action: Action | undefined, block: BrowserBlock | undefined, now?: number): void;
     check(): void;
     waitDuration(requestedMs: number, now?: number, deadline?: number): number;
 }
@@ -731,6 +854,10 @@ interface BrowserConnectorOptions {
     minScreenshots?: number;
     visuals?: ActionVisualizerOptions;
     recovery?: RecoveryOptions | false;
+    /** Opt in to current-viewport link, button, and native field observations with browser:click, browser:select, and browser:fill references. */
+    groundedControls?: boolean;
+    /** The host owns sign-in: typing into a password or one-time-code field stops the task as blocked by authentication. */
+    hostOnlyAuthentication?: boolean;
 }
 interface BrowserConnectorStateData {
     screenshot: Image;
@@ -748,12 +875,15 @@ declare class BrowserConnector implements AgentConnector {
     private responses;
     private pendingAction?;
     private cancelWait?;
+    private downloads?;
+    private controls?;
     constructor(options?: BrowserConnectorOptions);
     onStart(): Promise<void>;
     onStop(): Promise<void>;
     getActionSpace(): ActionDefinition<any>[];
     private onResponse;
     beforeAction(action: Action): Promise<void>;
+    beforePlan(): Promise<void>;
     onTaskStart(): void;
     wait(requestedMs: number): Promise<void>;
     getHarness(): WebHarness;
@@ -785,7 +915,7 @@ declare class BrowserAgent extends Agent {
     get page(): Page;
     get context(): BrowserContext;
     nav(url: string, options?: OperationOptions): Promise<void>;
-    extract<T extends Schema>(instructions: string, schema: T, options?: OperationOptions): Promise<z$1.infer<T>>;
+    extract<T extends Schema<ExtractedOutput>>(instructions: string, schema: T, options?: OperationOptions): Promise<z$1.infer<T>>;
     private _extract;
 }
 
@@ -850,6 +980,7 @@ interface AgentErrorOptions {
     adaptable?: boolean;
 }
 declare class AgentError extends Error {
+    readonly operation?: OperationDiagnostics;
     readonly options: Required<AgentErrorOptions>;
     constructor(message: string, options?: AgentErrorOptions);
 }
@@ -867,6 +998,14 @@ declare class OperationDeadlineError extends AgentError {
 declare class AgentBusyError extends AgentError {
     constructor();
 }
+declare class PlannerTimeoutError extends AgentError {
+    readonly timeoutMs: number;
+    constructor(timeoutMs: number);
+}
+declare class ExtractTimeoutError extends AgentError {
+    readonly timeoutMs: number;
+    constructor(timeoutMs: number);
+}
 
 /**
  * Represents any reason why a test case could have failed, for example:
@@ -1006,5 +1145,4 @@ declare function buildDefaultBrowserAgentOptions({ agentOptions, browserOptions
 
 declare const logger: pino.Logger<never, boolean>;
 
-export { ActionLimitError, Agent, AgentBusyError, AgentError, AgentMemory, BrowserAgent, BrowserBlockedError, BrowserConnector, BrowserProvider, DesktopConnector, Observation, OperationCancelledError, OperationDeadlineError, WebHarness, allBrowserAgentRoles, buildDefaultBrowserAgentOptions, createAction, createId, deepEquals, getCodebaseId, getMachineId, logger, mergeMessages, posthog, retryOnError, retryOnErrorIsSuccess, sendTelemetry, startBrowserAgent };
-export type { ActOptions, Action, ActionContext, ActionDefinition, ActionIntent, ActionPayload, AgentConnector, AgentErrorOptions, AgentEvents, AgentMemoryOptions, AgentOptions, AnthropicClient, ApiKeyFailure, AzureOpenAIClient, Base64Image, BasetenClient, BedrockClient, BlockReason, BrowserAgentRole, BrowserBlock, BrowserConnectorOptions, BrowserConnectorStateData, BrowserFailure, BrowserOptions, BugDetectedFailure, BugSeverity, CancelledFailure, CheckIntent, ClaudeCodeClient, ClickIntent, ClickWebAction, DesktopConnectorOptions, DesktopInterface, FailedTestCaseResult, FailureDescriptor, GoogleAIClient, GoogleVertexClient, HoverIntent, HoverWebAction, HttpDiagnostic, Intent, LLMClient, LLMClientIdentifier, MemoryRenderOptions, MisalignmentFailure, ModelUsage, NavigateWebAction, NetworkFailure, ObservableDataArray, ObservableDataObject, ObservableDataPrimitive, ObservationRetentionOptions, ObservationRole, ObservationSource, OpenAIClient, OpenAIGenericClient, OperationOptions, PixelCoordinate, RateLimitFailure, RecoveryOptions, RenderableContent, RetryMode, RetryOptions, RetryParams, ScrollIntent, ScrollWebAction, SerializedAgentMemory, SuccessfulTestCaseResult, SwitchTabIntent, SwitchTabWebAction, TestCaseDefinition, TestCaseResult, TestData, TestDataEntry, TestStepDefinition, TypeIntent, TypeWebAction, UnknownFailure, WebAction, WebHarnessEvents, WebHarnessOptions };
+export { type ActOptions, type Action, type ActionContext, type ActionDefinition, type ActionIntent, ActionLimitError, type ActionPayload, Agent, AgentBusyError, type AgentConnector, AgentError, type AgentErrorOptions, type AgentEvents, AgentMemory, type AgentMemoryOptions, type AgentOptions, type AnthropicClient, type ApiKeyFailure, type AzureOpenAIClient, type Base64Image, type BasetenClient, type BedrockClient, type BlockReason, BrowserAgent, type BrowserAgentRole, type BrowserBlock, BrowserBlockedError, type BrowserClickDiagnostics, BrowserConnector, type BrowserConnectorOptions, type BrowserConnectorStateData, type BrowserFailure, type BrowserOptions, BrowserProvider, type BugDetectedFailure, type BugSeverity, type CancelledFailure, type CheckIntent, type ClaudeCodeClient, type ClickIntent, type ClickWebAction, DesktopConnector, type DesktopConnectorOptions, type DesktopInterface, ExtractTimeoutError, type FailedTestCaseResult, type FailureDescriptor, type GoogleAIClient, type GoogleVertexClient, type HoverIntent, type HoverWebAction, type HttpDiagnostic, type Intent, type LLMClient, type LLMClientIdentifier, type MemoryRenderOptions, type MisalignmentFailure, type ModelUsage, type NavigateWebAction, type NetworkFailure, type ObservableDataArray, type ObservableDataObject, type ObservableDataPrimitive, Observation, type ObservationRetentionOptions, type ObservationRole, type ObservationSource, type OpenAIClient, type OpenAIGenericClient, OperationCancelledError, OperationDeadlineError, type OperationDiagnostics, type OperationKind, type OperationOptions, type OperationPhase, type OperationTiming, type PixelCoordinate, type PlannerCallDiagnostics, type PlannerOptions, PlannerTimeoutError, type ProviderAttemptDiagnostics, type RateLimitFailure, type RecoveryOptions, type RenderableContent, type RetryMode, type RetryOptions, type RetryParams, type ScrollIntent, type ScrollWebAction, type SerializedAgentMemory, type SuccessfulTestCaseResult, type SwitchTabIntent, type SwitchTabWebAction, type TestCaseDefinition, type TestCaseResult, type TestData, type TestDataEntry, type TestStepDefinition, type TimeoutPolicy, type TypeIntent, type TypeWebAction, type UnknownFailure, type WebAction, WebHarness, type WebHarnessEvents, type WebHarnessOptions, allBrowserAgentRoles, buildDefaultBrowserAgentOptions, createAction, createId, deepEquals, getCodebaseId, getMachineId, logger, mergeMessages, posthog, retryOnError, retryOnErrorIsSuccess, sendTelemetry, startBrowserAgent };