@aiscene/core 9.0.1 → 9.0.3

package/dist/types/agent/agent.d.ts

@@ -1,6 +1,6 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type AIMemory, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@aiscene/shared/env';
@@ -37,6 +37,8 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      */
     private frozenUIContext?;
     private get aiActContext();
+    private aiMemory;
+    resolveAIMemory(): Promise<string[]>;
     /**
      * Flag to track if VL model warning has been shown
      */
@@ -67,6 +69,15 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      */
     setAIActionContext(prompt: string): Promise<void>;
     setAIActContext(prompt: string): Promise<void>;
+    setAIMemory(memory?: AIMemory): void;
+    setMemory(memory?: AIMemory): void;
+    appendAIMemory(memory?: AIMemory): void;
+    appendMemory(memory?: AIMemory): void;
+    addAIMemory(memory?: AIMemory): void;
+    getAIMemory(): string[];
+    getMemory(): string[];
+    clearAIMemory(): void;
+    clearMemory(): void;
     resetDump(): ReportActionDump;
     appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
     dumpDataString(opt?: {

package/dist/types/agent/index.d.ts

@@ -5,5 +5,5 @@ export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, }
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
-export type { AgentOpt } from '../types';
+export type { AIMemory, AIMemoryRetriever, AgentOpt } from '../types';
 export type { AiActOptions } from './agent';

package/dist/types/agent/tasks.d.ts

@@ -27,6 +27,7 @@ export declare class TaskExecutor {
     replanningCycleLimit?: number;
     waitAfterAction?: number;
     useDeviceTime?: boolean;
+    private readonly memoryRetrieverFn?;
     get page(): AbstractInterface;
     constructor(interfaceInstance: AbstractInterface, service: Service, opts: {
         taskCache?: TaskCache;
@@ -34,6 +35,7 @@ export declare class TaskExecutor {
         replanningCycleLimit?: number;
         waitAfterAction?: number;
         useDeviceTime?: boolean;
+        memoryRetrieverFn?: () => readonly string[] | Promise<readonly string[]>;
         hooks?: TaskExecutorHooks;
         actionSpace: DeviceAction[];
     });

package/dist/types/ai-model/conversation-history.d.ts

@@ -1,7 +1,8 @@
-import type { SubGoal } from '../types';
+import type { AIMemory, SubGoal } from '../types';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 export interface ConversationHistoryOptions {
     initialMessages?: ChatCompletionMessageParam[];
+    initialMemories?: AIMemory;
 }
 export declare class ConversationHistory {
     private readonly messages;

package/dist/types/ai-model/inspect.d.ts

@@ -22,6 +22,7 @@ export declare function AiLocateElement(options: {
     targetElementDescription: TUserPrompt;
     searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;
     modelConfig: IModelConfig;
+    memories?: readonly string[];
     abortSignal?: AbortSignal;
 }): Promise<{
     parseResult: {
@@ -37,6 +38,7 @@ export declare function AiLocateSection(options: {
     context: UIContext;
     sectionDescription: TUserPrompt;
     modelConfig: IModelConfig;
+    memories?: readonly string[];
     abortSignal?: AbortSignal;
 }): Promise<{
     rect?: Rect;
@@ -53,6 +55,7 @@ export declare function AiExtractElementInfo<T>(options: {
     pageDescription?: string;
     extractOption?: ServiceExtractOption;
     modelConfig: IModelConfig;
+    memories?: readonly string[];
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
     rawResponse: string;

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,4 +1,4 @@
-import type { SubGoal } from '../../types';
+import type { AIMemory, SubGoal } from '../../types';
 /**
  * Extract content from an XML tag in a string, searching from the end.
  * This approach handles cases where models prepend thinking content (like <think>...</think>)
@@ -23,6 +23,9 @@ export declare function parseSubGoalsFromXML(xmlContent: string): SubGoal[];
  * Extract indexes of sub-goals marked as finished from <mark-sub-goal-done> content
  */
 export declare function parseMarkFinishedIndexes(xmlContent: string): number[];
+export declare function normalizeAIMemory(memory?: AIMemory): string[];
+export declare function formatAIMemories(memories?: readonly string[]): string;
+export declare function appendAIMemoryToText(text: string, memories?: readonly string[]): string;
 export declare const distanceThreshold = 16;
 export declare function distance(point1: {
     x: number;

package/dist/types/index.d.ts

@@ -5,7 +5,7 @@ import { getVersion } from './utils';
 export { plan, AiLocateElement, runConnectivityTest, getMidsceneLocationSchema, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, type ConnectivityCheckResultItem, type ConnectivityTestConfig, type ConnectivityTestResult, } from './ai-model/index';
 export { MIDSCENE_MODEL_NAME, type CreateOpenAIClientFn, } from '@aiscene/shared/env';
 export type * from './types';
-export { ServiceError, ActionAssertionError, isActionAssertionError, ExecutionDump, ReportActionDump, GroupedActionDump, type IExecutionDump, type IReportActionDump, type IGroupedActionDump, type ReportMeta, type GroupMeta, } from './types';
+export { ServiceError, ExecutionDump, ReportActionDump, GroupedActionDump, type IExecutionDump, type IReportActionDump, type IGroupedActionDump, type ReportMeta, type GroupMeta, } from './types';
 export { z };
 export default Service;
 export { TaskRunner, Service, getVersion };

package/dist/types/service/index.d.ts

@@ -10,11 +10,14 @@ export type AnyValue<T> = {
 };
 interface ServiceOptions {
     taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
+    memoryRetrieverFn?: () => readonly string[] | Promise<readonly string[]>;
 }
 export default class Service {
     contextRetrieverFn: () => Promise<UIContext> | UIContext;
     taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
+    private memoryRetrieverFn?;
     constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: ServiceOptions);
+    private getAIMemory;
     locate(query: PlanningLocateParam, opt: LocateOpts, modelConfig: IModelConfig, abortSignal?: AbortSignal): Promise<LocateResultWithDump>;
     extract<T>(dataDemand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt, context?: UIContext): Promise<ServiceExtractResult<T>>;
     describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {

package/dist/types/types.d.ts

@@ -163,41 +163,6 @@ export declare class ServiceError extends Error {
     dump: ServiceDump;
     constructor(message: string, dump: ServiceDump);
 }
-/**
- * Thrown by a DeviceAction whose semantics are "assert / verify".
- *
- * The task executor in [`tasks.ts`](packages/core/src/agent/tasks.ts) treats
- * regular action errors as soft failures and asks the planner to retry — that
- * is the right behavior for transient issues like a missed tap, but it is the
- * wrong behavior for actions like `ExpectToast` whose entire purpose is to
- * guarantee a postcondition. When an `ActionAssertionError` bubbles out of
- * `action.call()`, the executor MUST stop the current `aiAct` run and surface
- * the error to the caller without re-planning.
- *
- * Use this class (or a subclass) instead of a plain `Error` from any action
- * whose contract is verification rather than mutation.
- */
-export declare class ActionAssertionError extends Error {
-    /** Name of the action that raised the assertion (e.g. "ExpectToast"). */
-    readonly actionName?: string;
-    /**
-     * Optional structured detail — surfaced to the test report so users can see
-     * what was expected vs. observed without parsing the message string.
-     */
-    readonly detail?: Record<string, unknown>;
-    constructor(message: string, options?: {
-        actionName?: string;
-        detail?: Record<string, unknown>;
-    });
-}
-/**
- * Type guard usable across packages without instanceof issues.
- *
- * Walks the `.cause` chain (depth-limited) so that wrappers like
- * `TaskExecutionError` — which the task runner uses to re-throw the failure of
- * an individual action — still report as assertion errors.
- */
-export declare function isActionAssertionError(err: unknown): err is ActionAssertionError;
 export interface LiteUISection {
     name: string;
     description: string;
@@ -212,6 +177,8 @@ export type ServiceAssertionResponse = AIAssertionResponse & {
  * agent
  */
 export type OnTaskStartTip = (tip: string) => Promise<void> | void;
+export type AIMemory = string | readonly string[];
+export type AIMemoryRetriever = () => AIMemory | Promise<AIMemory>;
 export interface AgentWaitForOpt extends ServiceExtractOption {
     checkIntervalMs?: number;
     timeoutMs?: number;
@@ -636,6 +603,25 @@ export interface AgentOpt {
      */
     outputFormat?: 'single-html' | 'html-and-external-assets';
     onTaskStartTip?: OnTaskStartTip;
+    /**
+     * User-provided memories that will be included in model calls.
+     * Use this for stable task knowledge such as account hints, domain rules,
+     * app-specific labels, or previous facts the model should consider.
+     */
+    aiMemory?: AIMemory;
+    /**
+     * Alias for aiMemory.
+     */
+    memory?: AIMemory;
+    /**
+     * Dynamically resolves memories before each model call. Useful for services
+     * that maintain memory outside the Agent, for example in a database or cache.
+     */
+    aiMemoryRetriever?: AIMemoryRetriever;
+    /**
+     * Alias for aiMemoryRetriever.
+     */
+    memoryRetriever?: AIMemoryRetriever;
     aiActContext?: string;
     aiActionContext?: string;
     reportFileName?: string;

package/dist/types/yaml.d.ts

@@ -62,12 +62,14 @@ export interface MidsceneYamlTask {
  *   groupName: "E2E Test Suite"
  *   generateReport: true
  *   replanningCycleLimit: 30
+ *   aiMemory:
+ *     - "The Settings tab is represented by a gear icon."
  *   cache:
  *     id: "checkout-cache"
  *     strategy: "read-write"
  * ```
  */
-export type MidsceneYamlScriptAgentOpt = Pick<AgentOpt, 'testId' | 'groupName' | 'groupDescription' | 'generateReport' | 'persistExecutionDump' | 'autoPrintReportMsg' | 'reportFileName' | 'replanningCycleLimit' | 'aiActContext' | 'aiActionContext' | 'cache' | 'screenshotShrinkFactor'>;
+export type MidsceneYamlScriptAgentOpt = Pick<AgentOpt, 'testId' | 'groupName' | 'groupDescription' | 'generateReport' | 'persistExecutionDump' | 'autoPrintReportMsg' | 'reportFileName' | 'replanningCycleLimit' | 'aiMemory' | 'memory' | 'aiActContext' | 'aiActionContext' | 'cache' | 'screenshotShrinkFactor'>;
 export interface MidsceneYamlScriptConfig {
     output?: string;
     unstableLogContent?: boolean | string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aiscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "9.0.1",
+  "version": "9.0.3",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",