@aiscene/core 9.0.6 → 9.0.8

package/dist/types/report-stats.d.ts

@@ -0,0 +1,39 @@
+import type { IReportActionDump } from './types';
+export interface ReportModelUsageSummary {
+    modelName: string;
+    callCount: number;
+    promptTokens: number;
+    cachedInputTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+    modelCallTimeMs: number;
+}
+export interface ReportTokenSummary {
+    promptTokens: number;
+    cachedInputTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+}
+export interface ReportTimingSummary {
+    /**
+     * Elapsed time between the earliest and latest recorded task timestamps.
+     * Falls back to `wallTimeFallbackMs` only when no task timestamps exist.
+     */
+    wallTimeMs?: number;
+    wallTimeStart?: number;
+    wallTimeEnd?: number;
+    wallTimeSource: 'task-timestamps' | 'fallback' | 'unavailable';
+    /** Sum of every recorded model request duration. */
+    modelCallTimeMs: number;
+    modelCallCount: number;
+}
+export interface ReportSummary {
+    timing: ReportTimingSummary;
+    tokens: ReportTokenSummary;
+    models: ReportModelUsageSummary[];
+}
+export interface CollectReportSummaryOptions {
+    /** Generic fallback for reports without task timestamps, such as test time. */
+    wallTimeFallbackMs?: number;
+}
+export declare function collectReportSummary(report: Pick<IReportActionDump, 'executions'>, options?: CollectReportSummaryOptions): ReportSummary;

package/dist/types/screenshot-item.d.ts

@@ -26,6 +26,7 @@ export declare class ScreenshotItem {
     private constructor();
     /** Create a new ScreenshotItem from base64 data */
     static create(base64: string, capturedAt: number): ScreenshotItem;
+    static fromFile(filePath: string, mimeType: 'image/png' | 'image/jpeg', capturedAt: number): ScreenshotItem;
     get id(): string;
     /** Get the image format (png or jpeg) */
     get format(): 'png' | 'jpeg';

package/dist/types/service/index.d.ts

@@ -1,3 +1,4 @@
+import { type ModelRuntime } from '../ai-model/index';
 import type { AIDescribeElementResponse, LocateResultElement, LocateResultWithDump, PlanningLocateParam, Rect, ServiceExtractOption, ServiceExtractParam, ServiceExtractResult, ServiceTaskInfo, UIContext } from '../types';
 import type { IModelConfig } from '@aiscene/shared/env';
 import type { TMultimodalPrompt } from '../common';
@@ -12,16 +13,19 @@ interface ServiceOptions {
     taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
     memoryRetrieverFn?: () => readonly string[] | Promise<readonly string[]>;
 }
+type ModelRuntimeInput = IModelConfig | ModelRuntime;
 export default class Service {
     contextRetrieverFn: () => Promise<UIContext> | UIContext;
     taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;
     private memoryRetrieverFn?;
     constructor(context: UIContext | (() => Promise<UIContext> | UIContext), opt?: ServiceOptions);
     private getAIMemory;
-    locate(query: PlanningLocateParam, opt: LocateOpts, modelConfig: IModelConfig, abortSignal?: AbortSignal): Promise<LocateResultWithDump>;
+    locate(query: PlanningLocateParam, opt: LocateOpts, modelRuntimeOrConfig: ModelRuntimeInput, abortSignal?: AbortSignal): Promise<LocateResultWithDump>;
     extract<T>(dataDemand: ServiceExtractParam, modelConfig: IModelConfig, opt?: ServiceExtractOption, pageDescription?: string, multimodalPrompt?: TMultimodalPrompt, context?: UIContext): Promise<ServiceExtractResult<T>>;
-    describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
+    describe(target: Rect | [number, number], modelRuntimeOrConfig: ModelRuntimeInput, opt?: {
         deepLocate?: boolean;
+        deepDescribe?: boolean;
+        context?: UIContext;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }
 export {};

package/dist/types/task-runner.d.ts

@@ -1,4 +1,10 @@
 import { ExecutionDump, type ExecutionTask, type ExecutionTaskApply, type ExecutionTaskProgressOptions, type UIContext } from './types';
+export type TaskRunnerEventKind = 'append' | 'start' | 'finish' | 'error' | 'cancel';
+export interface TaskRunnerEvent {
+    kind: TaskRunnerEventKind;
+    task: ExecutionTask;
+    runner: TaskRunner;
+}
 type TaskRunnerInitOptions = ExecutionTaskProgressOptions & {
     tasks?: ExecutionTaskApply[];
     onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;

package/dist/types/types.d.ts

@@ -24,6 +24,7 @@ export type AIUsageInfo = Record<string, any> & {
      * planning call may use the default slot when no planning model is configured.
      */
     slot: string | undefined;
+    response_model_name: string | undefined;
     request_id: string | undefined;
 };
 export type { LocateResultElement };
@@ -36,6 +37,15 @@ export type AISingleElementResponseByPosition = {
     reason: string;
     text: string;
 };
+export type LocateResultPoint = [number, number];
+export type Bbox = [number, number, number, number];
+export type LocateResultBbox = Bbox;
+export type PixelBbox = Bbox;
+export interface AIElementLocateResponse {
+    bbox?: LocateResultBbox;
+    point?: LocateResultPoint;
+    error?: string;
+}
 export interface AIElementCoordinatesResponse {
     bbox: [number, number, number, number];
     errors?: string[];
@@ -47,8 +57,10 @@ export interface AIDataExtractionResponse<DataDemand> {
     thought?: string;
 }
 export interface AISectionLocatorResponse {
-    bbox: [number, number, number, number];
-    references_bbox?: [number, number, number, number][];
+    bbox?: LocateResultBbox;
+    point?: LocateResultPoint;
+    references_bbox?: LocateResultBbox[];
+    references_point?: LocateResultPoint[];
     error?: string;
 }
 export interface AIAssertionResponse {
@@ -67,11 +79,16 @@ export interface LocateValidatorResult {
     rect: Rect;
     center: [number, number];
     centerDistance?: number;
+    includedInRect?: boolean;
 }
 export interface AgentDescribeElementAtPointResult {
     prompt: string;
     deepLocate: boolean;
+    deepDescribe?: boolean;
     verifyResult?: LocateValidatorResult;
+    success?: boolean;
+    error?: string;
+    failureStage?: 'describe' | 'verify';
 }
 /**
  * context
@@ -81,6 +98,7 @@ export declare abstract class UIContext {
      * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),
      */
     abstract screenshot: ScreenshotItem;
+    abstract screenshotSequence?: ScreenshotItem[];
     /**
      * screenshot size after shrinking
      */
@@ -111,14 +129,17 @@ export interface LocateResult {
     rect?: Rect;
 }
 export type ThinkingLevel = 'off' | 'medium' | 'high';
+export type AiActEffort = 'fast' | 'balance' | 'deepThink';
 export type DeepThinkOption = 'unset' | true | false;
 export interface ServiceTaskInfo {
     durationMs: number;
     formatResponse?: string;
     rawResponse?: string;
+    rawChoiceMessage?: unknown;
     usage?: AIUsageInfo;
     searchArea?: Rect;
     searchAreaRawResponse?: string;
+    searchAreaRawChoiceMessage?: unknown;
     searchAreaUsage?: AIUsageInfo;
     reasoning_content?: string;
 }
@@ -130,6 +151,21 @@ export interface ReportDumpWithAttributes {
     dumpString: string;
     attributes?: ReportAttributes;
 }
+export interface RecordToReportScreenshot {
+    /**
+     * PNG/JPEG data URI, or raw PNG base64 body.
+     */
+    base64: string;
+    description?: string;
+}
+export interface RecordToReportOptions {
+    content?: string;
+    /**
+     * @deprecated Use `screenshots: [{ base64 }]` instead.
+     */
+    screenshotBase64?: string;
+    screenshots?: RecordToReportScreenshot[];
+}
 export interface ServiceDump extends DumpMeta {
     type: 'locate' | 'extract' | 'assert';
     logId: string;
@@ -185,6 +221,29 @@ export interface AgentWaitForOpt extends ServiceExtractOption {
 }
 export interface AgentAssertOpt {
     keepRawResponse?: boolean;
+    context?: string;
+    abortSignal?: AbortSignal;
+}
+export interface AgentAssertResult {
+    pass: boolean;
+    thought?: string;
+    message?: string;
+}
+export type QueryOptions = ServiceExtractOption;
+export type AssertOptions = AgentAssertOpt & ServiceExtractOption;
+export type ObservationQueryOptions = Omit<QueryOptions, 'domIncluded'> & {
+    domIncluded?: never;
+};
+export type ObservationAssertOptions = Omit<AssertOptions, 'domIncluded'> & {
+    domIncluded?: never;
+};
+export interface InsightAPI<QueryOpt extends QueryOptions = QueryOptions, AssertOpt extends AssertOptions = AssertOptions> {
+    aiQuery<ReturnType = any>(demand: ServiceExtractParam, options?: QueryOpt): Promise<ReturnType>;
+    aiBoolean(prompt: TUserPrompt, options?: QueryOpt): Promise<boolean>;
+    aiNumber(prompt: TUserPrompt, options?: QueryOpt): Promise<number>;
+    aiString(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;
+    aiAsk(prompt: TUserPrompt, options?: QueryOpt): Promise<string>;
+    aiAssert(assertion: TUserPrompt, message?: string, options?: AssertOpt): Promise<AgentAssertResult | undefined>;
 }
 /**
  * planning
@@ -193,12 +252,16 @@ export interface AgentAssertOpt {
 export interface PlanningLocateParam extends DetailedLocateParam {
     bbox?: [number, number, number, number];
 }
-export interface PlanningAction<ParamType = any> {
+type PlanningActionBase = {
     thought?: string;
     log?: string;
     type: string;
+};
+export type PlanningAction<ParamType = undefined> = PlanningActionBase & ([ParamType] extends [undefined] ? {
+    param?: any;
+} : {
     param: ParamType;
-}
+});
 export type SubGoalStatus = 'pending' | 'running' | 'finished';
 export interface SubGoal {
     index: number;
@@ -207,7 +270,7 @@ export interface SubGoal {
     logs?: string[];
 }
 export interface RawResponsePlanningAIResponse {
-    action: PlanningAction;
+    action: PlanningAction | null;
     thought?: string;
     log: string;
     memory?: string;
@@ -221,6 +284,7 @@ export interface PlanningAIResponse extends Omit<RawResponsePlanningAIResponse,
     actions?: PlanningAction[];
     usage?: AIUsageInfo;
     rawResponse?: string;
+    rawChoiceMessage?: unknown;
     yamlFlow?: MidsceneYamlFlowItem[];
     yamlString?: string;
     error?: string;
@@ -260,6 +324,33 @@ export interface PlaywrightParserOpt extends BaseAgentParserOpt {
 export interface ExecutionTaskProgressOptions {
     onTaskStart?: (task: ExecutionTask) => Promise<void> | void;
 }
+export interface AgentProgressEvent<TScope extends string = string, TData = unknown, TPhase extends string = string> {
+    scope: TScope;
+    phase: TPhase;
+    sequence: number;
+    data: TData;
+}
+export type AgentProgressListener<TScope extends string = string, TData = unknown, TPhase extends string = string> = (event: AgentProgressEvent<TScope, TData, TPhase>) => Promise<void> | void;
+export type AiActProgressPhase = 'start' | 'plan_thinking' | 'plan_planned' | 'plan_action' | 'plan_failed' | 'action_running' | 'action_done' | 'action_failed' | 'complete' | 'failed';
+export interface AiActProgressAction {
+    name: string;
+    target?: string;
+    point?: [number, number];
+    bbox?: [number, number, number, number];
+    param?: unknown;
+}
+export interface AiActProgressData {
+    prompt?: string;
+    planIndex?: number;
+    planLimit?: number;
+    screenshot?: ScreenshotItem;
+    action?: AiActProgressAction;
+    thought?: string;
+    log?: string;
+    output?: string;
+    durationMs?: number;
+    error?: string;
+}
 export interface ExecutionRecorderItem {
     type: 'screenshot';
     ts: number;
@@ -315,6 +406,7 @@ export type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = Executio
     };
     usage?: AIUsageInfo;
     searchAreaUsage?: AIUsageInfo;
+    searchArea?: Rect;
     reasoning_content?: string;
 };
 export interface IExecutionDump extends DumpMeta {
@@ -577,6 +669,7 @@ export interface WebElementInfo extends BaseElement {
 export type CacheConfig = {
     strategy?: 'read-only' | 'read-write' | 'write-only';
     id: string;
+    cacheDir?: string;
 };
 export type Cache = false | true | CacheConfig;
 export interface AgentOpt {

package/dist/types/yaml/cleanup.d.ts

@@ -0,0 +1,2 @@
+import type { FreeFn } from '../types';
+export declare function runFreeFnCleanup(freeFn: FreeFn[]): Promise<void>;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aiscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "9.0.6",
+  "version": "9.0.8",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -78,7 +78,7 @@
     "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
   },
   "dependencies": {
-    "@aiscene/shared": "workspace:*",
+    "@aiscene/shared": "8.0.5",
     "@ui-tars/action-parser": "1.2.3",
     "dayjs": "^1.11.11",
     "dotenv": "^16.4.5",