@rpascene/core 0.30.8

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -0,0 +1,9 @@
+import type { DeviceAction } from '../../types';
+import type { TVlModeTypes } from '@rpascene/shared/env';
+import type { ResponseFormatJSONSchema } from 'openai/resources/index';
+export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
+export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, }: {
+    actionSpace: DeviceAction<any>[];
+    vlMode: TVlModeTypes | undefined;
+}): Promise<string>;
+export declare const planSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/llm-section-locator.d.ts

@@ -0,0 +1,6 @@
+import { PromptTemplate } from '@langchain/core/prompts';
+import type { TVlModeTypes } from '@rpascene/shared/env';
+export declare function systemPromptToLocateSection(vlMode: TVlModeTypes | undefined): string;
+export declare const sectionLocatorInstruction: PromptTemplate<{
+    sectionDescription: any;
+}, any>;

package/dist/types/ai-model/prompt/playwright-generator.d.ts

@@ -0,0 +1,26 @@
+import type { StreamingAIResponse, StreamingCodeGenerationOptions } from '../../types';
+import type { IModelConfig } from '@rpascene/shared/env';
+import { type ChromeRecordedEvent, type EventCounts, type EventSummary, type InputDescription, type ProcessedEvent, createEventCounts, createMessageContent, extractInputDescriptions, filterEventsByType, getScreenshotsForLLM, prepareEventSummary, processEventsForLLM, validateEvents } from './yaml-generator';
+export interface PlaywrightGenerationOptions {
+    testName?: string;
+    includeScreenshots?: boolean;
+    includeTimestamps?: boolean;
+    maxScreenshots?: number;
+    description?: string;
+    viewportSize?: {
+        width: number;
+        height: number;
+    };
+    waitForNetworkIdle?: boolean;
+    waitForNetworkIdleTimeout?: number;
+}
+export type { ChromeRecordedEvent, EventCounts, InputDescription, ProcessedEvent, EventSummary, };
+export { getScreenshotsForLLM, filterEventsByType, createEventCounts, extractInputDescriptions, processEventsForLLM, prepareEventSummary, createMessageContent, validateEvents, };
+/**
+ * Generates Playwright test code from recorded events
+ */
+export declare const generatePlaywrightTest: (events: ChromeRecordedEvent[], options: PlaywrightGenerationOptions, modelConfig: IModelConfig) => Promise<string>;
+/**
+ * Generates Playwright test code from recorded events with streaming support
+ */
+export declare const generatePlaywrightTestStream: (events: ChromeRecordedEvent[], options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions, modelConfig: IModelConfig) => Promise<StreamingAIResponse>;

package/dist/types/ai-model/prompt/ui-tars-locator.d.ts

@@ -0,0 +1 @@
+export declare function systemPromptToLocateElementPosition(): string;

package/dist/types/ai-model/prompt/ui-tars-planning.d.ts

@@ -0,0 +1,2 @@
+export declare function getUiTarsPlanningPrompt(): string;
+export declare const getSummary: (prediction: string) => string;

package/dist/types/ai-model/prompt/util.d.ts

@@ -0,0 +1,47 @@
+import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
+import type { TVlModeTypes } from '@rpascene/shared/env';
+export declare function describeSize(size: Size): string;
+export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
+    id: string;
+})[]): string;
+export declare const distanceThreshold = 16;
+export declare function elementByPositionWithElementInfo(treeRoot: ElementTreeNode<BaseElement>, position: {
+    x: number;
+    y: number;
+}, options?: {
+    requireStrictDistance?: boolean;
+    filterPositionElements?: boolean;
+}): BaseElement | undefined;
+export declare function distance(point1: {
+    x: number;
+    y: number;
+}, point2: {
+    x: number;
+    y: number;
+}): number;
+export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
+export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt: {
+    truncateTextLength?: number;
+    filterNonTextContent?: boolean;
+    domIncluded?: boolean | 'visible-only';
+    visibleOnly?: boolean;
+    vlMode: TVlModeTypes | undefined;
+}): Promise<{
+    description: string;
+    elementById(idOrIndexId: string): ElementType;
+    elementByPosition(position: {
+        x: number;
+        y: number;
+    }, size: {
+        width: number;
+        height: number;
+    }): BaseElement | undefined;
+    insertElementByPosition(position: {
+        x: number;
+        y: number;
+    }): ElementType;
+    size: {
+        width: number;
+        height: number;
+    };
+}>;

package/dist/types/ai-model/prompt/yaml-generator.d.ts

@@ -0,0 +1,100 @@
+import type { StreamingAIResponse, StreamingCodeGenerationOptions } from '../../types';
+import type { IModelConfig } from '@rpascene/shared/env';
+export interface EventCounts {
+    navigation: number;
+    click: number;
+    input: number;
+    scroll: number;
+    total: number;
+}
+export interface InputDescription {
+    description: string;
+    value: string;
+}
+export interface ProcessedEvent {
+    type: string;
+    timestamp: number;
+    url?: string;
+    title?: string;
+    elementDescription?: string;
+    value?: string;
+    pageInfo?: any;
+    elementRect?: any;
+}
+export interface EventSummary {
+    testName: string;
+    startUrl: string;
+    eventCounts: EventCounts;
+    urls: string[];
+    clickDescriptions: string[];
+    inputDescriptions: InputDescription[];
+    events: ProcessedEvent[];
+}
+export interface ChromeRecordedEvent {
+    type: string;
+    timestamp: number;
+    url?: string;
+    title?: string;
+    elementDescription?: string;
+    value?: string;
+    pageInfo?: any;
+    elementRect?: any;
+    screenshotBefore?: string;
+    screenshotAfter?: string;
+    screenshotWithBox?: string;
+}
+export interface YamlGenerationOptions {
+    testName?: string;
+    includeTimestamps?: boolean;
+    maxScreenshots?: number;
+    description?: string;
+}
+export interface FilteredEvents {
+    navigationEvents: ChromeRecordedEvent[];
+    clickEvents: ChromeRecordedEvent[];
+    inputEvents: ChromeRecordedEvent[];
+    scrollEvents: ChromeRecordedEvent[];
+}
+/**
+ * Get screenshots from events for LLM context
+ */
+export declare const getScreenshotsForLLM: (events: ChromeRecordedEvent[], maxScreenshots?: number) => string[];
+/**
+ * Filter events by type for easier processing
+ */
+export declare const filterEventsByType: (events: ChromeRecordedEvent[]) => FilteredEvents;
+/**
+ * Create event counts summary
+ */
+export declare const createEventCounts: (filteredEvents: FilteredEvents, totalEvents: number) => EventCounts;
+/**
+ * Extract input descriptions from input events
+ */
+export declare const extractInputDescriptions: (inputEvents: ChromeRecordedEvent[]) => InputDescription[];
+/**
+ * Process events for LLM consumption
+ */
+export declare const processEventsForLLM: (events: ChromeRecordedEvent[]) => ProcessedEvent[];
+/**
+ * Prepare comprehensive event summary for LLM
+ */
+export declare const prepareEventSummary: (events: ChromeRecordedEvent[], options?: {
+    testName?: string;
+    maxScreenshots?: number;
+}) => EventSummary;
+/**
+ * Create message content for LLM with optional screenshots
+ */
+export declare const createMessageContent: (promptText: string, screenshots?: string[], includeScreenshots?: boolean) => any[];
+/**
+ * Validate events before processing
+ */
+export declare const validateEvents: (events: ChromeRecordedEvent[]) => void;
+/**
+ * Generates YAML test configuration from recorded events using AI
+ */
+export declare const generateYamlTest: (events: ChromeRecordedEvent[], options: YamlGenerationOptions, modelConfig: IModelConfig) => Promise<string>;
+/**
+ * Generates YAML test configuration from recorded events using AI with streaming support
+ */
+export declare const generateYamlTestStream: (events: ChromeRecordedEvent[], options: YamlGenerationOptions & StreamingCodeGenerationOptions, modelConfig: IModelConfig) => Promise<StreamingAIResponse>;

package/dist/types/ai-model/service-caller/index.d.ts

@@ -0,0 +1,48 @@
+import { type AIUsageInfo } from '../../types';
+import type { StreamingCallback } from '../../types';
+import { type IModelConfig, type TVlModeTypes } from '@rpascene/shared/env';
+import OpenAI from 'openai';
+import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import { AIActionType, type AIArgs } from '../common';
+export interface AIServiceConfig {
+    domain: string;
+    yht_access_token: string;
+    chatType: number;
+    model?: string;
+    modelCategory: number;
+    stream?: number;
+    temperature?: number;
+    top_p?: number;
+    extraParams?: Record<string, any>;
+    topic?: string;
+    baseURL: string;
+}
+export declare const defaultYhtConfig: AIServiceConfig;
+export declare function callAIOld(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig, options?: {
+    stream?: boolean;
+    onChunk?: StreamingCallback;
+}): Promise<{
+    content: string;
+    usage?: AIUsageInfo;
+    isStreamed: boolean;
+}>;
+export declare const getResponseFormat: (modelName: string, AIActionTypeValue: AIActionType) => OpenAI.ChatCompletionCreateParams["response_format"] | OpenAI.ResponseFormatJSONObject;
+export declare function callAIWithObjectResponse<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
+    content: T;
+    usage?: AIUsageInfo;
+}>;
+export declare function callAIWithStringResponse(msgs: AIArgs, AIActionTypeValue: AIActionType, modelConfig: IModelConfig): Promise<{
+    content: string;
+    usage?: AIUsageInfo;
+}>;
+export declare function extractJSONFromCodeBlock(response: string): string;
+export declare function preprocessDoubaoBboxJson(input: string): string;
+export declare function safeParseJson(input: string, vlMode: TVlModeTypes | undefined): any;
+export declare function callAI(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelConfig: IModelConfig, options?: {
+    stream?: boolean;
+    onChunk?: StreamingCallback;
+}): Promise<{
+    content: string;
+    usage?: AIUsageInfo;
+    isStreamed: boolean;
+}>;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -0,0 +1,59 @@
+import type { PlanningAIResponse, Size, UIContext } from '../types';
+import { type IModelConfig, UITarsModelVersion } from '@rpascene/shared/env';
+import type { ConversationHistory } from './conversation-history';
+type ActionType = 'click' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait';
+export declare function uiTarsPlanning(userInstruction: string, options: {
+    conversationHistory: ConversationHistory;
+    context: UIContext;
+    modelConfig: IModelConfig;
+}): Promise<PlanningAIResponse>;
+interface BaseAction {
+    action_type: ActionType;
+    action_inputs: Record<string, any>;
+    reflection: string | null;
+    thought: string | null;
+}
+interface ClickAction extends BaseAction {
+    action_type: 'click';
+    action_inputs: {
+        start_box: string;
+    };
+}
+interface DragAction extends BaseAction {
+    action_type: 'drag';
+    action_inputs: {
+        start_box: string;
+        end_box: string;
+    };
+}
+interface WaitAction extends BaseAction {
+    action_type: 'wait';
+    action_inputs: {
+        time: string;
+    };
+}
+interface TypeAction extends BaseAction {
+    action_type: 'type';
+    action_inputs: {
+        content: string;
+    };
+}
+interface HotkeyAction extends BaseAction {
+    action_type: 'hotkey';
+    action_inputs: {
+        key: string;
+    };
+}
+interface ScrollAction extends BaseAction {
+    action_type: 'scroll';
+    action_inputs: {
+        direction: 'up' | 'down';
+    };
+}
+interface FinishedAction extends BaseAction {
+    action_type: 'finished';
+    action_inputs: Record<string, never>;
+}
+export type Action = ClickAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction;
+export declare function resizeImageForUiTars(imageBase64: string, size: Size, uiTarsVersion: UITarsModelVersion | undefined): Promise<string>;
+export {};