@midscene/core 1.5.5 → 1.5.6

package/dist/es/agent/utils.mjs

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.5.5";
+const getMidsceneVersion = ()=>"1.5.6";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/service-caller/codex-app-server.mjs

@@ -0,0 +1,584 @@
+import { getDebug } from "@midscene/shared/logger";
+import { ifInBrowser } from "@midscene/shared/utils";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const CODEX_PROVIDER_SCHEME = 'codex://';
+const CODEX_DEFAULT_TIMEOUT_MS = 600000;
+const CODEX_DEFAULT_PROCESS_START_TIMEOUT_MS = 15000;
+const CODEX_DEFAULT_CLEANUP_TIMEOUT_MS = 8000;
+const CODEX_TEXT_INPUT_MAX_LENGTH = 262144;
+const debugCodex = getDebug('ai:call:codex');
+const warnCodex = getDebug('ai:call:codex', {
+    console: true
+});
+class SerializedRunner {
+    async run(work) {
+        const previous = this.tail;
+        let release;
+        this.tail = new Promise((resolve)=>{
+            release = resolve;
+        });
+        await previous;
+        try {
+            return await work();
+        } finally{
+            release();
+        }
+    }
+    constructor(){
+        _define_property(this, "tail", Promise.resolve());
+    }
+}
+const isCodexAppServerProvider = (baseURL)=>{
+    if (!baseURL) return false;
+    return baseURL.trim().toLowerCase().startsWith(CODEX_PROVIDER_SCHEME);
+};
+const isAbortError = (error)=>{
+    if (!error) return false;
+    if (error instanceof Error && 'AbortError' === error.name) return true;
+    const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
+    return /aborted|abort/i.test(message);
+};
+const toNonEmptyString = (value)=>{
+    if ('string' != typeof value) return;
+    const trimmed = value.trim();
+    return trimmed || void 0;
+};
+const normalizeCodexLocalImagePath = (imageUrl, platform = process.platform)=>{
+    if (!imageUrl.startsWith('file://')) return imageUrl;
+    try {
+        const parsed = new URL(imageUrl);
+        const pathname = decodeURIComponent(parsed.pathname);
+        const host = parsed.hostname.toLowerCase();
+        if ('win32' === platform) {
+            const windowsPath = pathname.replace(/\//g, '\\').replace(/^\\([A-Za-z]:)/, '$1');
+            if (host && 'localhost' !== host) return `\\\\${parsed.hostname}${windowsPath}`;
+            return windowsPath;
+        }
+        if (host && 'localhost' !== host) return `//${parsed.hostname}${pathname}`;
+        return pathname;
+    } catch  {
+        return decodeURIComponent(imageUrl.slice(7));
+    }
+};
+const extractTextFromMessage = (message)=>{
+    const content = message.content;
+    if ('string' == typeof content) return content;
+    if (Array.isArray(content)) return content.map((part)=>{
+        if (!part || 'object' != typeof part) return '';
+        if ('text' === part.type && 'string' == typeof part.text) return part.text;
+        if ('input_text' === part.type && 'string' == typeof part.text) return part.text;
+        return '';
+    }).filter(Boolean).join('\n');
+    return '';
+};
+const extractImageInputs = (message, imageDetailOverride)=>{
+    const content = message.content;
+    if (!Array.isArray(content)) return [];
+    const inputs = [];
+    for (const part of content){
+        if (!part || 'object' != typeof part) continue;
+        const partType = String(part.type || '');
+        const imageUrl = 'image_url' === partType ? toNonEmptyString(part.image_url?.url) : 'input_image' === partType ? toNonEmptyString(part.image_url || part.url) : void 0;
+        if (!imageUrl) continue;
+        const detail = imageDetailOverride || toNonEmptyString(part.image_url?.detail) || toNonEmptyString(part.detail);
+        if (imageUrl.startsWith('/') || imageUrl.startsWith('./') || imageUrl.startsWith('../') || imageUrl.startsWith('file://')) {
+            const path = imageUrl.startsWith('file://') ? normalizeCodexLocalImagePath(imageUrl) : imageUrl;
+            inputs.push({
+                type: 'localImage',
+                path,
+                ...detail ? {
+                    detail
+                } : {}
+            });
+            continue;
+        }
+        inputs.push({
+            type: 'image',
+            url: imageUrl,
+            ...detail ? {
+                detail
+            } : {}
+        });
+    }
+    return inputs;
+};
+const resolveCodexReasoningEffort = ({ deepThink, modelConfig })=>{
+    if (true === deepThink) return 'high';
+    if (false === deepThink) return 'low';
+    const normalized = modelConfig.reasoningEffort?.trim().toLowerCase();
+    if ('low' === normalized || 'medium' === normalized || 'high' === normalized || 'xhigh' === normalized) return normalized;
+    if (true === modelConfig.reasoningEnabled) return 'high';
+    if (false === modelConfig.reasoningEnabled) return 'low';
+};
+const buildCodexTurnPayloadFromMessages = (messages, options)=>{
+    const developerInstructionParts = [];
+    const transcriptParts = [];
+    const imageInputs = [];
+    for (const message of messages){
+        const role = String(message.role || 'user');
+        const text = extractTextFromMessage(message);
+        if ('system' === role) {
+            if (text.trim()) developerInstructionParts.push(text.trim());
+            continue;
+        }
+        const roleTag = role.toUpperCase();
+        if (text.trim()) transcriptParts.push(`[${roleTag}]\n${text.trim()}`);
+        else transcriptParts.push(`[${roleTag}]\n(no text content)`);
+        if ('user' === role) imageInputs.push(...extractImageInputs(message, options?.imageDetailOverride));
+    }
+    const fullTranscript = transcriptParts.join('\n\n');
+    const transcriptText = (fullTranscript.length > CODEX_TEXT_INPUT_MAX_LENGTH ? fullTranscript.slice(-CODEX_TEXT_INPUT_MAX_LENGTH) : fullTranscript) || 'Please answer the latest user request.';
+    const input = [
+        {
+            type: 'text',
+            text: transcriptText,
+            text_elements: []
+        },
+        ...imageInputs
+    ];
+    const developerInstructions = developerInstructionParts.length ? developerInstructionParts.join('\n\n') : void 0;
+    return {
+        developerInstructions,
+        input
+    };
+};
+class CodexAppServerConnection {
+    static async create() {
+        if (ifInBrowser) throw new Error('codex app-server provider is not supported in browser runtime');
+        const childProcessModuleName = 'node:child_process';
+        const readlineModuleName = 'node:readline';
+        const { spawn } = await import(childProcessModuleName);
+        const readline = await import(readlineModuleName);
+        const child = spawn('codex', [
+            'app-server'
+        ], {
+            stdio: [
+                'pipe',
+                'pipe',
+                'pipe'
+            ]
+        });
+        if (!child.stdin || !child.stdout || !child.stderr) throw new Error('failed to start codex app-server: stdio unavailable');
+        const lineReader = readline.createInterface({
+            input: child.stdout,
+            crlfDelay: 1 / 0
+        });
+        const connection = new CodexAppServerConnection(child, lineReader);
+        connection.detachFromEventLoop();
+        connection.attachProcessListeners();
+        await connection.initializeHandshake();
+        return connection;
+    }
+    isClosed() {
+        return this.closed;
+    }
+    async runTurn({ messages, modelConfig, stream, onChunk, deepThink, abortSignal }) {
+        const startTime = Date.now();
+        const timeoutMs = modelConfig.timeout || CODEX_DEFAULT_TIMEOUT_MS;
+        const deadlineAt = Date.now() + timeoutMs;
+        const isStreaming = !!(stream && onChunk);
+        const imageDetailOverride = 'gpt-5' === modelConfig.modelFamily ? 'original' : void 0;
+        const { developerInstructions, input } = buildCodexTurnPayloadFromMessages(messages, {
+            imageDetailOverride
+        });
+        const effort = resolveCodexReasoningEffort({
+            deepThink,
+            modelConfig
+        });
+        let threadId;
+        let turnId;
+        let latestErrorMessage;
+        let accumulatedText = '';
+        let accumulatedReasoning = '';
+        let latestUsage;
+        const emitChunk = ({ content, reasoning, isComplete, usage })=>{
+            if (!isStreaming || !onChunk) return;
+            const chunk = {
+                content,
+                reasoning_content: reasoning,
+                accumulated: accumulatedText,
+                isComplete,
+                usage
+            };
+            onChunk(chunk);
+        };
+        try {
+            const threadStartResponse = await this.request({
+                method: 'thread/start',
+                params: {
+                    model: modelConfig.modelName,
+                    cwd: process.cwd(),
+                    approvalPolicy: 'never',
+                    sandbox: 'read-only',
+                    ephemeral: true,
+                    experimentalRawEvents: false,
+                    persistExtendedHistory: false,
+                    developerInstructions: developerInstructions || null
+                },
+                deadlineAt,
+                abortSignal
+            });
+            threadId = threadStartResponse?.thread?.id;
+            if (!threadId) throw new Error('thread/start did not return a thread id');
+            const turnStartResponse = await this.request({
+                method: 'turn/start',
+                params: {
+                    threadId,
+                    input,
+                    effort
+                },
+                deadlineAt,
+                abortSignal
+            });
+            turnId = turnStartResponse?.turn?.id;
+            if (!turnId) throw new Error('turn/start did not return a turn id');
+            let turnStatus;
+            while(!turnStatus){
+                const message = await this.nextMessage({
+                    deadlineAt,
+                    abortSignal
+                });
+                if (this.isResponseMessage(message)) continue;
+                if (this.isRequestMessage(message)) {
+                    await this.respondToServerRequest(message);
+                    continue;
+                }
+                const notification = message;
+                const method = notification.method;
+                const params = notification.params || {};
+                if ('error' === method) {
+                    const messageText = params.error?.message || params.message || 'codex app-server reported turn error';
+                    latestErrorMessage = String(messageText);
+                    continue;
+                }
+                if ('item/agentMessage/delta' === method && params.threadId === threadId && params.turnId === turnId) {
+                    const delta = String(params.delta || '');
+                    if (delta) {
+                        accumulatedText += delta;
+                        emitChunk({
+                            content: delta,
+                            reasoning: '',
+                            isComplete: false
+                        });
+                    }
+                    continue;
+                }
+                if (('item/reasoning/summaryTextDelta' === method || 'item/reasoning/textDelta' === method) && params.threadId === threadId && params.turnId === turnId) {
+                    const delta = String(params.delta || '');
+                    if (delta) {
+                        accumulatedReasoning += delta;
+                        emitChunk({
+                            content: '',
+                            reasoning: delta,
+                            isComplete: false
+                        });
+                    }
+                    continue;
+                }
+                if ('item/completed' === method && params.threadId === threadId && params.turnId === turnId && params.item?.type === 'agentMessage' && 'string' == typeof params.item?.text && !accumulatedText) {
+                    accumulatedText = params.item.text;
+                    continue;
+                }
+                if ('thread/tokenUsage/updated' === method && params.threadId === threadId && params.turnId === turnId) {
+                    latestUsage = this.mapUsage({
+                        usage: params,
+                        modelConfig,
+                        turnId,
+                        startTime
+                    });
+                    continue;
+                }
+                if ('turn/completed' === method && params.threadId === threadId && params.turn?.id === turnId) {
+                    turnStatus = String(params.turn.status || '');
+                    latestErrorMessage = params.turn?.error?.message || latestErrorMessage || void 0;
+                    break;
+                }
+            }
+            if ('completed' !== turnStatus) throw new Error(latestErrorMessage || `codex turn finished with status "${turnStatus || 'unknown'}"`);
+            if (isStreaming) emitChunk({
+                content: '',
+                reasoning: '',
+                isComplete: true,
+                usage: latestUsage
+            });
+            return {
+                content: accumulatedText,
+                reasoning_content: accumulatedReasoning || void 0,
+                usage: latestUsage,
+                isStreamed: isStreaming
+            };
+        } catch (error) {
+            if (isAbortError(error) && threadId && turnId) await this.request({
+                method: 'turn/interrupt',
+                params: {
+                    threadId,
+                    turnId
+                },
+                deadlineAt: Date.now() + 5000
+            }).catch(()=>{});
+            throw error;
+        } finally{
+            if (threadId) await this.request({
+                method: 'thread/unsubscribe',
+                params: {
+                    threadId
+                },
+                deadlineAt: Date.now() + CODEX_DEFAULT_CLEANUP_TIMEOUT_MS
+            }).catch((error)=>{
+                warnCodex(`failed to unsubscribe codex thread ${threadId}: ${String(error)}`);
+            });
+        }
+    }
+    async dispose() {
+        if (this.closed) return;
+        this.closed = true;
+        try {
+            this.lineReader?.close?.();
+        } catch  {}
+        try {
+            this.child?.stdin?.end?.();
+        } catch  {}
+        try {
+            this.child?.kill?.();
+        } catch  {}
+    }
+    attachProcessListeners() {
+        this.lineReader.on('line', (line)=>{
+            this.lineBuffer.push(line);
+        });
+        this.child.stderr.on('data', (chunk)=>{
+            const text = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
+            this.stderrBuffer += text;
+            if (this.stderrBuffer.length > 8192) this.stderrBuffer = this.stderrBuffer.slice(-8192);
+        });
+        this.child.on('exit', (code)=>{
+            this.closed = true;
+            this.lastExitCode = code;
+        });
+        this.child.on('error', (error)=>{
+            this.closed = true;
+            this.processErrorMessage = error.message;
+        });
+    }
+    detachFromEventLoop() {
+        this.child.unref?.();
+        this.child.stdin?.unref?.();
+        this.child.stdout?.unref?.();
+        this.child.stderr?.unref?.();
+    }
+    async initializeHandshake() {
+        const deadlineAt = Date.now() + CODEX_DEFAULT_PROCESS_START_TIMEOUT_MS;
+        await this.request({
+            method: 'initialize',
+            params: {
+                clientInfo: {
+                    name: 'midscene_codex_provider',
+                    title: 'Midscene Codex Provider',
+                    version: '1.0.0'
+                },
+                capabilities: {
+                    experimentalApi: false
+                }
+            },
+            deadlineAt
+        });
+        await this.sendMessage({
+            method: 'initialized'
+        });
+    }
+    mapUsage({ usage, modelConfig, turnId, startTime }) {
+        const tokenUsage = usage.tokenUsage;
+        const picked = tokenUsage?.last || tokenUsage?.total;
+        if (!picked) return;
+        return {
+            prompt_tokens: picked.inputTokens ?? 0,
+            completion_tokens: picked.outputTokens ?? 0,
+            total_tokens: picked.totalTokens ?? 0,
+            cached_input: picked.cachedInputTokens ?? 0,
+            time_cost: Date.now() - startTime,
+            model_name: modelConfig.modelName,
+            model_description: modelConfig.modelDescription,
+            intent: modelConfig.intent,
+            request_id: turnId
+        };
+    }
+    isRequestMessage(message) {
+        return 'string' == typeof message?.method && message?.id !== void 0;
+    }
+    isResponseMessage(message) {
+        return message?.id !== void 0 && (message?.result !== void 0 || message?.error !== void 0) && 'string' != typeof message?.method;
+    }
+    async request({ method, params, deadlineAt, abortSignal }) {
+        const requestId = this.nextRequestId++;
+        await this.sendMessage({
+            id: requestId,
+            method,
+            params
+        });
+        while(true){
+            const message = await this.nextMessage({
+                deadlineAt,
+                abortSignal,
+                includePending: false
+            });
+            if (this.isResponseMessage(message) && message.id === requestId) {
+                if (message.error) throw new Error(`codex app-server ${method} failed: ${message.error.message || 'unknown error'}`);
+                return message.result || {};
+            }
+            if (this.isRequestMessage(message)) {
+                await this.respondToServerRequest(message);
+                continue;
+            }
+            this.pendingMessages.push(message);
+        }
+    }
+    async respondToServerRequest(request) {
+        const requestId = request.id;
+        const method = request.method;
+        let result = {};
+        if ('item/commandExecution/requestApproval' === method) result = {
+            decision: 'decline'
+        };
+        else if ('item/fileChange/requestApproval' === method) result = {
+            decision: 'decline'
+        };
+        else if ('mcpServer/elicitation/request' === method) result = {
+            action: 'cancel',
+            content: null
+        };
+        else {
+            if ('item/tool/requestUserInput' !== method) return void await this.sendMessage({
+                id: requestId,
+                error: {
+                    code: -32601,
+                    message: `unsupported server request: ${method}`
+                }
+            });
+            result = {
+                answers: []
+            };
+        }
+        await this.sendMessage({
+            id: requestId,
+            result
+        });
+    }
+    async nextMessage({ deadlineAt, abortSignal, includePending = true }) {
+        if (includePending && this.pendingMessages.length) return this.pendingMessages.shift();
+        while(true){
+            if (abortSignal?.aborted) throw new Error('codex app-server request aborted');
+            if (deadlineAt && Date.now() > deadlineAt) throw new Error('codex app-server request timed out');
+            if (this.lineBuffer.length) {
+                const line = this.lineBuffer.shift();
+                const trimmed = line.trim();
+                if (!trimmed) continue;
+                let parsed;
+                try {
+                    parsed = JSON.parse(trimmed);
+                } catch (error) {
+                    warnCodex(`ignored non-JSON message from codex app-server: ${trimmed}`);
+                    continue;
+                }
+                return parsed;
+            }
+            if (this.closed) throw this.createClosedConnectionError();
+            await new Promise((resolve)=>setTimeout(resolve, 20));
+        }
+    }
+    async sendMessage(payload) {
+        if (this.closed) throw this.createClosedConnectionError();
+        const line = JSON.stringify(payload);
+        await new Promise((resolve, reject)=>{
+            this.child.stdin.write(`${line}\n`, (error)=>{
+                if (error) return void reject(new Error(`failed writing to codex app-server stdin: ${error.message}`));
+                resolve();
+            });
+        });
+    }
+    createClosedConnectionError() {
+        const stderr = this.stderrBuffer.trim();
+        if (this.processErrorMessage) return new Error(stderr ? `codex app-server process error: ${this.processErrorMessage}. stderr=${stderr}` : `codex app-server process error: ${this.processErrorMessage}`);
+        return new Error(stderr ? `codex app-server connection closed (exitCode=${this.lastExitCode}). stderr=${stderr}` : `codex app-server connection closed (exitCode=${this.lastExitCode})`);
+    }
+    constructor(child, lineReader){
+        _define_property(this, "child", void 0);
+        _define_property(this, "lineReader", void 0);
+        _define_property(this, "pendingMessages", []);
+        _define_property(this, "lineBuffer", []);
+        _define_property(this, "nextRequestId", 1);
+        _define_property(this, "closed", false);
+        _define_property(this, "lastExitCode", null);
+        _define_property(this, "processErrorMessage", null);
+        _define_property(this, "stderrBuffer", '');
+        this.child = child;
+        this.lineReader = lineReader;
+    }
+}
+class CodexAppServerConnectionManager {
+    async runTurn({ messages, modelConfig, stream, onChunk, deepThink, abortSignal }) {
+        return this.runner.run(async ()=>{
+            const connection = await this.getConnection();
+            try {
+                return await connection.runTurn({
+                    messages,
+                    modelConfig,
+                    stream,
+                    onChunk,
+                    deepThink,
+                    abortSignal
+                });
+            } catch (error) {
+                if (connection.isClosed() || !isAbortError(error)) await this.resetConnection();
+                throw error;
+            }
+        });
+    }
+    async shutdownForTests() {
+        await this.resetConnection();
+    }
+    async getConnection() {
+        if (!this.connection || this.connection.isClosed()) {
+            this.connection = await CodexAppServerConnection.create();
+            debugCodex('started long-lived codex app-server connection');
+        }
+        return this.connection;
+    }
+    async resetConnection() {
+        if (!this.connection) return;
+        const staleConnection = this.connection;
+        this.connection = null;
+        await staleConnection.dispose();
+        debugCodex('reset codex app-server connection');
+    }
+    constructor(){
+        _define_property(this, "connection", null);
+        _define_property(this, "runner", new SerializedRunner());
+    }
+}
+const codexConnectionManager = new CodexAppServerConnectionManager();
+async function callAIWithCodexAppServer(messages, modelConfig, options) {
+    if (ifInBrowser) throw new Error('codex app-server provider is not supported in browser runtime');
+    return codexConnectionManager.runTurn({
+        messages,
+        modelConfig,
+        stream: options?.stream,
+        onChunk: options?.onChunk,
+        deepThink: options?.deepThink,
+        abortSignal: options?.abortSignal
+    });
+}
+async function __shutdownCodexAppServerForTests() {
+    await codexConnectionManager.shutdownForTests();
+}
+export { __shutdownCodexAppServerForTests, buildCodexTurnPayloadFromMessages, callAIWithCodexAppServer, isCodexAppServerProvider, normalizeCodexLocalImagePath, resolveCodexReasoningEffort };
+
+//# sourceMappingURL=codex-app-server.mjs.map