@aiscene/shared 8.0.2 → 8.0.4

package/dist/es/agent-tools/observation-record.mjs

@@ -0,0 +1,248 @@
+import { createHash } from "node:crypto";
+import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
+import { z } from "zod";
+import { getMidsceneRunSubDir } from "../common.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const observationRecordSchema = z.object({
+    type: z.literal('midscene_ui_observation'),
+    version: z.literal(1),
+    startedAt: z.number().finite().nonnegative(),
+    endedAt: z.number().finite().nonnegative(),
+    frames: z.array(z.object({
+        path: z.string().min(1),
+        mimeType: z["enum"]([
+            'image/png',
+            'image/jpeg'
+        ]),
+        capturedAt: z.number().finite().nonnegative()
+    }).strict()).min(1),
+    shotSize: z.object({
+        width: z.number().finite().positive(),
+        height: z.number().finite().positive()
+    }).strict(),
+    shrunkShotToLogicalRatio: z.number().finite().positive()
+}).strict().refine((record)=>record.endedAt >= record.startedAt, {
+    path: [
+        'endedAt'
+    ],
+    message: 'must be greater than or equal to startedAt'
+});
+function defaultObservationRecordPath() {
+    const suffix = Math.random().toString(36).slice(2, 10);
+    return resolve(getMidsceneRunSubDir('output'), `observation-${Date.now()}-${suffix}.json`);
+}
+function parseUIObservationRecord(input) {
+    const result = observationRecordSchema.safeParse(input);
+    if (!result.success) {
+        const issue = result.error.issues[0];
+        const location = issue.path.length > 0 ? issue.path.join('.') : 'record';
+        throw new Error(`Invalid UI observation record at ${location}: ${issue.message}`);
+    }
+    return result.data;
+}
+function cloneUIObservationRecord(record) {
+    return {
+        ...record,
+        frames: record.frames.map((frame)=>({
+                ...frame
+            })),
+        shotSize: {
+            ...record.shotSize
+        }
+    };
+}
+function readUIObservationRecord(filePath) {
+    const resolvedPath = resolve(filePath);
+    let parsed;
+    try {
+        parsed = JSON.parse(readFileSync(resolvedPath, 'utf8'));
+    } catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        throw new Error(`Failed to read UI observation record ${resolvedPath}: ${message}`);
+    }
+    const record = parseUIObservationRecord(parsed);
+    const manifestDirectory = dirname(resolvedPath);
+    return {
+        ...record,
+        frames: record.frames.map((frame, index)=>{
+            if (isAbsolute(frame.path)) throw new Error(`Invalid UI observation record at frames.${index}.path: expected a relative image path`);
+            const imagePath = resolve(manifestDirectory, frame.path);
+            const relativeToManifest = relative(manifestDirectory, imagePath);
+            if ('..' === relativeToManifest || relativeToManifest.startsWith(`..${'win32' === process.platform ? '\\' : '/'}`) || isAbsolute(relativeToManifest)) throw new Error(`Invalid UI observation record at frames.${index}.path: image path escapes the manifest directory`);
+            try {
+                if (!statSync(imagePath).isFile()) throw new Error('path is not a file');
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                throw new Error(`Invalid UI observation record at frames.${index}.path: cannot read image ${imagePath}: ${message}`);
+            }
+            return {
+                ...frame,
+                path: imagePath
+            };
+        })
+    };
+}
+class UIObservationRecordWriter {
+    persistFrame(dataUrl, capturedAt) {
+        if (this.disposed) throw new Error('UI observation record writer has been disposed');
+        if (this.finalized) throw new Error('UI observation record has already been finalized');
+        const match = /^data:image\/(png|jpe?g);base64,([\s\S]+)$/i.exec(dataUrl);
+        if (!match) throw new Error('UI observation frame must be a PNG or JPEG data URL');
+        const format = 'png' === match[1].toLowerCase() ? 'png' : 'jpeg';
+        const mimeType = 'png' === format ? 'image/png' : 'image/jpeg';
+        const bytes = Buffer.from(match[2], 'base64');
+        if (0 === bytes.length) throw new Error('UI observation frame contains no image data');
+        const digest = createHash('sha256').update(bytes).digest('hex');
+        const fileName = `${digest}.${format}`;
+        mkdirSync(this.temporaryFramesDirectory, {
+            recursive: true
+        });
+        const temporaryPath = join(this.temporaryFramesDirectory, fileName);
+        if (!existsSync(temporaryPath)) writeFileSync(temporaryPath, bytes);
+        return {
+            path: `${basename(this.framesDirectory)}/${fileName}`,
+            mimeType,
+            capturedAt
+        };
+    }
+    resolveFramePath(frame) {
+        if (this.disposed) throw new Error('UI observation record writer has been disposed');
+        const directory = this.finalized ? this.framesDirectory : this.temporaryFramesDirectory;
+        return join(directory, basename(frame.path));
+    }
+    pruneFrames(frames) {
+        if (this.disposed) throw new Error('UI observation record writer has been disposed');
+        if (this.finalized || !existsSync(this.temporaryFramesDirectory)) return;
+        const retainedFiles = new Set(frames.map((frame)=>basename(frame.path)));
+        for (const fileName of readdirSync(this.temporaryFramesDirectory))if (!retainedFiles.has(fileName)) rmSync(join(this.temporaryFramesDirectory, fileName), {
+            force: true
+        });
+    }
+    finalize(frames, metadata) {
+        if (this.disposed) throw new Error('UI observation record writer has been disposed');
+        if (this.finalizedRecord) return this.finalizedRecord;
+        const record = parseUIObservationRecord({
+            type: 'midscene_ui_observation',
+            version: 1,
+            frames,
+            ...metadata
+        });
+        this.pruneFrames(record.frames);
+        mkdirSync(dirname(this.outputPath), {
+            recursive: true
+        });
+        mkdirSync(this.temporaryFramesDirectory, {
+            recursive: true
+        });
+        rmSync(this.framesDirectory, {
+            recursive: true,
+            force: true
+        });
+        renameSync(this.temporaryFramesDirectory, this.framesDirectory);
+        this.finalized = true;
+        this.finalizedRecord = {
+            ...record,
+            frames: record.frames.map((frame)=>({
+                    ...frame,
+                    path: resolve(dirname(this.outputPath), frame.path)
+                }))
+        };
+        return this.finalizedRecord;
+    }
+    dispose() {
+        if (this.disposed) return;
+        rmSync(this.temporaryFramesDirectory, {
+            recursive: true,
+            force: true
+        });
+        rmSync(this.framesDirectory, {
+            recursive: true,
+            force: true
+        });
+        this.disposed = true;
+        this.finalizedRecord = null;
+    }
+    constructor(filePath){
+        _define_property(this, "outputPath", void 0);
+        _define_property(this, "framesDirectory", void 0);
+        _define_property(this, "temporaryFramesDirectory", void 0);
+        _define_property(this, "finalized", false);
+        _define_property(this, "disposed", false);
+        _define_property(this, "finalizedRecord", null);
+        this.outputPath = filePath ? resolve(filePath) : defaultObservationRecordPath();
+        const extension = extname(this.outputPath);
+        const stem = basename(this.outputPath, extension);
+        this.framesDirectory = join(dirname(this.outputPath), `${stem}.frames`);
+        this.temporaryFramesDirectory = `${this.framesDirectory}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+    }
+}
+function writeUIObservationRecord(record, filePath) {
+    const validated = parseUIObservationRecord(record);
+    const outputPath = filePath ? resolve(filePath) : defaultObservationRecordPath();
+    const extension = extname(outputPath);
+    const stem = basename(outputPath, extension);
+    const framesDirectory = join(dirname(outputPath), `${stem}.frames`);
+    const temporaryFramesDirectory = `${framesDirectory}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+    const temporaryManifest = `${outputPath}.tmp-${process.pid}-${Math.random().toString(36).slice(2, 10)}`;
+    const copiedPaths = new Map();
+    try {
+        mkdirSync(temporaryFramesDirectory, {
+            recursive: true
+        });
+        const frames = validated.frames.map((frame, index)=>{
+            const sourcePath = resolve(frame.path);
+            try {
+                if (!statSync(sourcePath).isFile()) throw new Error('path is not a file');
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                throw new Error(`Invalid UI observation record at frames.${index}.path: cannot read image ${sourcePath}: ${message}`);
+            }
+            let relativePath = copiedPaths.get(sourcePath);
+            if (!relativePath) {
+                const imageExtension = 'image/png' === frame.mimeType ? 'png' : 'jpeg';
+                const fileName = `${String(copiedPaths.size).padStart(4, '0')}.${imageExtension}`;
+                relativePath = `${basename(framesDirectory)}/${fileName}`;
+                copyFileSync(sourcePath, join(temporaryFramesDirectory, fileName));
+                copiedPaths.set(sourcePath, relativePath);
+            }
+            return {
+                ...frame,
+                path: relativePath
+            };
+        });
+        const serializedRecord = parseUIObservationRecord({
+            ...validated,
+            frames
+        });
+        mkdirSync(dirname(outputPath), {
+            recursive: true
+        });
+        writeFileSync(temporaryManifest, JSON.stringify(serializedRecord, null, 2), 'utf8');
+        rmSync(framesDirectory, {
+            recursive: true,
+            force: true
+        });
+        renameSync(temporaryFramesDirectory, framesDirectory);
+        renameSync(temporaryManifest, outputPath);
+        return outputPath;
+    } finally{
+        rmSync(temporaryFramesDirectory, {
+            recursive: true,
+            force: true
+        });
+        rmSync(temporaryManifest, {
+            force: true
+        });
+    }
+}
+export { UIObservationRecordWriter, cloneUIObservationRecord, defaultObservationRecordPath, parseUIObservationRecord, readUIObservationRecord, writeUIObservationRecord };

package/dist/es/agent-tools/tool-defaults.mjs

@@ -0,0 +1,54 @@
+const TOOL_BEHAVIOR_FLAGS = [
+    {
+        cli: 'deep-locate',
+        description: 'Force deep locate for every locating operation (better precision for small/ambiguous targets, a bit slower).',
+        defaults: {
+            locate: {
+                deepLocate: true
+            },
+            act: {
+                deepLocate: true
+            }
+        }
+    },
+    {
+        cli: 'deep-think',
+        description: 'Plan the act tool with deep thinking (richer context and sub-goal decomposition, a bit slower).',
+        defaults: {
+            act: {
+                deepThink: true
+            }
+        }
+    }
+];
+function mergeToolDefaults(a, b) {
+    const locate = {
+        ...a.locate,
+        ...b.locate
+    };
+    const act = {
+        ...a.act,
+        ...b.act
+    };
+    const result = {};
+    if (Object.keys(locate).length > 0) result.locate = locate;
+    if (Object.keys(act).length > 0) result.act = act;
+    return result;
+}
+function resolveToolDefaults(isEnabled) {
+    return TOOL_BEHAVIOR_FLAGS.reduce((acc, flag)=>isEnabled(flag.cli) ? mergeToolDefaults(acc, flag.defaults) : acc, {});
+}
+function stripBehaviorFlags(argv) {
+    const enabled = new Set();
+    const rawArgs = [];
+    for (const arg of argv){
+        const flag = TOOL_BEHAVIOR_FLAGS.find((f)=>arg === `--${f.cli}`);
+        if (flag) enabled.add(flag.cli);
+        else rawArgs.push(arg);
+    }
+    return {
+        rawArgs,
+        toolDefaults: resolveToolDefaults((cli)=>enabled.has(cli))
+    };
+}
+export { TOOL_BEHAVIOR_FLAGS, mergeToolDefaults, resolveToolDefaults, stripBehaviorFlags };