@midscene/core 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/ai-model.js +63 -85
- package/dist/es/image.js +10 -280
- package/dist/es/index.js +33 -39
- package/dist/lib/ai-model.js +57 -82
- package/dist/lib/image.js +9 -297
- package/dist/lib/index.js +28 -42
- package/dist/types/ai-model.d.ts +3 -3
- package/dist/types/image.d.ts +1 -118
- package/dist/types/{index-7a9ec3e1.d.ts → index-0c968ddd.d.ts} +1 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/{types-ed68710b.d.ts → types-ad0dfcb1.d.ts} +2 -2
- package/dist/types/utils.d.ts +1 -1
- package/package.json +3 -3
package/dist/lib/image.js
CHANGED
|
@@ -1,9 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __create = Object.create;
|
|
3
2
|
var __defProp = Object.defineProperty;
|
|
4
3
|
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
4
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
-
var __getProtoOf = Object.getPrototypeOf;
|
|
7
5
|
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
8
6
|
var __export = (target, all) => {
|
|
9
7
|
for (var name in all)
|
|
@@ -17,314 +15,28 @@ var __copyProps = (to, from, except, desc) => {
|
|
|
17
15
|
}
|
|
18
16
|
return to;
|
|
19
17
|
};
|
|
20
|
-
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
21
|
-
// If the importer is in node compatibility mode or this is not an ESM
|
|
22
|
-
// file that has been converted to a CommonJS file using a Babel-
|
|
23
|
-
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
24
|
-
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
25
|
-
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
26
|
-
mod
|
|
27
|
-
));
|
|
28
18
|
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
29
19
|
|
|
30
20
|
// src/image/index.ts
|
|
31
21
|
var image_exports = {};
|
|
32
22
|
__export(image_exports, {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
saveBase64Image: () => saveBase64Image,
|
|
41
|
-
transformImgPathToBase64: () => transformImgPathToBase64,
|
|
42
|
-
trimImage: () => trimImage
|
|
23
|
+
base64Encoded: () => import_img.base64Encoded,
|
|
24
|
+
calculateNewDimensions: () => import_img.calculateNewDimensions,
|
|
25
|
+
imageInfo: () => import_img.imageInfo,
|
|
26
|
+
imageInfoOfBase64: () => import_img.imageInfoOfBase64,
|
|
27
|
+
resizeImg: () => import_img.resizeImg,
|
|
28
|
+
saveBase64Image: () => import_img.saveBase64Image,
|
|
29
|
+
transformImgPathToBase64: () => import_img.transformImgPathToBase64
|
|
43
30
|
});
|
|
44
31
|
module.exports = __toCommonJS(image_exports);
|
|
45
|
-
|
|
46
|
-
// src/image/info.ts
|
|
47
|
-
var import_node_assert = __toESM(require("assert"));
|
|
48
|
-
var import_node_buffer = require("buffer");
|
|
49
|
-
var import_node_fs = require("fs");
|
|
50
|
-
var import_sharp = __toESM(require("sharp"));
|
|
51
|
-
async function imageInfo(image) {
|
|
52
|
-
const { width, height } = await (0, import_sharp.default)(image).metadata();
|
|
53
|
-
(0, import_node_assert.default)(width && height, `invalid image: ${image}`);
|
|
54
|
-
return { width, height };
|
|
55
|
-
}
|
|
56
|
-
async function imageInfoOfBase64(imageBase64) {
|
|
57
|
-
const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
|
|
58
|
-
return imageInfo(import_node_buffer.Buffer.from(base64Data, "base64"));
|
|
59
|
-
}
|
|
60
|
-
function base64Encoded(image, withHeader = true) {
|
|
61
|
-
const imageBuffer = (0, import_node_fs.readFileSync)(image);
|
|
62
|
-
if (!withHeader) {
|
|
63
|
-
return imageBuffer.toString("base64");
|
|
64
|
-
}
|
|
65
|
-
if (image.endsWith("png")) {
|
|
66
|
-
return `data:image/png;base64,${imageBuffer.toString("base64")}`;
|
|
67
|
-
}
|
|
68
|
-
if (image.endsWith("jpg") || image.endsWith("jpeg")) {
|
|
69
|
-
return `data:image/jpeg;base64,${imageBuffer.toString("base64")}`;
|
|
70
|
-
}
|
|
71
|
-
throw new Error("unsupported image type");
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// src/image/transform.ts
|
|
75
|
-
var import_node_buffer2 = require("buffer");
|
|
76
|
-
var import_sharp2 = __toESM(require("sharp"));
|
|
77
|
-
async function saveBase64Image(options) {
|
|
78
|
-
const { base64Data, outputPath } = options;
|
|
79
|
-
const base64Image = base64Data.split(";base64,").pop() || base64Data;
|
|
80
|
-
const imageBuffer = import_node_buffer2.Buffer.from(base64Image, "base64");
|
|
81
|
-
await (0, import_sharp2.default)(imageBuffer).toFile(outputPath);
|
|
82
|
-
console.log("Image successfully written to file.");
|
|
83
|
-
}
|
|
84
|
-
async function transformImgPathToBase64(inputPath) {
|
|
85
|
-
return await (0, import_sharp2.default)(inputPath).toBuffer().then((data) => {
|
|
86
|
-
const base64Data = data.toString("base64");
|
|
87
|
-
return base64Data;
|
|
88
|
-
});
|
|
89
|
-
}
|
|
90
|
-
async function resizeImg(base64Data) {
|
|
91
|
-
const base64Image = base64Data.split(";base64,").pop() || base64Data;
|
|
92
|
-
const imageBuffer = import_node_buffer2.Buffer.from(base64Image, "base64");
|
|
93
|
-
const metadata = await (0, import_sharp2.default)(imageBuffer).metadata();
|
|
94
|
-
const { width, height } = metadata;
|
|
95
|
-
if (!width || !height) {
|
|
96
|
-
throw Error("undefined width or height with url");
|
|
97
|
-
}
|
|
98
|
-
const newSize = calculateNewDimensions(width, height);
|
|
99
|
-
return await (0, import_sharp2.default)(imageBuffer).resize(newSize.width, newSize.height).toBuffer().then((data) => {
|
|
100
|
-
const base64Data2 = data.toString("base64");
|
|
101
|
-
return base64Data2;
|
|
102
|
-
});
|
|
103
|
-
}
|
|
104
|
-
function calculateNewDimensions(originalWidth, originalHeight) {
|
|
105
|
-
const maxWidth = 768;
|
|
106
|
-
const maxHeight = 2048;
|
|
107
|
-
let newWidth = originalWidth;
|
|
108
|
-
let newHeight = originalHeight;
|
|
109
|
-
const aspectRatio = originalWidth / originalHeight;
|
|
110
|
-
if (originalWidth > maxWidth) {
|
|
111
|
-
newWidth = maxWidth;
|
|
112
|
-
newHeight = newWidth / aspectRatio;
|
|
113
|
-
}
|
|
114
|
-
if (newHeight > maxHeight) {
|
|
115
|
-
newHeight = maxHeight;
|
|
116
|
-
newWidth = newHeight * aspectRatio;
|
|
117
|
-
}
|
|
118
|
-
return {
|
|
119
|
-
width: Math.round(newWidth),
|
|
120
|
-
height: Math.round(newHeight)
|
|
121
|
-
};
|
|
122
|
-
}
|
|
123
|
-
async function trimImage(image) {
|
|
124
|
-
const imgInstance = (0, import_sharp2.default)(image);
|
|
125
|
-
const instanceInfo = await imgInstance.metadata();
|
|
126
|
-
if (!instanceInfo.width || instanceInfo.width <= 3 || !instanceInfo.height || instanceInfo.height <= 3) {
|
|
127
|
-
return null;
|
|
128
|
-
}
|
|
129
|
-
const { info } = await imgInstance.trim().toBuffer({
|
|
130
|
-
resolveWithObject: true
|
|
131
|
-
});
|
|
132
|
-
if (typeof info.trimOffsetLeft === "undefined" || typeof info.trimOffsetTop === "undefined") {
|
|
133
|
-
return null;
|
|
134
|
-
}
|
|
135
|
-
return {
|
|
136
|
-
trimOffsetLeft: info.trimOffsetLeft,
|
|
137
|
-
trimOffsetTop: info.trimOffsetTop,
|
|
138
|
-
width: info.width,
|
|
139
|
-
height: info.height
|
|
140
|
-
};
|
|
141
|
-
}
|
|
142
|
-
async function alignCoordByTrim(image, centerRect) {
|
|
143
|
-
const imgInfo = await (0, import_sharp2.default)(image).metadata();
|
|
144
|
-
if (!(imgInfo == null ? void 0 : imgInfo.width) || !imgInfo.height || imgInfo.width <= 3 || imgInfo.height <= 3) {
|
|
145
|
-
return centerRect;
|
|
146
|
-
}
|
|
147
|
-
try {
|
|
148
|
-
const img = await (0, import_sharp2.default)(image).extract(centerRect).toBuffer();
|
|
149
|
-
const trimInfo = await trimImage(img);
|
|
150
|
-
if (!trimInfo) {
|
|
151
|
-
return centerRect;
|
|
152
|
-
}
|
|
153
|
-
return {
|
|
154
|
-
left: centerRect.left - trimInfo.trimOffsetLeft,
|
|
155
|
-
top: centerRect.top - trimInfo.trimOffsetTop,
|
|
156
|
-
width: trimInfo.width,
|
|
157
|
-
height: trimInfo.height
|
|
158
|
-
};
|
|
159
|
-
} catch (e) {
|
|
160
|
-
console.log(imgInfo);
|
|
161
|
-
throw e;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
// src/image/visualization.ts
|
|
166
|
-
var import_node_buffer3 = require("buffer");
|
|
167
|
-
|
|
168
|
-
// src/utils.ts
|
|
169
|
-
var import_node_assert2 = __toESM(require("assert"));
|
|
170
|
-
var import_node_crypto = require("crypto");
|
|
171
|
-
var import_node_fs2 = require("fs");
|
|
172
|
-
var import_node_os = require("os");
|
|
173
|
-
var import_node_path = __toESM(require("path"));
|
|
174
|
-
var pkg;
|
|
175
|
-
function getPkgInfo() {
|
|
176
|
-
if (pkg) {
|
|
177
|
-
return pkg;
|
|
178
|
-
}
|
|
179
|
-
const pkgDir = findNearestPackageJson(__dirname);
|
|
180
|
-
(0, import_node_assert2.default)(pkgDir, "package.json not found");
|
|
181
|
-
const pkgJsonFile = (0, import_node_path.join)(pkgDir, "package.json");
|
|
182
|
-
if (pkgJsonFile) {
|
|
183
|
-
const { name, version } = JSON.parse((0, import_node_fs2.readFileSync)(pkgJsonFile, "utf-8"));
|
|
184
|
-
pkg = { name, version, dir: pkgDir };
|
|
185
|
-
return pkg;
|
|
186
|
-
}
|
|
187
|
-
return {
|
|
188
|
-
name: "midscene-unknown-page-name",
|
|
189
|
-
version: "0.0.0",
|
|
190
|
-
dir: pkgDir
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
var logDir = (0, import_node_path.join)(process.cwd(), "./midscene_run/");
|
|
194
|
-
function getTmpDir() {
|
|
195
|
-
const path2 = (0, import_node_path.join)((0, import_node_os.tmpdir)(), getPkgInfo().name);
|
|
196
|
-
(0, import_node_fs2.mkdirSync)(path2, { recursive: true });
|
|
197
|
-
return path2;
|
|
198
|
-
}
|
|
199
|
-
function getTmpFile(fileExtWithoutDot) {
|
|
200
|
-
const filename = `${(0, import_node_crypto.randomUUID)()}.${fileExtWithoutDot}`;
|
|
201
|
-
return (0, import_node_path.join)(getTmpDir(), filename);
|
|
202
|
-
}
|
|
203
|
-
function findNearestPackageJson(dir) {
|
|
204
|
-
const packageJsonPath = import_node_path.default.join(dir, "package.json");
|
|
205
|
-
if ((0, import_node_fs2.existsSync)(packageJsonPath)) {
|
|
206
|
-
return dir;
|
|
207
|
-
}
|
|
208
|
-
const parentDir = import_node_path.default.dirname(dir);
|
|
209
|
-
if (parentDir === dir) {
|
|
210
|
-
return null;
|
|
211
|
-
}
|
|
212
|
-
return findNearestPackageJson(parentDir);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
// src/image/visualization.ts
|
|
216
|
-
var import_sharp3 = __toESM(require("sharp"));
|
|
217
|
-
var colors = [
|
|
218
|
-
{
|
|
219
|
-
name: "Red",
|
|
220
|
-
hex: "#FF0000"
|
|
221
|
-
},
|
|
222
|
-
{
|
|
223
|
-
name: "Green",
|
|
224
|
-
hex: "#00FF00"
|
|
225
|
-
},
|
|
226
|
-
{
|
|
227
|
-
name: "Blue",
|
|
228
|
-
hex: "#0000FF"
|
|
229
|
-
},
|
|
230
|
-
{
|
|
231
|
-
name: "Yellow",
|
|
232
|
-
hex: "#FFFF00"
|
|
233
|
-
},
|
|
234
|
-
{
|
|
235
|
-
name: "Cyan",
|
|
236
|
-
hex: "#00FFFF"
|
|
237
|
-
},
|
|
238
|
-
{
|
|
239
|
-
name: "Magenta",
|
|
240
|
-
hex: "#FF00FF"
|
|
241
|
-
},
|
|
242
|
-
{
|
|
243
|
-
name: "Orange",
|
|
244
|
-
hex: "#FFA500"
|
|
245
|
-
},
|
|
246
|
-
{
|
|
247
|
-
name: "Purple",
|
|
248
|
-
hex: "#800080"
|
|
249
|
-
},
|
|
250
|
-
{
|
|
251
|
-
name: "Brown",
|
|
252
|
-
hex: "#A52A2A"
|
|
253
|
-
},
|
|
254
|
-
{
|
|
255
|
-
name: "Pink",
|
|
256
|
-
hex: "#FFC0CB"
|
|
257
|
-
},
|
|
258
|
-
{
|
|
259
|
-
name: "Light Blue",
|
|
260
|
-
hex: "#ADD8E6"
|
|
261
|
-
},
|
|
262
|
-
{
|
|
263
|
-
name: "Lime",
|
|
264
|
-
hex: "#00FF00"
|
|
265
|
-
},
|
|
266
|
-
{
|
|
267
|
-
name: "Violet",
|
|
268
|
-
hex: "#EE82EE"
|
|
269
|
-
},
|
|
270
|
-
{
|
|
271
|
-
name: "Gold",
|
|
272
|
-
hex: "#FFD700"
|
|
273
|
-
},
|
|
274
|
-
{
|
|
275
|
-
name: "Teal",
|
|
276
|
-
hex: "#008080"
|
|
277
|
-
}
|
|
278
|
-
];
|
|
279
|
-
var sizeLimit = 512;
|
|
280
|
-
var textFontSize = 12;
|
|
281
|
-
async function composeSectionDiagram(sections, context) {
|
|
282
|
-
const { width, height } = await imageInfo(context.screenshotBase64);
|
|
283
|
-
const ratio = Math.min(sizeLimit / width, sizeLimit / height, 1);
|
|
284
|
-
const canvasWidth = width * ratio;
|
|
285
|
-
const canvasHeight = height * ratio;
|
|
286
|
-
const sectionNameColorMap = {};
|
|
287
|
-
const rects = sections.map((section, index) => {
|
|
288
|
-
const { left, top, width: width2, height: height2 } = section.rect;
|
|
289
|
-
const color = colors[index % colors.length];
|
|
290
|
-
sectionNameColorMap[section.name] = color;
|
|
291
|
-
return `
|
|
292
|
-
<rect x="${left * ratio}" y="${top * ratio}" width="${width2 * ratio}" height="${height2 * ratio}" fill="${color.hex}" />
|
|
293
|
-
<text x="${left * ratio}" y="${top * ratio + textFontSize}" font-family="Arial" font-size="${textFontSize}" fill="black">
|
|
294
|
-
${section.name}
|
|
295
|
-
</text>
|
|
296
|
-
`;
|
|
297
|
-
});
|
|
298
|
-
const rectangles = `
|
|
299
|
-
<svg width="${canvasWidth}" height="${canvasHeight}">
|
|
300
|
-
${rects.join("\n")}
|
|
301
|
-
</svg>
|
|
302
|
-
`;
|
|
303
|
-
const svgBuffer = import_node_buffer3.Buffer.from(rectangles);
|
|
304
|
-
const file = getTmpFile("png");
|
|
305
|
-
await (0, import_sharp3.default)({
|
|
306
|
-
create: {
|
|
307
|
-
width: canvasWidth,
|
|
308
|
-
height: canvasHeight,
|
|
309
|
-
channels: 4,
|
|
310
|
-
background: { r: 255, g: 255, b: 255, alpha: 1 }
|
|
311
|
-
}
|
|
312
|
-
}).composite([{ input: svgBuffer }]).png().toFile(file);
|
|
313
|
-
return {
|
|
314
|
-
file,
|
|
315
|
-
sectionNameColorMap
|
|
316
|
-
};
|
|
317
|
-
}
|
|
32
|
+
var import_img = require("@midscene/shared/img");
|
|
318
33
|
// Annotate the CommonJS export names for ESM import in node:
|
|
319
34
|
0 && (module.exports = {
|
|
320
|
-
alignCoordByTrim,
|
|
321
35
|
base64Encoded,
|
|
322
36
|
calculateNewDimensions,
|
|
323
|
-
composeSectionDiagram,
|
|
324
37
|
imageInfo,
|
|
325
38
|
imageInfoOfBase64,
|
|
326
39
|
resizeImg,
|
|
327
40
|
saveBase64Image,
|
|
328
|
-
transformImgPathToBase64
|
|
329
|
-
trimImage
|
|
41
|
+
transformImgPathToBase64
|
|
330
42
|
});
|
package/dist/lib/index.js
CHANGED
|
@@ -1267,12 +1267,13 @@ var Executor = class {
|
|
|
1267
1267
|
}
|
|
1268
1268
|
if (successfullyCompleted) {
|
|
1269
1269
|
this.status = "completed";
|
|
1270
|
-
if (this.tasks.length) {
|
|
1271
|
-
return this.tasks[this.tasks.length - 1].output;
|
|
1272
|
-
}
|
|
1273
1270
|
} else {
|
|
1274
1271
|
this.status = "error";
|
|
1275
1272
|
}
|
|
1273
|
+
if (this.tasks.length) {
|
|
1274
|
+
const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
|
|
1275
|
+
return this.tasks[outputIndex].output;
|
|
1276
|
+
}
|
|
1276
1277
|
}
|
|
1277
1278
|
isInErrorState() {
|
|
1278
1279
|
return this.status === "error";
|
|
@@ -1302,7 +1303,7 @@ var Executor = class {
|
|
|
1302
1303
|
};
|
|
1303
1304
|
|
|
1304
1305
|
// src/insight/index.ts
|
|
1305
|
-
var
|
|
1306
|
+
var import_node_assert9 = __toESM(require("assert"));
|
|
1306
1307
|
|
|
1307
1308
|
// src/ai-model/openai/index.ts
|
|
1308
1309
|
var import_node_assert3 = __toESM(require("assert"));
|
|
@@ -4535,6 +4536,7 @@ var import_openai = __toESM(require("openai"));
|
|
|
4535
4536
|
var MIDSCENE_OPENAI_INIT_CONFIG_JSON = "MIDSCENE_OPENAI_INIT_CONFIG_JSON";
|
|
4536
4537
|
var MIDSCENE_MODEL_NAME = "MIDSCENE_MODEL_NAME";
|
|
4537
4538
|
var MIDSCENE_LANGSMITH_DEBUG = "MIDSCENE_LANGSMITH_DEBUG";
|
|
4539
|
+
var MIDSCENE_DEBUG_AI_PROFILE = "MIDSCENE_DEBUG_AI_PROFILE";
|
|
4538
4540
|
var OPENAI_API_KEY = "OPENAI_API_KEY";
|
|
4539
4541
|
function useOpenAIModel(useModel) {
|
|
4540
4542
|
if (useModel && useModel !== "openAI")
|
|
@@ -4564,12 +4566,16 @@ async function createOpenAI() {
|
|
|
4564
4566
|
}
|
|
4565
4567
|
async function call(messages, responseFormat) {
|
|
4566
4568
|
const openai = await createOpenAI();
|
|
4569
|
+
const shouldPrintTiming = typeof process.env[MIDSCENE_DEBUG_AI_PROFILE] === "string";
|
|
4570
|
+
shouldPrintTiming && console.time("Midscene - AI call");
|
|
4567
4571
|
const completion = await openai.chat.completions.create({
|
|
4568
4572
|
model,
|
|
4569
4573
|
messages,
|
|
4570
4574
|
response_format: { type: responseFormat },
|
|
4571
4575
|
temperature: 0.2
|
|
4572
4576
|
});
|
|
4577
|
+
shouldPrintTiming && console.timeEnd("Midscene - AI call");
|
|
4578
|
+
shouldPrintTiming && console.log("Midscene - AI usage", completion.usage);
|
|
4573
4579
|
const { content } = completion.choices[0].message;
|
|
4574
4580
|
(0, import_node_assert3.default)(content, "empty content");
|
|
4575
4581
|
return content;
|
|
@@ -4717,30 +4723,10 @@ function multiDescription(multi) {
|
|
|
4717
4723
|
}
|
|
4718
4724
|
|
|
4719
4725
|
// src/ai-model/prompt/util.ts
|
|
4720
|
-
var import_node_assert5 = __toESM(require("assert"));
|
|
4721
|
-
|
|
4722
|
-
// src/image/info.ts
|
|
4723
4726
|
var import_node_assert4 = __toESM(require("assert"));
|
|
4724
|
-
var import_node_buffer = require("buffer");
|
|
4725
|
-
var import_node_fs2 = require("fs");
|
|
4726
|
-
var import_sharp = __toESM(require("sharp"));
|
|
4727
|
-
async function imageInfo(image) {
|
|
4728
|
-
const { width, height } = await (0, import_sharp.default)(image).metadata();
|
|
4729
|
-
(0, import_node_assert4.default)(width && height, `invalid image: ${image}`);
|
|
4730
|
-
return { width, height };
|
|
4731
|
-
}
|
|
4732
|
-
async function imageInfoOfBase64(imageBase64) {
|
|
4733
|
-
const base64Data = imageBase64.replace(/^data:image\/\w+;base64,/, "");
|
|
4734
|
-
return imageInfo(import_node_buffer.Buffer.from(base64Data, "base64"));
|
|
4735
|
-
}
|
|
4736
|
-
|
|
4737
|
-
// src/image/transform.ts
|
|
4738
|
-
var import_node_buffer2 = require("buffer");
|
|
4739
|
-
var import_sharp2 = __toESM(require("sharp"));
|
|
4740
4727
|
|
|
4741
|
-
// src/image/
|
|
4742
|
-
var
|
|
4743
|
-
var import_sharp3 = __toESM(require("sharp"));
|
|
4728
|
+
// src/image/index.ts
|
|
4729
|
+
var import_img = require("@midscene/shared/img");
|
|
4744
4730
|
|
|
4745
4731
|
// src/ai-model/prompt/util.ts
|
|
4746
4732
|
var characteristic = "You are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.";
|
|
@@ -4813,7 +4799,7 @@ async function describeUserPage(context) {
|
|
|
4813
4799
|
if (context.size) {
|
|
4814
4800
|
({ width, height } = context.size);
|
|
4815
4801
|
} else {
|
|
4816
|
-
const imgSize = await imageInfoOfBase64(screenshotBase64);
|
|
4802
|
+
const imgSize = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
4817
4803
|
({ width, height } = imgSize);
|
|
4818
4804
|
}
|
|
4819
4805
|
const elementsInfo = context.content;
|
|
@@ -4834,7 +4820,7 @@ async function describeUserPage(context) {
|
|
|
4834
4820
|
"elementInfos": ${JSON.stringify(elementInfosDescription)}
|
|
4835
4821
|
}`,
|
|
4836
4822
|
elementById(id) {
|
|
4837
|
-
(0,
|
|
4823
|
+
(0, import_node_assert4.default)(typeof id !== "undefined", "id is required for query");
|
|
4838
4824
|
const item = idElementMap[`${id}`];
|
|
4839
4825
|
return item;
|
|
4840
4826
|
}
|
|
@@ -4899,10 +4885,10 @@ function retrieveSection(prompt) {
|
|
|
4899
4885
|
}
|
|
4900
4886
|
|
|
4901
4887
|
// src/ai-model/inspect.ts
|
|
4902
|
-
var
|
|
4888
|
+
var import_node_assert6 = __toESM(require("assert"));
|
|
4903
4889
|
|
|
4904
4890
|
// src/ai-model/coze/index.ts
|
|
4905
|
-
var
|
|
4891
|
+
var import_node_assert5 = __toESM(require("assert"));
|
|
4906
4892
|
var import_node_fetch = __toESM(require("node-fetch"));
|
|
4907
4893
|
var COZE_INSPECT_ELEMENT_BOT_ID = process.env.COZE_INSPECT_ELEMENT_BOT_ID || "";
|
|
4908
4894
|
var COZE_AI_ACTION_BOT_ID = process.env.COZE_AI_ACTION_BOT_ID || "";
|
|
@@ -4955,7 +4941,7 @@ async function callCozeAi(options) {
|
|
|
4955
4941
|
throw new Error("aiResponse is undefined", aiResponse);
|
|
4956
4942
|
}
|
|
4957
4943
|
const parseContent = (_b = aiResponse == null ? void 0 : aiResponse.messages[0]) == null ? void 0 : _b.content;
|
|
4958
|
-
(0,
|
|
4944
|
+
(0, import_node_assert5.default)(parseContent, "empty content");
|
|
4959
4945
|
try {
|
|
4960
4946
|
return JSON.parse(parseContent);
|
|
4961
4947
|
} catch (err) {
|
|
@@ -5130,7 +5116,7 @@ DATA_DEMAND ends.
|
|
|
5130
5116
|
}
|
|
5131
5117
|
async function AiAssert(options) {
|
|
5132
5118
|
const { assertion, context, useModel } = options;
|
|
5133
|
-
(0,
|
|
5119
|
+
(0, import_node_assert6.default)(assertion, "assertion should be a string");
|
|
5134
5120
|
const { screenshotBase64 } = context;
|
|
5135
5121
|
const { description, elementById } = await describeUserPage(context);
|
|
5136
5122
|
const systemPrompt = systemPromptToAssert();
|
|
@@ -5174,7 +5160,7 @@ async function AiAssert(options) {
|
|
|
5174
5160
|
}
|
|
5175
5161
|
|
|
5176
5162
|
// src/ai-model/automation/index.ts
|
|
5177
|
-
var
|
|
5163
|
+
var import_node_assert7 = __toESM(require("assert"));
|
|
5178
5164
|
|
|
5179
5165
|
// src/ai-model/automation/planning.ts
|
|
5180
5166
|
function systemPromptToTaskPlanning() {
|
|
@@ -5280,8 +5266,8 @@ async function plan(userPrompt, opts, useModel) {
|
|
|
5280
5266
|
});
|
|
5281
5267
|
}
|
|
5282
5268
|
const actions = (planFromAI == null ? void 0 : planFromAI.actions) || [];
|
|
5283
|
-
(0,
|
|
5284
|
-
(0,
|
|
5269
|
+
(0, import_node_assert7.default)(planFromAI, "can't get planFromAI");
|
|
5270
|
+
(0, import_node_assert7.default)(actions && actions.length > 0, "no actions in ai plan");
|
|
5285
5271
|
if (planFromAI.error) {
|
|
5286
5272
|
throw new Error(planFromAI.error);
|
|
5287
5273
|
}
|
|
@@ -5289,9 +5275,9 @@ async function plan(userPrompt, opts, useModel) {
|
|
|
5289
5275
|
}
|
|
5290
5276
|
|
|
5291
5277
|
// src/insight/utils.ts
|
|
5292
|
-
var
|
|
5278
|
+
var import_node_assert8 = __toESM(require("assert"));
|
|
5293
5279
|
var import_node_crypto2 = require("crypto");
|
|
5294
|
-
var
|
|
5280
|
+
var import_node_fs2 = require("fs");
|
|
5295
5281
|
var import_node_path2 = require("path");
|
|
5296
5282
|
var logFileName = "";
|
|
5297
5283
|
var logContent = [];
|
|
@@ -5300,7 +5286,7 @@ var { pid } = process;
|
|
|
5300
5286
|
var logFileExt = insightDumpFileExt;
|
|
5301
5287
|
function writeInsightDump(data, logId, dumpSubscriber) {
|
|
5302
5288
|
const logDir2 = getLogDir();
|
|
5303
|
-
(0,
|
|
5289
|
+
(0, import_node_assert8.default)(logDir2, "logDir should be set before writing dump file");
|
|
5304
5290
|
const id = logId || (0, import_node_crypto2.randomUUID)();
|
|
5305
5291
|
const baseData = {
|
|
5306
5292
|
sdkVersion: getPkgInfo().version,
|
|
@@ -5312,7 +5298,7 @@ function writeInsightDump(data, logId, dumpSubscriber) {
|
|
|
5312
5298
|
dumpSubscriber == null ? void 0 : dumpSubscriber(finalData);
|
|
5313
5299
|
if (!logFileName) {
|
|
5314
5300
|
logFileName = `pid_${pid}_${baseData.logTime}`;
|
|
5315
|
-
while ((0,
|
|
5301
|
+
while ((0, import_node_fs2.existsSync)((0, import_node_path2.join)(logDir2, `${logFileName}.${logFileExt}`))) {
|
|
5316
5302
|
logFileName = `${pid}_${baseData.logTime}-${Math.random()}`;
|
|
5317
5303
|
}
|
|
5318
5304
|
}
|
|
@@ -5396,7 +5382,7 @@ var Insight = class {
|
|
|
5396
5382
|
__publicField(this, "aiVendorFn", callAiFn);
|
|
5397
5383
|
__publicField(this, "onceDumpUpdatedFn");
|
|
5398
5384
|
__publicField(this, "taskInfo");
|
|
5399
|
-
(0,
|
|
5385
|
+
(0, import_node_assert9.default)(context, "context is required for Insight");
|
|
5400
5386
|
if (typeof context === "function") {
|
|
5401
5387
|
this.contextRetrieverFn = context;
|
|
5402
5388
|
} else {
|
|
@@ -5412,7 +5398,7 @@ var Insight = class {
|
|
|
5412
5398
|
async locate(queryPrompt, opt) {
|
|
5413
5399
|
var _a;
|
|
5414
5400
|
const { callAI: callAI2, multi = false } = opt || {};
|
|
5415
|
-
(0,
|
|
5401
|
+
(0, import_node_assert9.default)(queryPrompt, "query is required for located");
|
|
5416
5402
|
const dumpSubscriber = this.onceDumpUpdatedFn;
|
|
5417
5403
|
this.onceDumpUpdatedFn = void 0;
|
|
5418
5404
|
const context = await this.contextRetrieverFn();
|
|
@@ -5484,7 +5470,7 @@ ${parseResult.errors.join("\n")}`;
|
|
|
5484
5470
|
}
|
|
5485
5471
|
async extract(dataDemand) {
|
|
5486
5472
|
var _a;
|
|
5487
|
-
(0,
|
|
5473
|
+
(0, import_node_assert9.default)(
|
|
5488
5474
|
typeof dataDemand === "object" || typeof dataDemand === "string",
|
|
5489
5475
|
`dataDemand should be object or string, but get ${typeof dataDemand}`
|
|
5490
5476
|
);
|
package/dist/types/ai-model.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { ChatCompletionMessageParam } from 'openai/resources';
|
|
2
2
|
export { ChatCompletionMessageParam } from 'openai/resources';
|
|
3
|
-
import { c as callAiFn } from './index-
|
|
4
|
-
export { d as describeUserPage, p as plan } from './index-
|
|
5
|
-
import { B as BaseElement, U as UIContext,
|
|
3
|
+
import { c as callAiFn } from './index-0c968ddd.js';
|
|
4
|
+
export { d as describeUserPage, p as plan } from './index-0c968ddd.js';
|
|
5
|
+
import { B as BaseElement, U as UIContext, A as AIElementParseResponse, b as AISectionParseResponse, c as AIAssertionResponse } from './types-ad0dfcb1.js';
|
|
6
6
|
|
|
7
7
|
declare function AiInspectElement<ElementType extends BaseElement = BaseElement>(options: {
|
|
8
8
|
context: UIContext<ElementType>;
|
package/dist/types/image.d.ts
CHANGED
|
@@ -1,118 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { S as Size, R as Rect, h as UISection, U as UIContext, G as Color } from './types-ed68710b.js';
|
|
3
|
-
import 'openai/resources';
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Retrieves the dimensions of an image asynchronously
|
|
7
|
-
*
|
|
8
|
-
* @param image - The image data, which can be a string path or a buffer
|
|
9
|
-
* @returns A Promise that resolves to an object containing the width and height of the image
|
|
10
|
-
* @throws Error if the image data is invalid
|
|
11
|
-
*/
|
|
12
|
-
declare function imageInfo(image: string | Buffer): Promise<Size>;
|
|
13
|
-
/**
|
|
14
|
-
* Retrieves the dimensions of an image from a base64-encoded string
|
|
15
|
-
*
|
|
16
|
-
* @param imageBase64 - The base64-encoded image data
|
|
17
|
-
* @returns A Promise that resolves to an object containing the width and height of the image
|
|
18
|
-
* @throws Error if the image data is invalid
|
|
19
|
-
*/
|
|
20
|
-
declare function imageInfoOfBase64(imageBase64: string): Promise<Size>;
|
|
21
|
-
/**
|
|
22
|
-
* Encodes an image file to a base64 encoded string
|
|
23
|
-
*
|
|
24
|
-
* @param image The path of the image file
|
|
25
|
-
* @param withHeader Determine whether to return data including the file header information, the default is true
|
|
26
|
-
*
|
|
27
|
-
* @returns The base64 encoded string of the image file, which may or may not include header information depending on the withHeader parameter
|
|
28
|
-
*
|
|
29
|
-
* @throws When the image type is not supported, an error will be thrown
|
|
30
|
-
*/
|
|
31
|
-
declare function base64Encoded(image: string, withHeader?: boolean): string;
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Saves a Base64-encoded image to a file
|
|
35
|
-
*
|
|
36
|
-
* @param options - An object containing the Base64-encoded image data and the output file path
|
|
37
|
-
* @param options.base64Data - The Base64-encoded image data
|
|
38
|
-
* @param options.outputPath - The path where the image will be saved
|
|
39
|
-
* @throws Error if there is an error during the saving process
|
|
40
|
-
*/
|
|
41
|
-
declare function saveBase64Image(options: {
|
|
42
|
-
base64Data: string;
|
|
43
|
-
outputPath: string;
|
|
44
|
-
}): Promise<void>;
|
|
45
|
-
/**
|
|
46
|
-
* Transforms an image path into a base64-encoded string
|
|
47
|
-
* @param inputPath - The path of the image file to be encoded
|
|
48
|
-
* @returns A Promise that resolves to a base64-encoded string representing the image file
|
|
49
|
-
*/
|
|
50
|
-
declare function transformImgPathToBase64(inputPath: string): Promise<string>;
|
|
51
|
-
/**
|
|
52
|
-
* Resizes an image from a base64-encoded string
|
|
53
|
-
*
|
|
54
|
-
* @param base64Data - A base64-encoded string representing the image
|
|
55
|
-
* @returns A Promise that resolves to a base64-encoded string representing the resized image
|
|
56
|
-
* @throws An error if the width or height cannot be determined from the metadata
|
|
57
|
-
*/
|
|
58
|
-
declare function resizeImg(base64Data: string): Promise<string>;
|
|
59
|
-
/**
|
|
60
|
-
* Calculates new dimensions for an image while maintaining its aspect ratio.
|
|
61
|
-
*
|
|
62
|
-
* This function is designed to resize an image to fit within a specified maximum width and height
|
|
63
|
-
* while maintaining the original aspect ratio. If the original width or height exceeds the maximum
|
|
64
|
-
* dimensions, the image will be scaled down to fit.
|
|
65
|
-
*
|
|
66
|
-
* @param {number} originalWidth - The original width of the image.
|
|
67
|
-
* @param {number} originalHeight - The original height of the image.
|
|
68
|
-
* @returns {Object} An object containing the new width and height.
|
|
69
|
-
* @throws {Error} Throws an error if the width or height is not a positive number.
|
|
70
|
-
*/
|
|
71
|
-
declare function calculateNewDimensions(originalWidth: number, originalHeight: number): {
|
|
72
|
-
width: number;
|
|
73
|
-
height: number;
|
|
74
|
-
};
|
|
75
|
-
/**
|
|
76
|
-
* Trims an image and returns the trimming information, including the offset from the left and top edges, and the trimmed width and height
|
|
77
|
-
*
|
|
78
|
-
* @param image - The image to be trimmed. This can be a file path or a Buffer object containing the image data
|
|
79
|
-
* @returns A Promise that resolves to an object containing the trimming information. If the image does not need to be trimmed, this object will be null
|
|
80
|
-
*/
|
|
81
|
-
declare function trimImage(image: string | Buffer): Promise<{
|
|
82
|
-
trimOffsetLeft: number;
|
|
83
|
-
trimOffsetTop: number;
|
|
84
|
-
width: number;
|
|
85
|
-
height: number;
|
|
86
|
-
} | null>;
|
|
87
|
-
/**
|
|
88
|
-
* Aligns an image's coordinate system based on trimming information
|
|
89
|
-
*
|
|
90
|
-
* This function takes an image and a center rectangle as input. It first extracts the center
|
|
91
|
-
* rectangle from the image using the Sharp library and converts it to a buffer. Then, it calls
|
|
92
|
-
* the trimImage function to obtain the trimming information of the buffer image. If there is no
|
|
93
|
-
* trimming information, the original center rectangle is returned. If there is trimming information,
|
|
94
|
-
* a new rectangle is created based on the trimming information, with its top-left corner
|
|
95
|
-
* positioned at the negative offset of the trimming from the original center rectangle's top-left
|
|
96
|
-
* corner, and its width and height set to the trimmed image's dimensions.
|
|
97
|
-
*
|
|
98
|
-
* @param image The image file path or buffer to be processed
|
|
99
|
-
* @param center The center rectangle of the image, which is used to extract and align
|
|
100
|
-
* @returns A Promise that resolves to a rectangle object representing the aligned coordinates
|
|
101
|
-
* @throws Error if there is an error during image processing
|
|
102
|
-
*/
|
|
103
|
-
declare function alignCoordByTrim(image: string | Buffer, centerRect: Rect): Promise<Rect>;
|
|
104
|
-
|
|
105
|
-
/**
|
|
106
|
-
* Composes a section diagram based on the given sections and context
|
|
107
|
-
* It creates an SVG representation of the sections and converts it to a PNG image file
|
|
108
|
-
*
|
|
109
|
-
* @param sections - An array of UISection objects representing the sections to be included in the diagram
|
|
110
|
-
* @param context - The UIContext object containing the size information for the diagram
|
|
111
|
-
* @returns {Promise<{ file: string; sectionNameColorMap: Record<string, Color>; }>}
|
|
112
|
-
*/
|
|
113
|
-
declare function composeSectionDiagram(sections: UISection[], context: UIContext): Promise<{
|
|
114
|
-
file: string;
|
|
115
|
-
sectionNameColorMap: Record<string, Color>;
|
|
116
|
-
}>;
|
|
117
|
-
|
|
118
|
-
export { alignCoordByTrim, base64Encoded, calculateNewDimensions, composeSectionDiagram, imageInfo, imageInfoOfBase64, resizeImg, saveBase64Image, transformImgPathToBase64, trimImage };
|
|
1
|
+
export { base64Encoded, calculateNewDimensions, imageInfo, imageInfoOfBase64, resizeImg, saveBase64Image, transformImgPathToBase64 } from '@midscene/shared/img';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { B as BaseElement, U as UIContext,
|
|
1
|
+
import { B as BaseElement, U as UIContext, P as PlanningAction } from './types-ad0dfcb1.js';
|
|
2
2
|
import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
|
|
3
3
|
|
|
4
4
|
type AIArgs = [
|