@midscene/web 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/debug.js +47 -117
- package/dist/es/index.js +45 -118
- package/dist/es/playwright-report.js +1 -4
- package/dist/es/playwright.js +4 -7
- package/dist/es/puppeteer.js +4 -7
- package/dist/lib/debug.js +46 -120
- package/dist/lib/index.js +46 -124
- package/dist/lib/playwright-report.js +1 -1
- package/dist/lib/playwright.js +7 -7
- package/dist/lib/puppeteer.js +7 -7
- package/dist/script/htmlElement.js +35 -12
- package/dist/script/htmlElementDebug.js +35 -12
- package/dist/types/debug.d.ts +48 -1
- package/dist/types/index.d.ts +2 -1
- package/dist/types/playwright.d.ts +2 -1
- package/dist/types/puppeteer.d.ts +2 -1
- package/dist/types/{tasks-75eae15e.d.ts → tasks-ce3a7499.d.ts} +1 -7
- package/package.json +4 -4
- package/dist/script/types/htmlElement.d.ts +0 -30
- package/dist/script/types/htmlElementDebug.d.ts +0 -2
package/dist/lib/debug.js
CHANGED
|
@@ -330,106 +330,29 @@ var debug_exports = {};
|
|
|
330
330
|
__export(debug_exports, {
|
|
331
331
|
generateExtractData: () => generateExtractData,
|
|
332
332
|
generateTestDataPath: () => generateTestDataPath,
|
|
333
|
+
getElementInfos: () => getElementInfos,
|
|
333
334
|
writeFileSyncWithDir: () => writeFileSyncWithDir
|
|
334
335
|
});
|
|
335
336
|
module.exports = __toCommonJS(debug_exports);
|
|
336
337
|
var import_node_fs2 = require("fs");
|
|
337
338
|
var import_node_path2 = __toESM(require("path"));
|
|
338
|
-
var import_image2 = require("@midscene/core/image");
|
|
339
339
|
|
|
340
|
-
// src/
|
|
341
|
-
var
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
var
|
|
345
|
-
let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
|
|
346
|
-
const colors = [
|
|
347
|
-
{ rect: "blue", text: "white" },
|
|
348
|
-
{ rect: "green", text: "white" }
|
|
349
|
-
];
|
|
350
|
-
svgContent += "<defs>";
|
|
351
|
-
elements.forEach((element, index) => {
|
|
352
|
-
svgContent += `
|
|
353
|
-
<clipPath id="clip${index}">
|
|
354
|
-
<rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
|
|
355
|
-
</clipPath>
|
|
356
|
-
`;
|
|
357
|
-
});
|
|
358
|
-
svgContent += "</defs>";
|
|
359
|
-
elements.forEach((element, index) => {
|
|
360
|
-
const textWidth = element.label.length * 8;
|
|
361
|
-
const textHeight = 12;
|
|
362
|
-
const rectWidth = textWidth + 5;
|
|
363
|
-
const rectHeight = textHeight + 4;
|
|
364
|
-
let rectX = element.x - rectWidth;
|
|
365
|
-
let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
|
|
366
|
-
let textX = rectX + rectWidth / 2;
|
|
367
|
-
let textY = rectY + rectHeight / 2 + 6;
|
|
368
|
-
if (rectX < 0) {
|
|
369
|
-
rectX = element.x;
|
|
370
|
-
rectY = element.y - rectHeight;
|
|
371
|
-
textX = rectX + rectWidth / 2;
|
|
372
|
-
textY = rectY + rectHeight / 2 + 6;
|
|
373
|
-
}
|
|
374
|
-
const color = colors[index % colors.length];
|
|
375
|
-
svgContent += `
|
|
376
|
-
<rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}"
|
|
377
|
-
style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
|
|
378
|
-
<rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
|
|
379
|
-
<text x="${textX}" y="${textY}"
|
|
380
|
-
text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
|
|
381
|
-
${element.label}
|
|
382
|
-
</text>
|
|
383
|
-
`;
|
|
384
|
-
});
|
|
385
|
-
svgContent += "</svg>";
|
|
386
|
-
return import_node_buffer.Buffer.from(svgContent);
|
|
387
|
-
};
|
|
388
|
-
var processImageElementInfo = async (options) => {
|
|
389
|
-
const base64Image = options.inputImgBase64.split(";base64,").pop();
|
|
390
|
-
(0, import_node_assert.default)(base64Image, "base64Image is undefined");
|
|
391
|
-
const imageBuffer = import_node_buffer.Buffer.from(base64Image, "base64");
|
|
392
|
-
const metadata = await (0, import_sharp.default)(imageBuffer).metadata();
|
|
393
|
-
const { width, height } = metadata;
|
|
394
|
-
if (width && height) {
|
|
395
|
-
const svgOverlay = createSvgOverlay(
|
|
396
|
-
options.elementsPositionInfo,
|
|
397
|
-
width,
|
|
398
|
-
height
|
|
399
|
-
);
|
|
400
|
-
const svgOverlayWithoutText = createSvgOverlay(
|
|
401
|
-
options.elementsPositionInfoWithoutText,
|
|
402
|
-
width,
|
|
403
|
-
height
|
|
404
|
-
);
|
|
405
|
-
const compositeElementInfoImgBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
|
|
406
|
-
return data.toString("base64");
|
|
407
|
-
}).catch((err) => {
|
|
408
|
-
throw err;
|
|
409
|
-
});
|
|
410
|
-
const compositeElementInfoImgWithoutTextBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
|
|
411
|
-
return data.toString("base64");
|
|
412
|
-
}).catch((err) => {
|
|
413
|
-
throw err;
|
|
414
|
-
});
|
|
415
|
-
return {
|
|
416
|
-
compositeElementInfoImgBase64,
|
|
417
|
-
compositeElementInfoImgWithoutTextBase64
|
|
418
|
-
};
|
|
419
|
-
}
|
|
420
|
-
throw Error("Image processing failed because width or height is undefined");
|
|
421
|
-
};
|
|
340
|
+
// src/extractor/constants.ts
|
|
341
|
+
var import_constants = require("@midscene/shared/constants");
|
|
342
|
+
|
|
343
|
+
// src/debug/index.ts
|
|
344
|
+
var import_img2 = require("@midscene/shared/img");
|
|
422
345
|
|
|
423
346
|
// src/common/utils.ts
|
|
424
|
-
var
|
|
347
|
+
var import_node_assert = __toESM(require("assert"));
|
|
425
348
|
var import_node_fs = __toESM(require("fs"));
|
|
426
349
|
var import_node_path = __toESM(require("path"));
|
|
427
|
-
var import_image = require("@midscene/core/image");
|
|
428
350
|
var import_utils = require("@midscene/core/utils");
|
|
351
|
+
var import_img = require("@midscene/shared/img");
|
|
429
352
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
430
353
|
async function getElementInfosFromPage(page) {
|
|
431
354
|
const pathDir = findNearestPackageJson(__dirname);
|
|
432
|
-
(0,
|
|
355
|
+
(0, import_node_assert.default)(pathDir, `can't find pathDir, with ${__dirname}`);
|
|
433
356
|
const scriptPath = import_node_path.default.join(pathDir, "./dist/script/htmlElement.js");
|
|
434
357
|
const elementInfosScriptContent = (0, import_node_fs.readFileSync)(scriptPath, "utf-8");
|
|
435
358
|
const extraReturnLogic = `${elementInfosScriptContent}midscene_element_inspector.extractTextWithPosition()`;
|
|
@@ -448,34 +371,6 @@ function findNearestPackageJson(dir) {
|
|
|
448
371
|
return findNearestPackageJson(parentDir);
|
|
449
372
|
}
|
|
450
373
|
|
|
451
|
-
// src/debug/img/util.ts
|
|
452
|
-
async function getElementInfos(page) {
|
|
453
|
-
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
454
|
-
const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
|
|
455
|
-
return {
|
|
456
|
-
label: elementInfo.indexId.toString(),
|
|
457
|
-
x: elementInfo.rect.left,
|
|
458
|
-
y: elementInfo.rect.top,
|
|
459
|
-
width: elementInfo.rect.width,
|
|
460
|
-
height: elementInfo.rect.height,
|
|
461
|
-
attributes: elementInfo.attributes
|
|
462
|
-
};
|
|
463
|
-
});
|
|
464
|
-
const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
|
|
465
|
-
(elementInfo) => {
|
|
466
|
-
if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
|
|
467
|
-
return false;
|
|
468
|
-
}
|
|
469
|
-
return true;
|
|
470
|
-
}
|
|
471
|
-
);
|
|
472
|
-
return {
|
|
473
|
-
elementsPositionInfo,
|
|
474
|
-
captureElementSnapshot,
|
|
475
|
-
elementsPositionInfoWithoutText
|
|
476
|
-
};
|
|
477
|
-
}
|
|
478
|
-
|
|
479
374
|
// src/debug/index.ts
|
|
480
375
|
async function generateExtractData(page, targetDir, saveImgType) {
|
|
481
376
|
const buffer = await page.screenshot({
|
|
@@ -495,15 +390,19 @@ async function generateExtractData(page, targetDir, saveImgType) {
|
|
|
495
390
|
);
|
|
496
391
|
const resizeOutputImgPath = import_node_path2.default.join(targetDir, "resize-output.png");
|
|
497
392
|
const snapshotJsonPath = import_node_path2.default.join(targetDir, "element-snapshot.json");
|
|
393
|
+
const startTime = Date.now();
|
|
498
394
|
const {
|
|
499
395
|
compositeElementInfoImgBase64,
|
|
500
396
|
compositeElementInfoImgWithoutTextBase64
|
|
501
|
-
} = await processImageElementInfo({
|
|
397
|
+
} = await (0, import_img2.processImageElementInfo)({
|
|
502
398
|
elementsPositionInfo,
|
|
503
399
|
elementsPositionInfoWithoutText,
|
|
504
400
|
inputImgBase64
|
|
505
401
|
});
|
|
506
|
-
const
|
|
402
|
+
const endTime = Date.now();
|
|
403
|
+
const executionTime = (endTime - startTime) / 1e3;
|
|
404
|
+
console.log(`Execution time: ${executionTime.toFixed(2)}s`);
|
|
405
|
+
const resizeImgBase64 = await (0, import_img2.resizeImg)(inputImgBase64);
|
|
507
406
|
if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
|
|
508
407
|
writeFileSyncWithDir(
|
|
509
408
|
snapshotJsonPath,
|
|
@@ -511,25 +410,25 @@ async function generateExtractData(page, targetDir, saveImgType) {
|
|
|
511
410
|
);
|
|
512
411
|
}
|
|
513
412
|
if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
|
|
514
|
-
await (0,
|
|
413
|
+
await (0, import_img2.saveBase64Image)({
|
|
515
414
|
base64Data: inputImgBase64,
|
|
516
415
|
outputPath: inputImagePath
|
|
517
416
|
});
|
|
518
417
|
}
|
|
519
418
|
if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
|
|
520
|
-
await (0,
|
|
419
|
+
await (0, import_img2.saveBase64Image)({
|
|
521
420
|
base64Data: compositeElementInfoImgBase64,
|
|
522
421
|
outputPath: outputImagePath
|
|
523
422
|
});
|
|
524
423
|
}
|
|
525
424
|
if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
|
|
526
|
-
await (0,
|
|
425
|
+
await (0, import_img2.saveBase64Image)({
|
|
527
426
|
base64Data: compositeElementInfoImgWithoutTextBase64,
|
|
528
427
|
outputPath: outputWithoutTextImgPath
|
|
529
428
|
});
|
|
530
429
|
}
|
|
531
430
|
if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
|
|
532
|
-
await (0,
|
|
431
|
+
await (0, import_img2.saveBase64Image)({
|
|
533
432
|
base64Data: resizeImgBase64,
|
|
534
433
|
outputPath: resizeOutputImgPath
|
|
535
434
|
});
|
|
@@ -555,9 +454,36 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
|
|
|
555
454
|
ensureDirectoryExistence(filePath);
|
|
556
455
|
(0, import_node_fs2.writeFileSync)(filePath, content, options);
|
|
557
456
|
}
|
|
457
|
+
async function getElementInfos(page) {
|
|
458
|
+
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
459
|
+
const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
|
|
460
|
+
return {
|
|
461
|
+
label: elementInfo.indexId.toString(),
|
|
462
|
+
x: elementInfo.rect.left,
|
|
463
|
+
y: elementInfo.rect.top,
|
|
464
|
+
width: elementInfo.rect.width,
|
|
465
|
+
height: elementInfo.rect.height,
|
|
466
|
+
attributes: elementInfo.attributes
|
|
467
|
+
};
|
|
468
|
+
});
|
|
469
|
+
const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
|
|
470
|
+
(elementInfo) => {
|
|
471
|
+
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
472
|
+
return false;
|
|
473
|
+
}
|
|
474
|
+
return true;
|
|
475
|
+
}
|
|
476
|
+
);
|
|
477
|
+
return {
|
|
478
|
+
elementsPositionInfo,
|
|
479
|
+
captureElementSnapshot,
|
|
480
|
+
elementsPositionInfoWithoutText
|
|
481
|
+
};
|
|
482
|
+
}
|
|
558
483
|
// Annotate the CommonJS export names for ESM import in node:
|
|
559
484
|
0 && (module.exports = {
|
|
560
485
|
generateExtractData,
|
|
561
486
|
generateTestDataPath,
|
|
487
|
+
getElementInfos,
|
|
562
488
|
writeFileSyncWithDir
|
|
563
489
|
});
|
package/dist/lib/index.js
CHANGED
|
@@ -360,8 +360,8 @@ var import_utils4 = require("@midscene/core/utils");
|
|
|
360
360
|
// src/common/tasks.ts
|
|
361
361
|
var import_node_assert2 = __toESM(require("assert"));
|
|
362
362
|
var import_core = __toESM(require("@midscene/core"));
|
|
363
|
-
var import_image2 = require("@midscene/core/image");
|
|
364
363
|
var import_utils2 = require("@midscene/core/utils");
|
|
364
|
+
var import_img2 = require("@midscene/shared/img");
|
|
365
365
|
|
|
366
366
|
// src/common/task-cache.ts
|
|
367
367
|
var TaskCache = class {
|
|
@@ -422,8 +422,8 @@ var TaskCache = class {
|
|
|
422
422
|
var import_node_assert = __toESM(require("assert"));
|
|
423
423
|
var import_node_fs = __toESM(require("fs"));
|
|
424
424
|
var import_node_path = __toESM(require("path"));
|
|
425
|
-
var import_image = require("@midscene/core/image");
|
|
426
425
|
var import_utils = require("@midscene/core/utils");
|
|
426
|
+
var import_img = require("@midscene/shared/img");
|
|
427
427
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
428
428
|
|
|
429
429
|
// src/web-element.ts
|
|
@@ -456,14 +456,14 @@ async function parseContextFromWebPage(page, _opt) {
|
|
|
456
456
|
const file = (0, import_utils.getTmpFile)("jpeg");
|
|
457
457
|
await page.screenshot({ path: file, type: "jpeg", quality: 75 });
|
|
458
458
|
const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
|
|
459
|
-
const screenshotBase64 = (0,
|
|
459
|
+
const screenshotBase64 = (0, import_img.base64Encoded)(file);
|
|
460
460
|
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
461
461
|
const elementsInfo = await alignElements(
|
|
462
462
|
screenshotBuffer,
|
|
463
463
|
captureElementSnapshot,
|
|
464
464
|
page
|
|
465
465
|
);
|
|
466
|
-
const size = await (0,
|
|
466
|
+
const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
467
467
|
return {
|
|
468
468
|
content: elementsInfo,
|
|
469
469
|
size,
|
|
@@ -482,10 +482,10 @@ async function getElementInfosFromPage(page) {
|
|
|
482
482
|
}
|
|
483
483
|
var sizeThreshold = 3;
|
|
484
484
|
async function alignElements(screenshotBuffer, elements, page) {
|
|
485
|
-
const textsAligned = [];
|
|
486
485
|
const validElements = elements.filter((item) => {
|
|
487
486
|
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
488
487
|
});
|
|
488
|
+
const textsAligned = [];
|
|
489
489
|
for (const item of validElements) {
|
|
490
490
|
const { rect, id, content, attributes, locator } = item;
|
|
491
491
|
textsAligned.push(
|
|
@@ -537,7 +537,7 @@ var PageTaskExecutor = class {
|
|
|
537
537
|
const item = {
|
|
538
538
|
type: "screenshot",
|
|
539
539
|
ts: Date.now(),
|
|
540
|
-
screenshot: (0,
|
|
540
|
+
screenshot: (0, import_img2.base64Encoded)(file),
|
|
541
541
|
timing
|
|
542
542
|
};
|
|
543
543
|
return item;
|
|
@@ -1018,7 +1018,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
|
1018
1018
|
this.writeOutActionDumps();
|
|
1019
1019
|
if (!(output == null ? void 0 : output.pass)) {
|
|
1020
1020
|
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1021
|
-
const reasonMsg = `Reason: ${output == null ? void 0 : output.thought
|
|
1021
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1022
1022
|
throw new Error(`${errMsg}
|
|
1023
1023
|
${reasonMsg}`);
|
|
1024
1024
|
}
|
|
@@ -1245,120 +1245,12 @@ var PlaywrightAiFixture = () => {
|
|
|
1245
1245
|
// src/debug/index.ts
|
|
1246
1246
|
var import_node_fs3 = require("fs");
|
|
1247
1247
|
var import_node_path3 = __toESM(require("path"));
|
|
1248
|
-
var import_image3 = require("@midscene/core/image");
|
|
1249
1248
|
|
|
1250
|
-
// src/
|
|
1251
|
-
var
|
|
1252
|
-
var import_node_buffer = require("buffer");
|
|
1253
|
-
var import_sharp = __toESM(require("sharp"));
|
|
1254
|
-
var createSvgOverlay = (elements, imageWidth, imageHeight) => {
|
|
1255
|
-
let svgContent = `<svg width="${imageWidth}" height="${imageHeight}" xmlns="http://www.w3.org/2000/svg">`;
|
|
1256
|
-
const colors = [
|
|
1257
|
-
{ rect: "blue", text: "white" },
|
|
1258
|
-
{ rect: "green", text: "white" }
|
|
1259
|
-
];
|
|
1260
|
-
svgContent += "<defs>";
|
|
1261
|
-
elements.forEach((element, index) => {
|
|
1262
|
-
svgContent += `
|
|
1263
|
-
<clipPath id="clip${index}">
|
|
1264
|
-
<rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}" />
|
|
1265
|
-
</clipPath>
|
|
1266
|
-
`;
|
|
1267
|
-
});
|
|
1268
|
-
svgContent += "</defs>";
|
|
1269
|
-
elements.forEach((element, index) => {
|
|
1270
|
-
const textWidth = element.label.length * 8;
|
|
1271
|
-
const textHeight = 12;
|
|
1272
|
-
const rectWidth = textWidth + 5;
|
|
1273
|
-
const rectHeight = textHeight + 4;
|
|
1274
|
-
let rectX = element.x - rectWidth;
|
|
1275
|
-
let rectY = element.y + element.height / 2 - textHeight / 2 - 2;
|
|
1276
|
-
let textX = rectX + rectWidth / 2;
|
|
1277
|
-
let textY = rectY + rectHeight / 2 + 6;
|
|
1278
|
-
if (rectX < 0) {
|
|
1279
|
-
rectX = element.x;
|
|
1280
|
-
rectY = element.y - rectHeight;
|
|
1281
|
-
textX = rectX + rectWidth / 2;
|
|
1282
|
-
textY = rectY + rectHeight / 2 + 6;
|
|
1283
|
-
}
|
|
1284
|
-
const color = colors[index % colors.length];
|
|
1285
|
-
svgContent += `
|
|
1286
|
-
<rect x="${element.x}" y="${element.y}" width="${element.width}" height="${element.height}"
|
|
1287
|
-
style="fill:none;stroke:${color.rect};stroke-width:4" clip-path="url(#clip${index})" />
|
|
1288
|
-
<rect x="${rectX}" y="${rectY}" width="${rectWidth}" height="${rectHeight}" style="fill:${color.rect};" />
|
|
1289
|
-
<text x="${textX}" y="${textY}"
|
|
1290
|
-
text-anchor="middle" dominant-baseline="middle" style="fill:${color.text};font-size:12px;font-weight:bold;">
|
|
1291
|
-
${element.label}
|
|
1292
|
-
</text>
|
|
1293
|
-
`;
|
|
1294
|
-
});
|
|
1295
|
-
svgContent += "</svg>";
|
|
1296
|
-
return import_node_buffer.Buffer.from(svgContent);
|
|
1297
|
-
};
|
|
1298
|
-
var processImageElementInfo = async (options) => {
|
|
1299
|
-
const base64Image = options.inputImgBase64.split(";base64,").pop();
|
|
1300
|
-
(0, import_node_assert3.default)(base64Image, "base64Image is undefined");
|
|
1301
|
-
const imageBuffer = import_node_buffer.Buffer.from(base64Image, "base64");
|
|
1302
|
-
const metadata = await (0, import_sharp.default)(imageBuffer).metadata();
|
|
1303
|
-
const { width, height } = metadata;
|
|
1304
|
-
if (width && height) {
|
|
1305
|
-
const svgOverlay = createSvgOverlay(
|
|
1306
|
-
options.elementsPositionInfo,
|
|
1307
|
-
width,
|
|
1308
|
-
height
|
|
1309
|
-
);
|
|
1310
|
-
const svgOverlayWithoutText = createSvgOverlay(
|
|
1311
|
-
options.elementsPositionInfoWithoutText,
|
|
1312
|
-
width,
|
|
1313
|
-
height
|
|
1314
|
-
);
|
|
1315
|
-
const compositeElementInfoImgBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlay, blend: "over" }]).toBuffer().then((data) => {
|
|
1316
|
-
return data.toString("base64");
|
|
1317
|
-
}).catch((err) => {
|
|
1318
|
-
throw err;
|
|
1319
|
-
});
|
|
1320
|
-
const compositeElementInfoImgWithoutTextBase64 = await (0, import_sharp.default)(imageBuffer).composite([{ input: svgOverlayWithoutText, blend: "over" }]).toBuffer().then((data) => {
|
|
1321
|
-
return data.toString("base64");
|
|
1322
|
-
}).catch((err) => {
|
|
1323
|
-
throw err;
|
|
1324
|
-
});
|
|
1325
|
-
return {
|
|
1326
|
-
compositeElementInfoImgBase64,
|
|
1327
|
-
compositeElementInfoImgWithoutTextBase64
|
|
1328
|
-
};
|
|
1329
|
-
}
|
|
1330
|
-
throw Error("Image processing failed because width or height is undefined");
|
|
1331
|
-
};
|
|
1332
|
-
|
|
1333
|
-
// src/debug/img/util.ts
|
|
1334
|
-
async function getElementInfos(page) {
|
|
1335
|
-
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
1336
|
-
const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
|
|
1337
|
-
return {
|
|
1338
|
-
label: elementInfo.indexId.toString(),
|
|
1339
|
-
x: elementInfo.rect.left,
|
|
1340
|
-
y: elementInfo.rect.top,
|
|
1341
|
-
width: elementInfo.rect.width,
|
|
1342
|
-
height: elementInfo.rect.height,
|
|
1343
|
-
attributes: elementInfo.attributes
|
|
1344
|
-
};
|
|
1345
|
-
});
|
|
1346
|
-
const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
|
|
1347
|
-
(elementInfo) => {
|
|
1348
|
-
if (elementInfo.attributes.nodeType === "TEXT Node" /* TEXT */) {
|
|
1349
|
-
return false;
|
|
1350
|
-
}
|
|
1351
|
-
return true;
|
|
1352
|
-
}
|
|
1353
|
-
);
|
|
1354
|
-
return {
|
|
1355
|
-
elementsPositionInfo,
|
|
1356
|
-
captureElementSnapshot,
|
|
1357
|
-
elementsPositionInfoWithoutText
|
|
1358
|
-
};
|
|
1359
|
-
}
|
|
1249
|
+
// src/extractor/constants.ts
|
|
1250
|
+
var import_constants = require("@midscene/shared/constants");
|
|
1360
1251
|
|
|
1361
1252
|
// src/debug/index.ts
|
|
1253
|
+
var import_img3 = require("@midscene/shared/img");
|
|
1362
1254
|
async function generateExtractData(page, targetDir, saveImgType) {
|
|
1363
1255
|
const buffer = await page.screenshot({
|
|
1364
1256
|
encoding: "base64"
|
|
@@ -1377,15 +1269,19 @@ async function generateExtractData(page, targetDir, saveImgType) {
|
|
|
1377
1269
|
);
|
|
1378
1270
|
const resizeOutputImgPath = import_node_path3.default.join(targetDir, "resize-output.png");
|
|
1379
1271
|
const snapshotJsonPath = import_node_path3.default.join(targetDir, "element-snapshot.json");
|
|
1272
|
+
const startTime = Date.now();
|
|
1380
1273
|
const {
|
|
1381
1274
|
compositeElementInfoImgBase64,
|
|
1382
1275
|
compositeElementInfoImgWithoutTextBase64
|
|
1383
|
-
} = await processImageElementInfo({
|
|
1276
|
+
} = await (0, import_img3.processImageElementInfo)({
|
|
1384
1277
|
elementsPositionInfo,
|
|
1385
1278
|
elementsPositionInfoWithoutText,
|
|
1386
1279
|
inputImgBase64
|
|
1387
1280
|
});
|
|
1388
|
-
const
|
|
1281
|
+
const endTime = Date.now();
|
|
1282
|
+
const executionTime = (endTime - startTime) / 1e3;
|
|
1283
|
+
console.log(`Execution time: ${executionTime.toFixed(2)}s`);
|
|
1284
|
+
const resizeImgBase64 = await (0, import_img3.resizeImg)(inputImgBase64);
|
|
1389
1285
|
if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
|
|
1390
1286
|
writeFileSyncWithDir(
|
|
1391
1287
|
snapshotJsonPath,
|
|
@@ -1393,25 +1289,25 @@ async function generateExtractData(page, targetDir, saveImgType) {
|
|
|
1393
1289
|
);
|
|
1394
1290
|
}
|
|
1395
1291
|
if (!(saveImgType == null ? void 0 : saveImgType.disableInputImage)) {
|
|
1396
|
-
await (0,
|
|
1292
|
+
await (0, import_img3.saveBase64Image)({
|
|
1397
1293
|
base64Data: inputImgBase64,
|
|
1398
1294
|
outputPath: inputImagePath
|
|
1399
1295
|
});
|
|
1400
1296
|
}
|
|
1401
1297
|
if (!(saveImgType == null ? void 0 : saveImgType.disableOutputImage)) {
|
|
1402
|
-
await (0,
|
|
1298
|
+
await (0, import_img3.saveBase64Image)({
|
|
1403
1299
|
base64Data: compositeElementInfoImgBase64,
|
|
1404
1300
|
outputPath: outputImagePath
|
|
1405
1301
|
});
|
|
1406
1302
|
}
|
|
1407
1303
|
if (!(saveImgType == null ? void 0 : saveImgType.disableOutputWithoutTextImg)) {
|
|
1408
|
-
await (0,
|
|
1304
|
+
await (0, import_img3.saveBase64Image)({
|
|
1409
1305
|
base64Data: compositeElementInfoImgWithoutTextBase64,
|
|
1410
1306
|
outputPath: outputWithoutTextImgPath
|
|
1411
1307
|
});
|
|
1412
1308
|
}
|
|
1413
1309
|
if (!(saveImgType == null ? void 0 : saveImgType.disableResizeOutputImg)) {
|
|
1414
|
-
await (0,
|
|
1310
|
+
await (0, import_img3.saveBase64Image)({
|
|
1415
1311
|
base64Data: resizeImgBase64,
|
|
1416
1312
|
outputPath: resizeOutputImgPath
|
|
1417
1313
|
});
|
|
@@ -1429,6 +1325,32 @@ function writeFileSyncWithDir(filePath, content, options = {}) {
|
|
|
1429
1325
|
ensureDirectoryExistence(filePath);
|
|
1430
1326
|
(0, import_node_fs3.writeFileSync)(filePath, content, options);
|
|
1431
1327
|
}
|
|
1328
|
+
async function getElementInfos(page) {
|
|
1329
|
+
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
1330
|
+
const elementsPositionInfo = captureElementSnapshot.map((elementInfo) => {
|
|
1331
|
+
return {
|
|
1332
|
+
label: elementInfo.indexId.toString(),
|
|
1333
|
+
x: elementInfo.rect.left,
|
|
1334
|
+
y: elementInfo.rect.top,
|
|
1335
|
+
width: elementInfo.rect.width,
|
|
1336
|
+
height: elementInfo.rect.height,
|
|
1337
|
+
attributes: elementInfo.attributes
|
|
1338
|
+
};
|
|
1339
|
+
});
|
|
1340
|
+
const elementsPositionInfoWithoutText = elementsPositionInfo.filter(
|
|
1341
|
+
(elementInfo) => {
|
|
1342
|
+
if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
|
|
1343
|
+
return false;
|
|
1344
|
+
}
|
|
1345
|
+
return true;
|
|
1346
|
+
}
|
|
1347
|
+
);
|
|
1348
|
+
return {
|
|
1349
|
+
elementsPositionInfo,
|
|
1350
|
+
captureElementSnapshot,
|
|
1351
|
+
elementsPositionInfoWithoutText
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1432
1354
|
// Annotate the CommonJS export names for ESM import in node:
|
|
1433
1355
|
0 && (module.exports = {
|
|
1434
1356
|
PlaywrightAiFixture,
|
|
@@ -336,8 +336,8 @@ module.exports = __toCommonJS(reporter_exports);
|
|
|
336
336
|
var import_node_assert = __toESM(require("assert"));
|
|
337
337
|
var import_node_fs = __toESM(require("fs"));
|
|
338
338
|
var import_node_path = __toESM(require("path"));
|
|
339
|
-
var import_image = require("@midscene/core/image");
|
|
340
339
|
var import_utils = require("@midscene/core/utils");
|
|
340
|
+
var import_img = require("@midscene/shared/img");
|
|
341
341
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
342
342
|
function reportFileName(tag = "web") {
|
|
343
343
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
package/dist/lib/playwright.js
CHANGED
|
@@ -358,8 +358,8 @@ var import_utils4 = require("@midscene/core/utils");
|
|
|
358
358
|
// src/common/tasks.ts
|
|
359
359
|
var import_node_assert2 = __toESM(require("assert"));
|
|
360
360
|
var import_core = __toESM(require("@midscene/core"));
|
|
361
|
-
var import_image2 = require("@midscene/core/image");
|
|
362
361
|
var import_utils2 = require("@midscene/core/utils");
|
|
362
|
+
var import_img2 = require("@midscene/shared/img");
|
|
363
363
|
|
|
364
364
|
// src/common/task-cache.ts
|
|
365
365
|
var TaskCache = class {
|
|
@@ -420,8 +420,8 @@ var TaskCache = class {
|
|
|
420
420
|
var import_node_assert = __toESM(require("assert"));
|
|
421
421
|
var import_node_fs = __toESM(require("fs"));
|
|
422
422
|
var import_node_path = __toESM(require("path"));
|
|
423
|
-
var import_image = require("@midscene/core/image");
|
|
424
423
|
var import_utils = require("@midscene/core/utils");
|
|
424
|
+
var import_img = require("@midscene/shared/img");
|
|
425
425
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
426
426
|
|
|
427
427
|
// src/web-element.ts
|
|
@@ -454,14 +454,14 @@ async function parseContextFromWebPage(page, _opt) {
|
|
|
454
454
|
const file = (0, import_utils.getTmpFile)("jpeg");
|
|
455
455
|
await page.screenshot({ path: file, type: "jpeg", quality: 75 });
|
|
456
456
|
const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
|
|
457
|
-
const screenshotBase64 = (0,
|
|
457
|
+
const screenshotBase64 = (0, import_img.base64Encoded)(file);
|
|
458
458
|
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
459
459
|
const elementsInfo = await alignElements(
|
|
460
460
|
screenshotBuffer,
|
|
461
461
|
captureElementSnapshot,
|
|
462
462
|
page
|
|
463
463
|
);
|
|
464
|
-
const size = await (0,
|
|
464
|
+
const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
465
465
|
return {
|
|
466
466
|
content: elementsInfo,
|
|
467
467
|
size,
|
|
@@ -480,10 +480,10 @@ async function getElementInfosFromPage(page) {
|
|
|
480
480
|
}
|
|
481
481
|
var sizeThreshold = 3;
|
|
482
482
|
async function alignElements(screenshotBuffer, elements, page) {
|
|
483
|
-
const textsAligned = [];
|
|
484
483
|
const validElements = elements.filter((item) => {
|
|
485
484
|
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
486
485
|
});
|
|
486
|
+
const textsAligned = [];
|
|
487
487
|
for (const item of validElements) {
|
|
488
488
|
const { rect, id, content, attributes, locator } = item;
|
|
489
489
|
textsAligned.push(
|
|
@@ -535,7 +535,7 @@ var PageTaskExecutor = class {
|
|
|
535
535
|
const item = {
|
|
536
536
|
type: "screenshot",
|
|
537
537
|
ts: Date.now(),
|
|
538
|
-
screenshot: (0,
|
|
538
|
+
screenshot: (0, import_img2.base64Encoded)(file),
|
|
539
539
|
timing
|
|
540
540
|
};
|
|
541
541
|
return item;
|
|
@@ -1016,7 +1016,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
|
1016
1016
|
this.writeOutActionDumps();
|
|
1017
1017
|
if (!(output == null ? void 0 : output.pass)) {
|
|
1018
1018
|
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1019
|
-
const reasonMsg = `Reason: ${output == null ? void 0 : output.thought
|
|
1019
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1020
1020
|
throw new Error(`${errMsg}
|
|
1021
1021
|
${reasonMsg}`);
|
|
1022
1022
|
}
|
package/dist/lib/puppeteer.js
CHANGED
|
@@ -355,8 +355,8 @@ var import_utils4 = require("@midscene/core/utils");
|
|
|
355
355
|
// src/common/tasks.ts
|
|
356
356
|
var import_node_assert2 = __toESM(require("assert"));
|
|
357
357
|
var import_core = __toESM(require("@midscene/core"));
|
|
358
|
-
var import_image2 = require("@midscene/core/image");
|
|
359
358
|
var import_utils2 = require("@midscene/core/utils");
|
|
359
|
+
var import_img2 = require("@midscene/shared/img");
|
|
360
360
|
|
|
361
361
|
// src/common/task-cache.ts
|
|
362
362
|
var TaskCache = class {
|
|
@@ -417,8 +417,8 @@ var TaskCache = class {
|
|
|
417
417
|
var import_node_assert = __toESM(require("assert"));
|
|
418
418
|
var import_node_fs = __toESM(require("fs"));
|
|
419
419
|
var import_node_path = __toESM(require("path"));
|
|
420
|
-
var import_image = require("@midscene/core/image");
|
|
421
420
|
var import_utils = require("@midscene/core/utils");
|
|
421
|
+
var import_img = require("@midscene/shared/img");
|
|
422
422
|
var import_dayjs = __toESM(require_dayjs_min());
|
|
423
423
|
|
|
424
424
|
// src/web-element.ts
|
|
@@ -451,14 +451,14 @@ async function parseContextFromWebPage(page, _opt) {
|
|
|
451
451
|
const file = (0, import_utils.getTmpFile)("jpeg");
|
|
452
452
|
await page.screenshot({ path: file, type: "jpeg", quality: 75 });
|
|
453
453
|
const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
|
|
454
|
-
const screenshotBase64 = (0,
|
|
454
|
+
const screenshotBase64 = (0, import_img.base64Encoded)(file);
|
|
455
455
|
const captureElementSnapshot = await getElementInfosFromPage(page);
|
|
456
456
|
const elementsInfo = await alignElements(
|
|
457
457
|
screenshotBuffer,
|
|
458
458
|
captureElementSnapshot,
|
|
459
459
|
page
|
|
460
460
|
);
|
|
461
|
-
const size = await (0,
|
|
461
|
+
const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
|
|
462
462
|
return {
|
|
463
463
|
content: elementsInfo,
|
|
464
464
|
size,
|
|
@@ -477,10 +477,10 @@ async function getElementInfosFromPage(page) {
|
|
|
477
477
|
}
|
|
478
478
|
var sizeThreshold = 3;
|
|
479
479
|
async function alignElements(screenshotBuffer, elements, page) {
|
|
480
|
-
const textsAligned = [];
|
|
481
480
|
const validElements = elements.filter((item) => {
|
|
482
481
|
return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
|
|
483
482
|
});
|
|
483
|
+
const textsAligned = [];
|
|
484
484
|
for (const item of validElements) {
|
|
485
485
|
const { rect, id, content, attributes, locator } = item;
|
|
486
486
|
textsAligned.push(
|
|
@@ -532,7 +532,7 @@ var PageTaskExecutor = class {
|
|
|
532
532
|
const item = {
|
|
533
533
|
type: "screenshot",
|
|
534
534
|
ts: Date.now(),
|
|
535
|
-
screenshot: (0,
|
|
535
|
+
screenshot: (0, import_img2.base64Encoded)(file),
|
|
536
536
|
timing
|
|
537
537
|
};
|
|
538
538
|
return item;
|
|
@@ -1013,7 +1013,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
|
1013
1013
|
this.writeOutActionDumps();
|
|
1014
1014
|
if (!(output == null ? void 0 : output.pass)) {
|
|
1015
1015
|
const errMsg = msg || `Assertion failed: ${assertion}`;
|
|
1016
|
-
const reasonMsg = `Reason: ${output == null ? void 0 : output.thought
|
|
1016
|
+
const reasonMsg = `Reason: ${(output == null ? void 0 : output.thought) || "(no_reason)"}`;
|
|
1017
1017
|
throw new Error(`${errMsg}
|
|
1018
1018
|
${reasonMsg}`);
|
|
1019
1019
|
}
|