@midscene/web 0.9.1 → 0.9.2-beta-20250114083542.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/appium.js +222 -593
- package/dist/es/bridge-mode-browser.js +103 -45
- package/dist/es/bridge-mode.js +219 -39
- package/dist/es/chrome-extension.js +308 -70
- package/dist/es/index.js +224 -593
- package/dist/es/midscene-playground.js +215 -35
- package/dist/es/playground.js +215 -35
- package/dist/es/playwright.js +219 -39
- package/dist/es/puppeteer.js +219 -39
- package/dist/es/utils.js +16 -9
- package/dist/lib/appium.js +222 -593
- package/dist/lib/bridge-mode-browser.js +103 -45
- package/dist/lib/bridge-mode.js +219 -39
- package/dist/lib/chrome-extension.js +308 -70
- package/dist/lib/index.js +224 -593
- package/dist/lib/midscene-playground.js +215 -35
- package/dist/lib/playground.js +215 -35
- package/dist/lib/playwright.js +219 -39
- package/dist/lib/puppeteer.js +219 -39
- package/dist/lib/utils.js +16 -9
- package/dist/script/htmlElement.js +88 -65
- package/dist/script/htmlElementDebug.js +71 -58
- package/dist/types/appium.d.ts +4 -3
- package/dist/types/bridge-mode-browser.d.ts +2 -2
- package/dist/types/bridge-mode.d.ts +4 -3
- package/dist/types/chrome-extension.d.ts +5 -4
- package/dist/types/debug.d.ts +2 -2
- package/dist/types/index.d.ts +6 -4
- package/dist/types/{page-6eadc13a.d.ts → page-47198ebc.d.ts} +6 -4
- package/dist/types/playground.d.ts +5 -4
- package/dist/types/playwright.d.ts +6 -5
- package/dist/types/puppeteer.d.ts +5 -4
- package/dist/types/{tasks-b184955a.d.ts → tasks-7ce949d1.d.ts} +21 -4
- package/dist/types/{utils-187a5ff1.d.ts → utils-f90a139b.d.ts} +1 -1
- package/dist/types/utils.d.ts +2 -2
- package/dist/types/yaml.d.ts +5 -4
- package/package.json +3 -3
package/dist/es/appium.js
CHANGED
|
@@ -325,525 +325,6 @@ var require_dayjs_min = __commonJS({
|
|
|
325
325
|
}
|
|
326
326
|
});
|
|
327
327
|
|
|
328
|
-
// ../../node_modules/.pnpm/js-sha256@0.11.0/node_modules/js-sha256/src/sha256.js
|
|
329
|
-
var require_sha256 = __commonJS({
|
|
330
|
-
"../../node_modules/.pnpm/js-sha256@0.11.0/node_modules/js-sha256/src/sha256.js"(exports, module2) {
|
|
331
|
-
"use strict";
|
|
332
|
-
(function() {
|
|
333
|
-
"use strict";
|
|
334
|
-
var ERROR = "input is invalid type";
|
|
335
|
-
var WINDOW = typeof window === "object";
|
|
336
|
-
var root = WINDOW ? window : {};
|
|
337
|
-
if (root.JS_SHA256_NO_WINDOW) {
|
|
338
|
-
WINDOW = false;
|
|
339
|
-
}
|
|
340
|
-
var WEB_WORKER = !WINDOW && typeof self === "object";
|
|
341
|
-
var NODE_JS = !root.JS_SHA256_NO_NODE_JS && typeof process === "object" && process.versions && process.versions.node;
|
|
342
|
-
if (NODE_JS) {
|
|
343
|
-
root = global;
|
|
344
|
-
} else if (WEB_WORKER) {
|
|
345
|
-
root = self;
|
|
346
|
-
}
|
|
347
|
-
var COMMON_JS = !root.JS_SHA256_NO_COMMON_JS && typeof module2 === "object" && module2.exports;
|
|
348
|
-
var AMD = typeof define === "function" && define.amd;
|
|
349
|
-
var ARRAY_BUFFER = !root.JS_SHA256_NO_ARRAY_BUFFER && typeof ArrayBuffer !== "undefined";
|
|
350
|
-
var HEX_CHARS = "0123456789abcdef".split("");
|
|
351
|
-
var EXTRA = [-2147483648, 8388608, 32768, 128];
|
|
352
|
-
var SHIFT = [24, 16, 8, 0];
|
|
353
|
-
var K = [
|
|
354
|
-
1116352408,
|
|
355
|
-
1899447441,
|
|
356
|
-
3049323471,
|
|
357
|
-
3921009573,
|
|
358
|
-
961987163,
|
|
359
|
-
1508970993,
|
|
360
|
-
2453635748,
|
|
361
|
-
2870763221,
|
|
362
|
-
3624381080,
|
|
363
|
-
310598401,
|
|
364
|
-
607225278,
|
|
365
|
-
1426881987,
|
|
366
|
-
1925078388,
|
|
367
|
-
2162078206,
|
|
368
|
-
2614888103,
|
|
369
|
-
3248222580,
|
|
370
|
-
3835390401,
|
|
371
|
-
4022224774,
|
|
372
|
-
264347078,
|
|
373
|
-
604807628,
|
|
374
|
-
770255983,
|
|
375
|
-
1249150122,
|
|
376
|
-
1555081692,
|
|
377
|
-
1996064986,
|
|
378
|
-
2554220882,
|
|
379
|
-
2821834349,
|
|
380
|
-
2952996808,
|
|
381
|
-
3210313671,
|
|
382
|
-
3336571891,
|
|
383
|
-
3584528711,
|
|
384
|
-
113926993,
|
|
385
|
-
338241895,
|
|
386
|
-
666307205,
|
|
387
|
-
773529912,
|
|
388
|
-
1294757372,
|
|
389
|
-
1396182291,
|
|
390
|
-
1695183700,
|
|
391
|
-
1986661051,
|
|
392
|
-
2177026350,
|
|
393
|
-
2456956037,
|
|
394
|
-
2730485921,
|
|
395
|
-
2820302411,
|
|
396
|
-
3259730800,
|
|
397
|
-
3345764771,
|
|
398
|
-
3516065817,
|
|
399
|
-
3600352804,
|
|
400
|
-
4094571909,
|
|
401
|
-
275423344,
|
|
402
|
-
430227734,
|
|
403
|
-
506948616,
|
|
404
|
-
659060556,
|
|
405
|
-
883997877,
|
|
406
|
-
958139571,
|
|
407
|
-
1322822218,
|
|
408
|
-
1537002063,
|
|
409
|
-
1747873779,
|
|
410
|
-
1955562222,
|
|
411
|
-
2024104815,
|
|
412
|
-
2227730452,
|
|
413
|
-
2361852424,
|
|
414
|
-
2428436474,
|
|
415
|
-
2756734187,
|
|
416
|
-
3204031479,
|
|
417
|
-
3329325298
|
|
418
|
-
];
|
|
419
|
-
var OUTPUT_TYPES = ["hex", "array", "digest", "arrayBuffer"];
|
|
420
|
-
var blocks = [];
|
|
421
|
-
if (root.JS_SHA256_NO_NODE_JS || !Array.isArray) {
|
|
422
|
-
Array.isArray = function(obj) {
|
|
423
|
-
return Object.prototype.toString.call(obj) === "[object Array]";
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
if (ARRAY_BUFFER && (root.JS_SHA256_NO_ARRAY_BUFFER_IS_VIEW || !ArrayBuffer.isView)) {
|
|
427
|
-
ArrayBuffer.isView = function(obj) {
|
|
428
|
-
return typeof obj === "object" && obj.buffer && obj.buffer.constructor === ArrayBuffer;
|
|
429
|
-
};
|
|
430
|
-
}
|
|
431
|
-
var createOutputMethod = function(outputType, is224) {
|
|
432
|
-
return function(message) {
|
|
433
|
-
return new Sha256(is224, true).update(message)[outputType]();
|
|
434
|
-
};
|
|
435
|
-
};
|
|
436
|
-
var createMethod = function(is224) {
|
|
437
|
-
var method = createOutputMethod("hex", is224);
|
|
438
|
-
if (NODE_JS) {
|
|
439
|
-
method = nodeWrap(method, is224);
|
|
440
|
-
}
|
|
441
|
-
method.create = function() {
|
|
442
|
-
return new Sha256(is224);
|
|
443
|
-
};
|
|
444
|
-
method.update = function(message) {
|
|
445
|
-
return method.create().update(message);
|
|
446
|
-
};
|
|
447
|
-
for (var i = 0; i < OUTPUT_TYPES.length; ++i) {
|
|
448
|
-
var type = OUTPUT_TYPES[i];
|
|
449
|
-
method[type] = createOutputMethod(type, is224);
|
|
450
|
-
}
|
|
451
|
-
return method;
|
|
452
|
-
};
|
|
453
|
-
var nodeWrap = function(method, is224) {
|
|
454
|
-
var crypto = require("crypto");
|
|
455
|
-
var Buffer2 = require("buffer").Buffer;
|
|
456
|
-
var algorithm = is224 ? "sha224" : "sha256";
|
|
457
|
-
var bufferFrom;
|
|
458
|
-
if (Buffer2.from && !root.JS_SHA256_NO_BUFFER_FROM) {
|
|
459
|
-
bufferFrom = Buffer2.from;
|
|
460
|
-
} else {
|
|
461
|
-
bufferFrom = function(message) {
|
|
462
|
-
return new Buffer2(message);
|
|
463
|
-
};
|
|
464
|
-
}
|
|
465
|
-
var nodeMethod = function(message) {
|
|
466
|
-
if (typeof message === "string") {
|
|
467
|
-
return crypto.createHash(algorithm).update(message, "utf8").digest("hex");
|
|
468
|
-
} else {
|
|
469
|
-
if (message === null || message === void 0) {
|
|
470
|
-
throw new Error(ERROR);
|
|
471
|
-
} else if (message.constructor === ArrayBuffer) {
|
|
472
|
-
message = new Uint8Array(message);
|
|
473
|
-
}
|
|
474
|
-
}
|
|
475
|
-
if (Array.isArray(message) || ArrayBuffer.isView(message) || message.constructor === Buffer2) {
|
|
476
|
-
return crypto.createHash(algorithm).update(bufferFrom(message)).digest("hex");
|
|
477
|
-
} else {
|
|
478
|
-
return method(message);
|
|
479
|
-
}
|
|
480
|
-
};
|
|
481
|
-
return nodeMethod;
|
|
482
|
-
};
|
|
483
|
-
var createHmacOutputMethod = function(outputType, is224) {
|
|
484
|
-
return function(key, message) {
|
|
485
|
-
return new HmacSha256(key, is224, true).update(message)[outputType]();
|
|
486
|
-
};
|
|
487
|
-
};
|
|
488
|
-
var createHmacMethod = function(is224) {
|
|
489
|
-
var method = createHmacOutputMethod("hex", is224);
|
|
490
|
-
method.create = function(key) {
|
|
491
|
-
return new HmacSha256(key, is224);
|
|
492
|
-
};
|
|
493
|
-
method.update = function(key, message) {
|
|
494
|
-
return method.create(key).update(message);
|
|
495
|
-
};
|
|
496
|
-
for (var i = 0; i < OUTPUT_TYPES.length; ++i) {
|
|
497
|
-
var type = OUTPUT_TYPES[i];
|
|
498
|
-
method[type] = createHmacOutputMethod(type, is224);
|
|
499
|
-
}
|
|
500
|
-
return method;
|
|
501
|
-
};
|
|
502
|
-
function Sha256(is224, sharedMemory) {
|
|
503
|
-
if (sharedMemory) {
|
|
504
|
-
blocks[0] = blocks[16] = blocks[1] = blocks[2] = blocks[3] = blocks[4] = blocks[5] = blocks[6] = blocks[7] = blocks[8] = blocks[9] = blocks[10] = blocks[11] = blocks[12] = blocks[13] = blocks[14] = blocks[15] = 0;
|
|
505
|
-
this.blocks = blocks;
|
|
506
|
-
} else {
|
|
507
|
-
this.blocks = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
|
|
508
|
-
}
|
|
509
|
-
if (is224) {
|
|
510
|
-
this.h0 = 3238371032;
|
|
511
|
-
this.h1 = 914150663;
|
|
512
|
-
this.h2 = 812702999;
|
|
513
|
-
this.h3 = 4144912697;
|
|
514
|
-
this.h4 = 4290775857;
|
|
515
|
-
this.h5 = 1750603025;
|
|
516
|
-
this.h6 = 1694076839;
|
|
517
|
-
this.h7 = 3204075428;
|
|
518
|
-
} else {
|
|
519
|
-
this.h0 = 1779033703;
|
|
520
|
-
this.h1 = 3144134277;
|
|
521
|
-
this.h2 = 1013904242;
|
|
522
|
-
this.h3 = 2773480762;
|
|
523
|
-
this.h4 = 1359893119;
|
|
524
|
-
this.h5 = 2600822924;
|
|
525
|
-
this.h6 = 528734635;
|
|
526
|
-
this.h7 = 1541459225;
|
|
527
|
-
}
|
|
528
|
-
this.block = this.start = this.bytes = this.hBytes = 0;
|
|
529
|
-
this.finalized = this.hashed = false;
|
|
530
|
-
this.first = true;
|
|
531
|
-
this.is224 = is224;
|
|
532
|
-
}
|
|
533
|
-
Sha256.prototype.update = function(message) {
|
|
534
|
-
if (this.finalized) {
|
|
535
|
-
return;
|
|
536
|
-
}
|
|
537
|
-
var notString, type = typeof message;
|
|
538
|
-
if (type !== "string") {
|
|
539
|
-
if (type === "object") {
|
|
540
|
-
if (message === null) {
|
|
541
|
-
throw new Error(ERROR);
|
|
542
|
-
} else if (ARRAY_BUFFER && message.constructor === ArrayBuffer) {
|
|
543
|
-
message = new Uint8Array(message);
|
|
544
|
-
} else if (!Array.isArray(message)) {
|
|
545
|
-
if (!ARRAY_BUFFER || !ArrayBuffer.isView(message)) {
|
|
546
|
-
throw new Error(ERROR);
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
} else {
|
|
550
|
-
throw new Error(ERROR);
|
|
551
|
-
}
|
|
552
|
-
notString = true;
|
|
553
|
-
}
|
|
554
|
-
var code, index = 0, i, length = message.length, blocks2 = this.blocks;
|
|
555
|
-
while (index < length) {
|
|
556
|
-
if (this.hashed) {
|
|
557
|
-
this.hashed = false;
|
|
558
|
-
blocks2[0] = this.block;
|
|
559
|
-
this.block = blocks2[16] = blocks2[1] = blocks2[2] = blocks2[3] = blocks2[4] = blocks2[5] = blocks2[6] = blocks2[7] = blocks2[8] = blocks2[9] = blocks2[10] = blocks2[11] = blocks2[12] = blocks2[13] = blocks2[14] = blocks2[15] = 0;
|
|
560
|
-
}
|
|
561
|
-
if (notString) {
|
|
562
|
-
for (i = this.start; index < length && i < 64; ++index) {
|
|
563
|
-
blocks2[i >>> 2] |= message[index] << SHIFT[i++ & 3];
|
|
564
|
-
}
|
|
565
|
-
} else {
|
|
566
|
-
for (i = this.start; index < length && i < 64; ++index) {
|
|
567
|
-
code = message.charCodeAt(index);
|
|
568
|
-
if (code < 128) {
|
|
569
|
-
blocks2[i >>> 2] |= code << SHIFT[i++ & 3];
|
|
570
|
-
} else if (code < 2048) {
|
|
571
|
-
blocks2[i >>> 2] |= (192 | code >>> 6) << SHIFT[i++ & 3];
|
|
572
|
-
blocks2[i >>> 2] |= (128 | code & 63) << SHIFT[i++ & 3];
|
|
573
|
-
} else if (code < 55296 || code >= 57344) {
|
|
574
|
-
blocks2[i >>> 2] |= (224 | code >>> 12) << SHIFT[i++ & 3];
|
|
575
|
-
blocks2[i >>> 2] |= (128 | code >>> 6 & 63) << SHIFT[i++ & 3];
|
|
576
|
-
blocks2[i >>> 2] |= (128 | code & 63) << SHIFT[i++ & 3];
|
|
577
|
-
} else {
|
|
578
|
-
code = 65536 + ((code & 1023) << 10 | message.charCodeAt(++index) & 1023);
|
|
579
|
-
blocks2[i >>> 2] |= (240 | code >>> 18) << SHIFT[i++ & 3];
|
|
580
|
-
blocks2[i >>> 2] |= (128 | code >>> 12 & 63) << SHIFT[i++ & 3];
|
|
581
|
-
blocks2[i >>> 2] |= (128 | code >>> 6 & 63) << SHIFT[i++ & 3];
|
|
582
|
-
blocks2[i >>> 2] |= (128 | code & 63) << SHIFT[i++ & 3];
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
}
|
|
586
|
-
this.lastByteIndex = i;
|
|
587
|
-
this.bytes += i - this.start;
|
|
588
|
-
if (i >= 64) {
|
|
589
|
-
this.block = blocks2[16];
|
|
590
|
-
this.start = i - 64;
|
|
591
|
-
this.hash();
|
|
592
|
-
this.hashed = true;
|
|
593
|
-
} else {
|
|
594
|
-
this.start = i;
|
|
595
|
-
}
|
|
596
|
-
}
|
|
597
|
-
if (this.bytes > 4294967295) {
|
|
598
|
-
this.hBytes += this.bytes / 4294967296 << 0;
|
|
599
|
-
this.bytes = this.bytes % 4294967296;
|
|
600
|
-
}
|
|
601
|
-
return this;
|
|
602
|
-
};
|
|
603
|
-
Sha256.prototype.finalize = function() {
|
|
604
|
-
if (this.finalized) {
|
|
605
|
-
return;
|
|
606
|
-
}
|
|
607
|
-
this.finalized = true;
|
|
608
|
-
var blocks2 = this.blocks, i = this.lastByteIndex;
|
|
609
|
-
blocks2[16] = this.block;
|
|
610
|
-
blocks2[i >>> 2] |= EXTRA[i & 3];
|
|
611
|
-
this.block = blocks2[16];
|
|
612
|
-
if (i >= 56) {
|
|
613
|
-
if (!this.hashed) {
|
|
614
|
-
this.hash();
|
|
615
|
-
}
|
|
616
|
-
blocks2[0] = this.block;
|
|
617
|
-
blocks2[16] = blocks2[1] = blocks2[2] = blocks2[3] = blocks2[4] = blocks2[5] = blocks2[6] = blocks2[7] = blocks2[8] = blocks2[9] = blocks2[10] = blocks2[11] = blocks2[12] = blocks2[13] = blocks2[14] = blocks2[15] = 0;
|
|
618
|
-
}
|
|
619
|
-
blocks2[14] = this.hBytes << 3 | this.bytes >>> 29;
|
|
620
|
-
blocks2[15] = this.bytes << 3;
|
|
621
|
-
this.hash();
|
|
622
|
-
};
|
|
623
|
-
Sha256.prototype.hash = function() {
|
|
624
|
-
var a = this.h0, b = this.h1, c = this.h2, d = this.h3, e = this.h4, f = this.h5, g = this.h6, h = this.h7, blocks2 = this.blocks, j, s0, s1, maj, t1, t2, ch, ab, da, cd, bc;
|
|
625
|
-
for (j = 16; j < 64; ++j) {
|
|
626
|
-
t1 = blocks2[j - 15];
|
|
627
|
-
s0 = (t1 >>> 7 | t1 << 25) ^ (t1 >>> 18 | t1 << 14) ^ t1 >>> 3;
|
|
628
|
-
t1 = blocks2[j - 2];
|
|
629
|
-
s1 = (t1 >>> 17 | t1 << 15) ^ (t1 >>> 19 | t1 << 13) ^ t1 >>> 10;
|
|
630
|
-
blocks2[j] = blocks2[j - 16] + s0 + blocks2[j - 7] + s1 << 0;
|
|
631
|
-
}
|
|
632
|
-
bc = b & c;
|
|
633
|
-
for (j = 0; j < 64; j += 4) {
|
|
634
|
-
if (this.first) {
|
|
635
|
-
if (this.is224) {
|
|
636
|
-
ab = 300032;
|
|
637
|
-
t1 = blocks2[0] - 1413257819;
|
|
638
|
-
h = t1 - 150054599 << 0;
|
|
639
|
-
d = t1 + 24177077 << 0;
|
|
640
|
-
} else {
|
|
641
|
-
ab = 704751109;
|
|
642
|
-
t1 = blocks2[0] - 210244248;
|
|
643
|
-
h = t1 - 1521486534 << 0;
|
|
644
|
-
d = t1 + 143694565 << 0;
|
|
645
|
-
}
|
|
646
|
-
this.first = false;
|
|
647
|
-
} else {
|
|
648
|
-
s0 = (a >>> 2 | a << 30) ^ (a >>> 13 | a << 19) ^ (a >>> 22 | a << 10);
|
|
649
|
-
s1 = (e >>> 6 | e << 26) ^ (e >>> 11 | e << 21) ^ (e >>> 25 | e << 7);
|
|
650
|
-
ab = a & b;
|
|
651
|
-
maj = ab ^ a & c ^ bc;
|
|
652
|
-
ch = e & f ^ ~e & g;
|
|
653
|
-
t1 = h + s1 + ch + K[j] + blocks2[j];
|
|
654
|
-
t2 = s0 + maj;
|
|
655
|
-
h = d + t1 << 0;
|
|
656
|
-
d = t1 + t2 << 0;
|
|
657
|
-
}
|
|
658
|
-
s0 = (d >>> 2 | d << 30) ^ (d >>> 13 | d << 19) ^ (d >>> 22 | d << 10);
|
|
659
|
-
s1 = (h >>> 6 | h << 26) ^ (h >>> 11 | h << 21) ^ (h >>> 25 | h << 7);
|
|
660
|
-
da = d & a;
|
|
661
|
-
maj = da ^ d & b ^ ab;
|
|
662
|
-
ch = h & e ^ ~h & f;
|
|
663
|
-
t1 = g + s1 + ch + K[j + 1] + blocks2[j + 1];
|
|
664
|
-
t2 = s0 + maj;
|
|
665
|
-
g = c + t1 << 0;
|
|
666
|
-
c = t1 + t2 << 0;
|
|
667
|
-
s0 = (c >>> 2 | c << 30) ^ (c >>> 13 | c << 19) ^ (c >>> 22 | c << 10);
|
|
668
|
-
s1 = (g >>> 6 | g << 26) ^ (g >>> 11 | g << 21) ^ (g >>> 25 | g << 7);
|
|
669
|
-
cd = c & d;
|
|
670
|
-
maj = cd ^ c & a ^ da;
|
|
671
|
-
ch = g & h ^ ~g & e;
|
|
672
|
-
t1 = f + s1 + ch + K[j + 2] + blocks2[j + 2];
|
|
673
|
-
t2 = s0 + maj;
|
|
674
|
-
f = b + t1 << 0;
|
|
675
|
-
b = t1 + t2 << 0;
|
|
676
|
-
s0 = (b >>> 2 | b << 30) ^ (b >>> 13 | b << 19) ^ (b >>> 22 | b << 10);
|
|
677
|
-
s1 = (f >>> 6 | f << 26) ^ (f >>> 11 | f << 21) ^ (f >>> 25 | f << 7);
|
|
678
|
-
bc = b & c;
|
|
679
|
-
maj = bc ^ b & d ^ cd;
|
|
680
|
-
ch = f & g ^ ~f & h;
|
|
681
|
-
t1 = e + s1 + ch + K[j + 3] + blocks2[j + 3];
|
|
682
|
-
t2 = s0 + maj;
|
|
683
|
-
e = a + t1 << 0;
|
|
684
|
-
a = t1 + t2 << 0;
|
|
685
|
-
this.chromeBugWorkAround = true;
|
|
686
|
-
}
|
|
687
|
-
this.h0 = this.h0 + a << 0;
|
|
688
|
-
this.h1 = this.h1 + b << 0;
|
|
689
|
-
this.h2 = this.h2 + c << 0;
|
|
690
|
-
this.h3 = this.h3 + d << 0;
|
|
691
|
-
this.h4 = this.h4 + e << 0;
|
|
692
|
-
this.h5 = this.h5 + f << 0;
|
|
693
|
-
this.h6 = this.h6 + g << 0;
|
|
694
|
-
this.h7 = this.h7 + h << 0;
|
|
695
|
-
};
|
|
696
|
-
Sha256.prototype.hex = function() {
|
|
697
|
-
this.finalize();
|
|
698
|
-
var h0 = this.h0, h1 = this.h1, h2 = this.h2, h3 = this.h3, h4 = this.h4, h5 = this.h5, h6 = this.h6, h7 = this.h7;
|
|
699
|
-
var hex = HEX_CHARS[h0 >>> 28 & 15] + HEX_CHARS[h0 >>> 24 & 15] + HEX_CHARS[h0 >>> 20 & 15] + HEX_CHARS[h0 >>> 16 & 15] + HEX_CHARS[h0 >>> 12 & 15] + HEX_CHARS[h0 >>> 8 & 15] + HEX_CHARS[h0 >>> 4 & 15] + HEX_CHARS[h0 & 15] + HEX_CHARS[h1 >>> 28 & 15] + HEX_CHARS[h1 >>> 24 & 15] + HEX_CHARS[h1 >>> 20 & 15] + HEX_CHARS[h1 >>> 16 & 15] + HEX_CHARS[h1 >>> 12 & 15] + HEX_CHARS[h1 >>> 8 & 15] + HEX_CHARS[h1 >>> 4 & 15] + HEX_CHARS[h1 & 15] + HEX_CHARS[h2 >>> 28 & 15] + HEX_CHARS[h2 >>> 24 & 15] + HEX_CHARS[h2 >>> 20 & 15] + HEX_CHARS[h2 >>> 16 & 15] + HEX_CHARS[h2 >>> 12 & 15] + HEX_CHARS[h2 >>> 8 & 15] + HEX_CHARS[h2 >>> 4 & 15] + HEX_CHARS[h2 & 15] + HEX_CHARS[h3 >>> 28 & 15] + HEX_CHARS[h3 >>> 24 & 15] + HEX_CHARS[h3 >>> 20 & 15] + HEX_CHARS[h3 >>> 16 & 15] + HEX_CHARS[h3 >>> 12 & 15] + HEX_CHARS[h3 >>> 8 & 15] + HEX_CHARS[h3 >>> 4 & 15] + HEX_CHARS[h3 & 15] + HEX_CHARS[h4 >>> 28 & 15] + HEX_CHARS[h4 >>> 24 & 15] + HEX_CHARS[h4 >>> 20 & 15] + HEX_CHARS[h4 >>> 16 & 15] + HEX_CHARS[h4 >>> 12 & 15] + HEX_CHARS[h4 >>> 8 & 15] + HEX_CHARS[h4 >>> 4 & 15] + HEX_CHARS[h4 & 15] + HEX_CHARS[h5 >>> 28 & 15] + HEX_CHARS[h5 >>> 24 & 15] + HEX_CHARS[h5 >>> 20 & 15] + HEX_CHARS[h5 >>> 16 & 15] + HEX_CHARS[h5 >>> 12 & 15] + HEX_CHARS[h5 >>> 8 & 15] + HEX_CHARS[h5 >>> 4 & 15] + HEX_CHARS[h5 & 15] + HEX_CHARS[h6 >>> 28 & 15] + HEX_CHARS[h6 >>> 24 & 15] + HEX_CHARS[h6 >>> 20 & 15] + HEX_CHARS[h6 >>> 16 & 15] + HEX_CHARS[h6 >>> 12 & 15] + HEX_CHARS[h6 >>> 8 & 15] + HEX_CHARS[h6 >>> 4 & 15] + HEX_CHARS[h6 & 15];
|
|
700
|
-
if (!this.is224) {
|
|
701
|
-
hex += HEX_CHARS[h7 >>> 28 & 15] + HEX_CHARS[h7 >>> 24 & 15] + HEX_CHARS[h7 >>> 20 & 15] + HEX_CHARS[h7 >>> 16 & 15] + HEX_CHARS[h7 >>> 12 & 15] + HEX_CHARS[h7 >>> 8 & 15] + HEX_CHARS[h7 >>> 4 & 15] + HEX_CHARS[h7 & 15];
|
|
702
|
-
}
|
|
703
|
-
return hex;
|
|
704
|
-
};
|
|
705
|
-
Sha256.prototype.toString = Sha256.prototype.hex;
|
|
706
|
-
Sha256.prototype.digest = function() {
|
|
707
|
-
this.finalize();
|
|
708
|
-
var h0 = this.h0, h1 = this.h1, h2 = this.h2, h3 = this.h3, h4 = this.h4, h5 = this.h5, h6 = this.h6, h7 = this.h7;
|
|
709
|
-
var arr = [
|
|
710
|
-
h0 >>> 24 & 255,
|
|
711
|
-
h0 >>> 16 & 255,
|
|
712
|
-
h0 >>> 8 & 255,
|
|
713
|
-
h0 & 255,
|
|
714
|
-
h1 >>> 24 & 255,
|
|
715
|
-
h1 >>> 16 & 255,
|
|
716
|
-
h1 >>> 8 & 255,
|
|
717
|
-
h1 & 255,
|
|
718
|
-
h2 >>> 24 & 255,
|
|
719
|
-
h2 >>> 16 & 255,
|
|
720
|
-
h2 >>> 8 & 255,
|
|
721
|
-
h2 & 255,
|
|
722
|
-
h3 >>> 24 & 255,
|
|
723
|
-
h3 >>> 16 & 255,
|
|
724
|
-
h3 >>> 8 & 255,
|
|
725
|
-
h3 & 255,
|
|
726
|
-
h4 >>> 24 & 255,
|
|
727
|
-
h4 >>> 16 & 255,
|
|
728
|
-
h4 >>> 8 & 255,
|
|
729
|
-
h4 & 255,
|
|
730
|
-
h5 >>> 24 & 255,
|
|
731
|
-
h5 >>> 16 & 255,
|
|
732
|
-
h5 >>> 8 & 255,
|
|
733
|
-
h5 & 255,
|
|
734
|
-
h6 >>> 24 & 255,
|
|
735
|
-
h6 >>> 16 & 255,
|
|
736
|
-
h6 >>> 8 & 255,
|
|
737
|
-
h6 & 255
|
|
738
|
-
];
|
|
739
|
-
if (!this.is224) {
|
|
740
|
-
arr.push(h7 >>> 24 & 255, h7 >>> 16 & 255, h7 >>> 8 & 255, h7 & 255);
|
|
741
|
-
}
|
|
742
|
-
return arr;
|
|
743
|
-
};
|
|
744
|
-
Sha256.prototype.array = Sha256.prototype.digest;
|
|
745
|
-
Sha256.prototype.arrayBuffer = function() {
|
|
746
|
-
this.finalize();
|
|
747
|
-
var buffer = new ArrayBuffer(this.is224 ? 28 : 32);
|
|
748
|
-
var dataView = new DataView(buffer);
|
|
749
|
-
dataView.setUint32(0, this.h0);
|
|
750
|
-
dataView.setUint32(4, this.h1);
|
|
751
|
-
dataView.setUint32(8, this.h2);
|
|
752
|
-
dataView.setUint32(12, this.h3);
|
|
753
|
-
dataView.setUint32(16, this.h4);
|
|
754
|
-
dataView.setUint32(20, this.h5);
|
|
755
|
-
dataView.setUint32(24, this.h6);
|
|
756
|
-
if (!this.is224) {
|
|
757
|
-
dataView.setUint32(28, this.h7);
|
|
758
|
-
}
|
|
759
|
-
return buffer;
|
|
760
|
-
};
|
|
761
|
-
function HmacSha256(key, is224, sharedMemory) {
|
|
762
|
-
var i, type = typeof key;
|
|
763
|
-
if (type === "string") {
|
|
764
|
-
var bytes = [], length = key.length, index = 0, code;
|
|
765
|
-
for (i = 0; i < length; ++i) {
|
|
766
|
-
code = key.charCodeAt(i);
|
|
767
|
-
if (code < 128) {
|
|
768
|
-
bytes[index++] = code;
|
|
769
|
-
} else if (code < 2048) {
|
|
770
|
-
bytes[index++] = 192 | code >>> 6;
|
|
771
|
-
bytes[index++] = 128 | code & 63;
|
|
772
|
-
} else if (code < 55296 || code >= 57344) {
|
|
773
|
-
bytes[index++] = 224 | code >>> 12;
|
|
774
|
-
bytes[index++] = 128 | code >>> 6 & 63;
|
|
775
|
-
bytes[index++] = 128 | code & 63;
|
|
776
|
-
} else {
|
|
777
|
-
code = 65536 + ((code & 1023) << 10 | key.charCodeAt(++i) & 1023);
|
|
778
|
-
bytes[index++] = 240 | code >>> 18;
|
|
779
|
-
bytes[index++] = 128 | code >>> 12 & 63;
|
|
780
|
-
bytes[index++] = 128 | code >>> 6 & 63;
|
|
781
|
-
bytes[index++] = 128 | code & 63;
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
key = bytes;
|
|
785
|
-
} else {
|
|
786
|
-
if (type === "object") {
|
|
787
|
-
if (key === null) {
|
|
788
|
-
throw new Error(ERROR);
|
|
789
|
-
} else if (ARRAY_BUFFER && key.constructor === ArrayBuffer) {
|
|
790
|
-
key = new Uint8Array(key);
|
|
791
|
-
} else if (!Array.isArray(key)) {
|
|
792
|
-
if (!ARRAY_BUFFER || !ArrayBuffer.isView(key)) {
|
|
793
|
-
throw new Error(ERROR);
|
|
794
|
-
}
|
|
795
|
-
}
|
|
796
|
-
} else {
|
|
797
|
-
throw new Error(ERROR);
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
if (key.length > 64) {
|
|
801
|
-
key = new Sha256(is224, true).update(key).array();
|
|
802
|
-
}
|
|
803
|
-
var oKeyPad = [], iKeyPad = [];
|
|
804
|
-
for (i = 0; i < 64; ++i) {
|
|
805
|
-
var b = key[i] || 0;
|
|
806
|
-
oKeyPad[i] = 92 ^ b;
|
|
807
|
-
iKeyPad[i] = 54 ^ b;
|
|
808
|
-
}
|
|
809
|
-
Sha256.call(this, is224, sharedMemory);
|
|
810
|
-
this.update(iKeyPad);
|
|
811
|
-
this.oKeyPad = oKeyPad;
|
|
812
|
-
this.inner = true;
|
|
813
|
-
this.sharedMemory = sharedMemory;
|
|
814
|
-
}
|
|
815
|
-
HmacSha256.prototype = new Sha256();
|
|
816
|
-
HmacSha256.prototype.finalize = function() {
|
|
817
|
-
Sha256.prototype.finalize.call(this);
|
|
818
|
-
if (this.inner) {
|
|
819
|
-
this.inner = false;
|
|
820
|
-
var innerHash = this.array();
|
|
821
|
-
Sha256.call(this, this.is224, this.sharedMemory);
|
|
822
|
-
this.update(this.oKeyPad);
|
|
823
|
-
this.update(innerHash);
|
|
824
|
-
Sha256.prototype.finalize.call(this);
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
var exports2 = createMethod();
|
|
828
|
-
exports2.sha256 = exports2;
|
|
829
|
-
exports2.sha224 = createMethod(true);
|
|
830
|
-
exports2.sha256.hmac = createHmacMethod();
|
|
831
|
-
exports2.sha224.hmac = createHmacMethod(true);
|
|
832
|
-
if (COMMON_JS) {
|
|
833
|
-
module2.exports = exports2;
|
|
834
|
-
} else {
|
|
835
|
-
root.sha256 = exports2.sha256;
|
|
836
|
-
root.sha224 = exports2.sha224;
|
|
837
|
-
if (AMD) {
|
|
838
|
-
define(function() {
|
|
839
|
-
return exports2;
|
|
840
|
-
});
|
|
841
|
-
}
|
|
842
|
-
}
|
|
843
|
-
})();
|
|
844
|
-
}
|
|
845
|
-
});
|
|
846
|
-
|
|
847
328
|
// src/appium/index.ts
|
|
848
329
|
var appium_exports = {};
|
|
849
330
|
__export(appium_exports, {
|
|
@@ -853,19 +334,21 @@ __export(appium_exports, {
|
|
|
853
334
|
module.exports = __toCommonJS(appium_exports);
|
|
854
335
|
|
|
855
336
|
// src/common/agent.ts
|
|
856
|
-
var
|
|
337
|
+
var import_core2 = require("@midscene/core");
|
|
857
338
|
var import_constants2 = require("@midscene/shared/constants");
|
|
339
|
+
var import_env3 = require("@midscene/core/env");
|
|
858
340
|
var import_utils6 = require("@midscene/core/utils");
|
|
859
341
|
|
|
860
342
|
// src/common/tasks.ts
|
|
861
343
|
var import_node_assert = __toESM(require("assert"));
|
|
862
|
-
var
|
|
344
|
+
var import_core = require("@midscene/core");
|
|
345
|
+
var import_ai_model = require("@midscene/core/ai-model");
|
|
863
346
|
var import_utils3 = require("@midscene/core/utils");
|
|
864
347
|
|
|
865
348
|
// src/common/task-cache.ts
|
|
866
349
|
var import_node_fs = require("fs");
|
|
867
350
|
var import_node_path = require("path");
|
|
868
|
-
var
|
|
351
|
+
var import_env = require("@midscene/core/env");
|
|
869
352
|
var import_utils = require("@midscene/core/utils");
|
|
870
353
|
var import_fs = require("@midscene/shared/fs");
|
|
871
354
|
var import_utils2 = require("@midscene/shared/utils");
|
|
@@ -957,7 +440,7 @@ var TaskCache = class {
|
|
|
957
440
|
return void 0;
|
|
958
441
|
}
|
|
959
442
|
const cacheFile = (0, import_node_path.join)((0, import_utils.getLogDirByType)("cache"), `${this.cacheId}.json`);
|
|
960
|
-
if ((0,
|
|
443
|
+
if ((0, import_env.getAIConfig)("MIDSCENE_CACHE") === "true" && (0, import_node_fs.existsSync)(cacheFile)) {
|
|
961
444
|
try {
|
|
962
445
|
const data = (0, import_node_fs.readFileSync)(cacheFile, "utf8");
|
|
963
446
|
const jsonData = JSON.parse(data);
|
|
@@ -1001,6 +484,7 @@ var TaskCache = class {
|
|
|
1001
484
|
// src/common/tasks.ts
|
|
1002
485
|
var PageTaskExecutor = class {
|
|
1003
486
|
constructor(page, insight, opts) {
|
|
487
|
+
this.conversationHistory = [];
|
|
1004
488
|
this.page = page;
|
|
1005
489
|
this.insight = insight;
|
|
1006
490
|
this.taskCache = new TaskCache({
|
|
@@ -1090,10 +574,13 @@ var PageTaskExecutor = class {
|
|
|
1090
574
|
);
|
|
1091
575
|
let locateResult;
|
|
1092
576
|
const callAI = this.insight.aiVendorFn;
|
|
577
|
+
const quickAnswer = {
|
|
578
|
+
id: param == null ? void 0 : param.id,
|
|
579
|
+
position: param == null ? void 0 : param.position
|
|
580
|
+
};
|
|
581
|
+
const startTime = Date.now();
|
|
1093
582
|
const element = await this.insight.locate(param.prompt, {
|
|
1094
|
-
quickAnswer
|
|
1095
|
-
id: param.id
|
|
1096
|
-
} : void 0,
|
|
583
|
+
quickAnswer,
|
|
1097
584
|
callAI: async (...message) => {
|
|
1098
585
|
if (locateCache) {
|
|
1099
586
|
locateResult = locateCache;
|
|
@@ -1103,6 +590,7 @@ var PageTaskExecutor = class {
|
|
|
1103
590
|
return { content: aiResult, usage };
|
|
1104
591
|
}
|
|
1105
592
|
});
|
|
593
|
+
const aiCost = Date.now() - startTime;
|
|
1106
594
|
if (locateResult) {
|
|
1107
595
|
cacheGroup == null ? void 0 : cacheGroup.saveCache({
|
|
1108
596
|
type: "locate",
|
|
@@ -1130,7 +618,8 @@ var PageTaskExecutor = class {
|
|
|
1130
618
|
cache: {
|
|
1131
619
|
hit: Boolean(locateCache)
|
|
1132
620
|
},
|
|
1133
|
-
recorder: [recordItem]
|
|
621
|
+
recorder: [recordItem],
|
|
622
|
+
aiCost
|
|
1134
623
|
};
|
|
1135
624
|
}
|
|
1136
625
|
};
|
|
@@ -1188,6 +677,8 @@ var PageTaskExecutor = class {
|
|
|
1188
677
|
return;
|
|
1189
678
|
}
|
|
1190
679
|
await this.page.keyboard.type(taskParam.value);
|
|
680
|
+
} else {
|
|
681
|
+
await this.page.keyboard.type(taskParam.value);
|
|
1191
682
|
}
|
|
1192
683
|
}
|
|
1193
684
|
};
|
|
@@ -1322,6 +813,17 @@ var PageTaskExecutor = class {
|
|
|
1322
813
|
}
|
|
1323
814
|
};
|
|
1324
815
|
tasks.push(taskActionFalsyConditionStatement);
|
|
816
|
+
} else if (plan2.type === "Finished") {
|
|
817
|
+
const taskActionFinished = {
|
|
818
|
+
type: "Action",
|
|
819
|
+
subType: "Finished",
|
|
820
|
+
param: null,
|
|
821
|
+
thought: plan2.thought,
|
|
822
|
+
locate: plan2.locate,
|
|
823
|
+
executor: async (param) => {
|
|
824
|
+
}
|
|
825
|
+
};
|
|
826
|
+
tasks.push(taskActionFinished);
|
|
1325
827
|
} else {
|
|
1326
828
|
throw new Error(`Unknown or unsupported task type: ${plan2.type}`);
|
|
1327
829
|
}
|
|
@@ -1366,13 +868,13 @@ var PageTaskExecutor = class {
|
|
|
1366
868
|
if (planCache) {
|
|
1367
869
|
planResult = planCache;
|
|
1368
870
|
} else {
|
|
1369
|
-
planResult = await (0,
|
|
871
|
+
planResult = await (0, import_core.plan)(param.userPrompt, {
|
|
1370
872
|
context: pageContext,
|
|
1371
873
|
whatHaveDone: param.whatHaveDone,
|
|
1372
874
|
originalPrompt: param.originalPrompt
|
|
1373
875
|
});
|
|
1374
876
|
}
|
|
1375
|
-
const { actions, furtherPlan, taskWillBeAccomplished } = planResult;
|
|
877
|
+
const { actions, furtherPlan, taskWillBeAccomplished, error } = planResult;
|
|
1376
878
|
let stopCollecting = false;
|
|
1377
879
|
const finalActions = actions.reduce(
|
|
1378
880
|
(acc, planningAction) => {
|
|
@@ -1399,7 +901,10 @@ var PageTaskExecutor = class {
|
|
|
1399
901
|
},
|
|
1400
902
|
[]
|
|
1401
903
|
);
|
|
1402
|
-
(0, import_node_assert.default)(
|
|
904
|
+
(0, import_node_assert.default)(
|
|
905
|
+
finalActions.length > 0,
|
|
906
|
+
error ? `No plan: ${error}` : "No plans found"
|
|
907
|
+
);
|
|
1403
908
|
cacheGroup.saveCache({
|
|
1404
909
|
type: "plan",
|
|
1405
910
|
pageContext: {
|
|
@@ -1426,9 +931,67 @@ var PageTaskExecutor = class {
|
|
|
1426
931
|
};
|
|
1427
932
|
return task;
|
|
1428
933
|
}
|
|
934
|
+
planningTaskToGoal(userPrompt) {
|
|
935
|
+
const task = {
|
|
936
|
+
type: "Planning",
|
|
937
|
+
locate: null,
|
|
938
|
+
param: {
|
|
939
|
+
userPrompt
|
|
940
|
+
},
|
|
941
|
+
executor: async (param, executorContext) => {
|
|
942
|
+
const shotTime = Date.now();
|
|
943
|
+
const pageContext = await this.insight.contextRetrieverFn("locate");
|
|
944
|
+
const recordItem = {
|
|
945
|
+
type: "screenshot",
|
|
946
|
+
ts: shotTime,
|
|
947
|
+
screenshot: pageContext.screenshotBase64,
|
|
948
|
+
timing: "before planning"
|
|
949
|
+
};
|
|
950
|
+
executorContext.task.recorder = [recordItem];
|
|
951
|
+
executorContext.task.pageContext = pageContext;
|
|
952
|
+
this.appendConversationHistory({
|
|
953
|
+
role: "user",
|
|
954
|
+
content: [
|
|
955
|
+
{
|
|
956
|
+
type: "image_url",
|
|
957
|
+
image_url: {
|
|
958
|
+
url: pageContext.screenshotBase64
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
]
|
|
962
|
+
});
|
|
963
|
+
const startTime = Date.now();
|
|
964
|
+
const planResult = await (0, import_ai_model.vlmPlanning)({
|
|
965
|
+
userInstruction: param.userPrompt,
|
|
966
|
+
conversationHistory: this.conversationHistory,
|
|
967
|
+
size: pageContext.size
|
|
968
|
+
});
|
|
969
|
+
const aiCost = Date.now() - startTime;
|
|
970
|
+
const { actions, action_summary } = planResult;
|
|
971
|
+
this.appendConversationHistory({
|
|
972
|
+
role: "assistant",
|
|
973
|
+
content: action_summary
|
|
974
|
+
});
|
|
975
|
+
return {
|
|
976
|
+
output: {
|
|
977
|
+
actions,
|
|
978
|
+
thought: actions[0].thought,
|
|
979
|
+
actionType: actions[0].type,
|
|
980
|
+
taskWillBeAccomplished: false,
|
|
981
|
+
furtherPlan: {
|
|
982
|
+
whatToDoNext: "",
|
|
983
|
+
whatHaveDone: ""
|
|
984
|
+
}
|
|
985
|
+
},
|
|
986
|
+
aiCost
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
};
|
|
990
|
+
return task;
|
|
991
|
+
}
|
|
1429
992
|
async action(userPrompt, options) {
|
|
1430
993
|
var _a;
|
|
1431
|
-
const taskExecutor = new
|
|
994
|
+
const taskExecutor = new import_core.Executor(userPrompt, void 0, void 0, {
|
|
1432
995
|
onTaskStart: options == null ? void 0 : options.onTaskStart
|
|
1433
996
|
});
|
|
1434
997
|
const cacheGroup = this.taskCache.getCacheGroupByPrompt(userPrompt);
|
|
@@ -1490,9 +1053,57 @@ var PageTaskExecutor = class {
|
|
|
1490
1053
|
executor: taskExecutor
|
|
1491
1054
|
};
|
|
1492
1055
|
}
|
|
1056
|
+
async actionToGoal(userPrompt, options) {
|
|
1057
|
+
const taskExecutor = new import_core.Executor(userPrompt, void 0, void 0, {
|
|
1058
|
+
onTaskStart: options == null ? void 0 : options.onTaskStart
|
|
1059
|
+
});
|
|
1060
|
+
this.conversationHistory = [];
|
|
1061
|
+
const isCompleted = false;
|
|
1062
|
+
let currentActionNumber = 0;
|
|
1063
|
+
const maxActionNumber = 20;
|
|
1064
|
+
while (!isCompleted && currentActionNumber < maxActionNumber) {
|
|
1065
|
+
currentActionNumber++;
|
|
1066
|
+
const planningTask = this.planningTaskToGoal(userPrompt);
|
|
1067
|
+
await taskExecutor.append(planningTask);
|
|
1068
|
+
const output = await taskExecutor.flush();
|
|
1069
|
+
if (taskExecutor.isInErrorState()) {
|
|
1070
|
+
return {
|
|
1071
|
+
output,
|
|
1072
|
+
executor: taskExecutor
|
|
1073
|
+
};
|
|
1074
|
+
}
|
|
1075
|
+
const plans = output.actions;
|
|
1076
|
+
let executables;
|
|
1077
|
+
try {
|
|
1078
|
+
executables = await this.convertPlanToExecutable(plans);
|
|
1079
|
+
taskExecutor.append(executables.tasks);
|
|
1080
|
+
} catch (error) {
|
|
1081
|
+
return this.appendErrorPlan(
|
|
1082
|
+
taskExecutor,
|
|
1083
|
+
`Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(
|
|
1084
|
+
plans
|
|
1085
|
+
)}`
|
|
1086
|
+
);
|
|
1087
|
+
}
|
|
1088
|
+
const result = await taskExecutor.flush();
|
|
1089
|
+
if (taskExecutor.isInErrorState()) {
|
|
1090
|
+
return {
|
|
1091
|
+
output: result,
|
|
1092
|
+
executor: taskExecutor
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
if (plans[0].type === "Finished") {
|
|
1096
|
+
break;
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
return {
|
|
1100
|
+
output: {},
|
|
1101
|
+
executor: taskExecutor
|
|
1102
|
+
};
|
|
1103
|
+
}
|
|
1493
1104
|
async query(demand, options) {
|
|
1494
1105
|
const description = typeof demand === "string" ? demand : JSON.stringify(demand);
|
|
1495
|
-
const taskExecutor = new
|
|
1106
|
+
const taskExecutor = new import_core.Executor(description, void 0, void 0, {
|
|
1496
1107
|
onTaskStart: options == null ? void 0 : options.onTaskStart
|
|
1497
1108
|
});
|
|
1498
1109
|
const queryTask = {
|
|
@@ -1524,7 +1135,7 @@ var PageTaskExecutor = class {
|
|
|
1524
1135
|
}
|
|
1525
1136
|
async assert(assertion, options) {
|
|
1526
1137
|
const description = `assert: ${assertion}`;
|
|
1527
|
-
const taskExecutor = new
|
|
1138
|
+
const taskExecutor = new import_core.Executor(description, void 0, void 0, {
|
|
1528
1139
|
onTaskStart: options == null ? void 0 : options.onTaskStart
|
|
1529
1140
|
});
|
|
1530
1141
|
const assertionPlan = {
|
|
@@ -1542,6 +1153,31 @@ var PageTaskExecutor = class {
|
|
|
1542
1153
|
executor: taskExecutor
|
|
1543
1154
|
};
|
|
1544
1155
|
}
|
|
1156
|
+
/**
|
|
1157
|
+
* Append a message to the conversation history
|
|
1158
|
+
* For user messages with images:
|
|
1159
|
+
* - Keep max 4 user image messages in history
|
|
1160
|
+
* - Remove oldest user image message when limit reached
|
|
1161
|
+
* For assistant messages:
|
|
1162
|
+
* - Simply append to history
|
|
1163
|
+
* @param conversationHistory Message to append
|
|
1164
|
+
*/
|
|
1165
|
+
appendConversationHistory(conversationHistory) {
|
|
1166
|
+
if (conversationHistory.role === "user") {
|
|
1167
|
+
const userImgItems = this.conversationHistory.filter(
|
|
1168
|
+
(item) => item.role === "user"
|
|
1169
|
+
);
|
|
1170
|
+
if (userImgItems.length >= 4 && conversationHistory.role === "user") {
|
|
1171
|
+
const firstUserImgIndex = this.conversationHistory.findIndex(
|
|
1172
|
+
(item) => item.role === "user"
|
|
1173
|
+
);
|
|
1174
|
+
if (firstUserImgIndex >= 0) {
|
|
1175
|
+
this.conversationHistory.splice(firstUserImgIndex, 1);
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
this.conversationHistory.push(conversationHistory);
|
|
1180
|
+
}
|
|
1545
1181
|
async appendErrorPlan(taskExecutor, errorMsg) {
|
|
1546
1182
|
const errorPlan = {
|
|
1547
1183
|
type: "Error",
|
|
@@ -1560,7 +1196,7 @@ var PageTaskExecutor = class {
|
|
|
1560
1196
|
}
|
|
1561
1197
|
async waitFor(assertion, opt) {
|
|
1562
1198
|
const description = `waitFor: ${assertion}`;
|
|
1563
|
-
const taskExecutor = new
|
|
1199
|
+
const taskExecutor = new import_core.Executor(description);
|
|
1564
1200
|
const { timeoutMs, checkIntervalMs } = opt;
|
|
1565
1201
|
(0, import_node_assert.default)(assertion, "No assertion for waitFor");
|
|
1566
1202
|
(0, import_node_assert.default)(timeoutMs, "No timeoutMs for waitFor");
|
|
@@ -1680,7 +1316,7 @@ function paramStr(task) {
|
|
|
1680
1316
|
var import_node_assert2 = __toESM(require("assert"));
|
|
1681
1317
|
var import_node_fs2 = require("fs");
|
|
1682
1318
|
var import_node_path2 = __toESM(require("path"));
|
|
1683
|
-
var
|
|
1319
|
+
var import_env2 = require("@midscene/core/env");
|
|
1684
1320
|
var import_utils4 = require("@midscene/core/utils");
|
|
1685
1321
|
var import_constants = require("@midscene/shared/constants");
|
|
1686
1322
|
var import_fs2 = require("@midscene/shared/fs");
|
|
@@ -1714,15 +1350,22 @@ async function parseContextFromWebPage(page, _opt) {
|
|
|
1714
1350
|
}
|
|
1715
1351
|
);
|
|
1716
1352
|
const size = await page.size();
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1353
|
+
let screenshotBase64WithElementMarker = screenshotBase64;
|
|
1354
|
+
if (!(0, import_env2.getAIConfig)(import_env2.MIDSCENE_USE_VLM_UI_TARS)) {
|
|
1355
|
+
if (_opt == null ? void 0 : _opt.ignoreMarker) {
|
|
1356
|
+
screenshotBase64WithElementMarker = await (0, import_img.compositeElementInfoImg)({
|
|
1357
|
+
inputImgBase64: screenshotBase64,
|
|
1358
|
+
elementsPositionInfo: [],
|
|
1359
|
+
size
|
|
1360
|
+
});
|
|
1361
|
+
} else {
|
|
1362
|
+
screenshotBase64WithElementMarker = await (0, import_img.compositeElementInfoImg)({
|
|
1363
|
+
inputImgBase64: screenshotBase64,
|
|
1364
|
+
elementsPositionInfo: elementsPositionInfoWithoutText,
|
|
1365
|
+
size
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1726
1369
|
return {
|
|
1727
1370
|
content: elementsInfo,
|
|
1728
1371
|
size,
|
|
@@ -1754,7 +1397,7 @@ async function alignElements(elements, page) {
|
|
|
1754
1397
|
return textsAligned;
|
|
1755
1398
|
}
|
|
1756
1399
|
function reportFileName(tag = "web") {
|
|
1757
|
-
const reportTagName = (0,
|
|
1400
|
+
const reportTagName = (0, import_env2.getAIConfig)(import_env2.MIDSCENE_REPORT_TAG_NAME);
|
|
1758
1401
|
const dateTimeInFileName = (0, import_dayjs.default)().format("YYYY-MM-DD_HH-mm-ss-SSS");
|
|
1759
1402
|
return `${reportTagName || tag}-${dateTimeInFileName}`;
|
|
1760
1403
|
}
|
|
@@ -1779,7 +1422,7 @@ var PageAgent = class {
|
|
|
1779
1422
|
},
|
|
1780
1423
|
opts || {}
|
|
1781
1424
|
);
|
|
1782
|
-
this.insight = new
|
|
1425
|
+
this.insight = new import_core2.Insight(
|
|
1783
1426
|
async (action) => {
|
|
1784
1427
|
return this.getUIContext(action);
|
|
1785
1428
|
},
|
|
@@ -1845,20 +1488,38 @@ var PageAgent = class {
|
|
|
1845
1488
|
}
|
|
1846
1489
|
async callbackOnTaskStartTip(task) {
|
|
1847
1490
|
if (this.opts.onTaskStartTip) {
|
|
1848
|
-
const
|
|
1849
|
-
|
|
1491
|
+
const param = paramStr(task);
|
|
1492
|
+
if (param) {
|
|
1493
|
+
const tip = `${typeStr(task)} - ${param}`;
|
|
1494
|
+
await this.opts.onTaskStartTip(tip);
|
|
1495
|
+
} else {
|
|
1496
|
+
await this.opts.onTaskStartTip(typeStr(task));
|
|
1497
|
+
}
|
|
1850
1498
|
}
|
|
1851
1499
|
}
|
|
1852
1500
|
async aiAction(taskPrompt) {
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1501
|
+
if ((0, import_env3.getAIConfig)(import_env3.MIDSCENE_USE_VLM_UI_TARS)) {
|
|
1502
|
+
const { executor } = await this.taskExecutor.actionToGoal(taskPrompt, {
|
|
1503
|
+
onTaskStart: this.callbackOnTaskStartTip.bind(this)
|
|
1504
|
+
});
|
|
1505
|
+
this.appendExecutionDump(executor.dump());
|
|
1506
|
+
this.writeOutActionDumps();
|
|
1507
|
+
if (executor.isInErrorState()) {
|
|
1508
|
+
const errorTask = executor.latestErrorTask();
|
|
1509
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1861
1510
|
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1511
|
+
}
|
|
1512
|
+
} else {
|
|
1513
|
+
const { executor } = await this.taskExecutor.action(taskPrompt, {
|
|
1514
|
+
onTaskStart: this.callbackOnTaskStartTip.bind(this)
|
|
1515
|
+
});
|
|
1516
|
+
this.appendExecutionDump(executor.dump());
|
|
1517
|
+
this.writeOutActionDumps();
|
|
1518
|
+
if (executor.isInErrorState()) {
|
|
1519
|
+
const errorTask = executor.latestErrorTask();
|
|
1520
|
+
throw new Error(`${errorTask == null ? void 0 : errorTask.error}
|
|
1521
|
+
${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
1522
|
+
}
|
|
1862
1523
|
}
|
|
1863
1524
|
}
|
|
1864
1525
|
async aiQuery(demand) {
|
|
@@ -1926,47 +1587,27 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
|
|
|
1926
1587
|
|
|
1927
1588
|
// src/appium/page.ts
|
|
1928
1589
|
var import_node_fs3 = __toESM(require("fs"));
|
|
1929
|
-
var
|
|
1590
|
+
var import_utils11 = require("@midscene/core/utils");
|
|
1930
1591
|
var import_img2 = require("@midscene/shared/img");
|
|
1931
1592
|
var import_xmldom = require("@xmldom/xmldom");
|
|
1932
1593
|
|
|
1933
1594
|
// src/extractor/web-extractor.ts
|
|
1934
1595
|
var import_constants3 = require("@midscene/shared/constants");
|
|
1596
|
+
var import_utils10 = require("@midscene/shared/utils");
|
|
1935
1597
|
|
|
1936
1598
|
// src/extractor/util.ts
|
|
1937
|
-
var
|
|
1938
|
-
var frameId = 0;
|
|
1939
|
-
function getFrameId() {
|
|
1940
|
-
return frameId;
|
|
1941
|
-
}
|
|
1599
|
+
var import_utils9 = require("@midscene/shared/utils");
|
|
1942
1600
|
var nodeHashCacheList = [];
|
|
1943
1601
|
if (typeof window !== "undefined") {
|
|
1944
1602
|
window.midsceneNodeHashCacheList = window.midsceneNodeHashCacheList || [];
|
|
1945
1603
|
nodeHashCacheList = window.midsceneNodeHashCacheList;
|
|
1946
1604
|
}
|
|
1947
|
-
var hashMap = {};
|
|
1948
1605
|
function midsceneGenerateHash(node, content, rect) {
|
|
1949
1606
|
var _a;
|
|
1950
1607
|
if (node && nodeHashCacheList.find((item) => item.node === node)) {
|
|
1951
1608
|
return ((_a = nodeHashCacheList.find((item) => item.node === node)) == null ? void 0 : _a.id) || "";
|
|
1952
1609
|
}
|
|
1953
|
-
const
|
|
1954
|
-
content,
|
|
1955
|
-
rect,
|
|
1956
|
-
_midscene_frame_id: getFrameId()
|
|
1957
|
-
});
|
|
1958
|
-
let sliceLength = 8;
|
|
1959
|
-
let slicedHash = "";
|
|
1960
|
-
const hashHex = import_js_sha256.sha256.create().update(combined).hex();
|
|
1961
|
-
while (sliceLength < hashHex.length - 1) {
|
|
1962
|
-
slicedHash = hashHex.slice(0, sliceLength);
|
|
1963
|
-
if (hashMap[slicedHash] && hashMap[slicedHash] !== combined) {
|
|
1964
|
-
sliceLength++;
|
|
1965
|
-
continue;
|
|
1966
|
-
}
|
|
1967
|
-
hashMap[slicedHash] = combined;
|
|
1968
|
-
break;
|
|
1969
|
-
}
|
|
1610
|
+
const slicedHash = (0, import_utils9.generateHashId)(rect, content);
|
|
1970
1611
|
if (node && typeof window !== "undefined") {
|
|
1971
1612
|
window.midsceneNodeHashCacheList.push({ node, id: slicedHash });
|
|
1972
1613
|
}
|
|
@@ -2162,7 +1803,7 @@ var Page = class {
|
|
|
2162
1803
|
}
|
|
2163
1804
|
async screenshotBase64() {
|
|
2164
1805
|
const { width, height } = await this.size();
|
|
2165
|
-
const path2 = (0,
|
|
1806
|
+
const path2 = (0, import_utils11.getTmpFile)("png");
|
|
2166
1807
|
const screenshotBuffer = await this.browser.saveScreenshot(path2);
|
|
2167
1808
|
const resizedScreenshotBuffer = await (0, import_img2.resizeImg)(screenshotBuffer, {
|
|
2168
1809
|
width,
|
|
@@ -2365,15 +2006,3 @@ var Page = class {
|
|
|
2365
2006
|
AppiumAgent,
|
|
2366
2007
|
AppiumPage
|
|
2367
2008
|
});
|
|
2368
|
-
/*! Bundled license information:
|
|
2369
|
-
|
|
2370
|
-
js-sha256/src/sha256.js:
|
|
2371
|
-
(**
|
|
2372
|
-
* [js-sha256]{@link https://github.com/emn178/js-sha256}
|
|
2373
|
-
*
|
|
2374
|
-
* @version 0.11.0
|
|
2375
|
-
* @author Chen, Yi-Cyuan [emn178@gmail.com]
|
|
2376
|
-
* @copyright Chen, Yi-Cyuan 2014-2024
|
|
2377
|
-
* @license MIT
|
|
2378
|
-
*)
|
|
2379
|
-
*/
|