@midscene/web 0.4.0 → 0.4.1-beta-20240827150150.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/es/index.js +8 -2
  2. package/dist/es/playwright.js +8 -2
  3. package/dist/es/puppeteer.js +8 -2
  4. package/dist/lib/index.js +8 -2
  5. package/dist/lib/playwright.js +8 -2
  6. package/dist/lib/puppeteer.js +8 -2
  7. package/dist/script/htmlElement.js +27 -1
  8. package/dist/script/htmlElementDebug.js +28 -1
  9. package/package.json +3 -3
package/dist/es/index.js CHANGED
@@ -560,7 +560,8 @@ var PageTaskExecutor = class {
560
560
  type: "Insight",
561
561
  subType: "Locate",
562
562
  param: plan2.param,
563
- executor: async (param) => {
563
+ executor: async (param, taskContext) => {
564
+ const { task } = taskContext;
564
565
  let insightDump;
565
566
  const dumpCollector = (dump) => {
566
567
  insightDump = dump;
@@ -585,7 +586,6 @@ var PageTaskExecutor = class {
585
586
  return locateResult;
586
587
  }
587
588
  });
588
- assert2(element, `Element not found: ${param.prompt}`);
589
589
  if (locateResult) {
590
590
  this.taskCache.saveCache({
591
591
  type: "locate",
@@ -597,6 +597,12 @@ var PageTaskExecutor = class {
597
597
  response: locateResult
598
598
  });
599
599
  }
600
+ if (!element) {
601
+ task.log = {
602
+ dump: insightDump
603
+ };
604
+ throw new Error(`Element not found: ${param.prompt}`);
605
+ }
600
606
  return {
601
607
  output: {
602
608
  element
package/dist/es/playwright.js CHANGED
@@ -560,7 +560,8 @@ var PageTaskExecutor = class {
560
560
  type: "Insight",
561
561
  subType: "Locate",
562
562
  param: plan2.param,
563
- executor: async (param) => {
563
+ executor: async (param, taskContext) => {
564
+ const { task } = taskContext;
564
565
  let insightDump;
565
566
  const dumpCollector = (dump) => {
566
567
  insightDump = dump;
@@ -585,7 +586,6 @@ var PageTaskExecutor = class {
585
586
  return locateResult;
586
587
  }
587
588
  });
588
- assert2(element, `Element not found: ${param.prompt}`);
589
589
  if (locateResult) {
590
590
  this.taskCache.saveCache({
591
591
  type: "locate",
@@ -597,6 +597,12 @@ var PageTaskExecutor = class {
597
597
  response: locateResult
598
598
  });
599
599
  }
600
+ if (!element) {
601
+ task.log = {
602
+ dump: insightDump
603
+ };
604
+ throw new Error(`Element not found: ${param.prompt}`);
605
+ }
600
606
  return {
601
607
  output: {
602
608
  element
package/dist/es/puppeteer.js CHANGED
@@ -557,7 +557,8 @@ var PageTaskExecutor = class {
557
557
  type: "Insight",
558
558
  subType: "Locate",
559
559
  param: plan2.param,
560
- executor: async (param) => {
560
+ executor: async (param, taskContext) => {
561
+ const { task } = taskContext;
561
562
  let insightDump;
562
563
  const dumpCollector = (dump) => {
563
564
  insightDump = dump;
@@ -582,7 +583,6 @@ var PageTaskExecutor = class {
582
583
  return locateResult;
583
584
  }
584
585
  });
585
- assert2(element, `Element not found: ${param.prompt}`);
586
586
  if (locateResult) {
587
587
  this.taskCache.saveCache({
588
588
  type: "locate",
@@ -594,6 +594,12 @@ var PageTaskExecutor = class {
594
594
  response: locateResult
595
595
  });
596
596
  }
597
+ if (!element) {
598
+ task.log = {
599
+ dump: insightDump
600
+ };
601
+ throw new Error(`Element not found: ${param.prompt}`);
602
+ }
597
603
  return {
598
604
  output: {
599
605
  element
package/dist/lib/index.js CHANGED
@@ -568,7 +568,8 @@ var PageTaskExecutor = class {
568
568
  type: "Insight",
569
569
  subType: "Locate",
570
570
  param: plan2.param,
571
- executor: async (param) => {
571
+ executor: async (param, taskContext) => {
572
+ const { task } = taskContext;
572
573
  let insightDump;
573
574
  const dumpCollector = (dump) => {
574
575
  insightDump = dump;
@@ -593,7 +594,6 @@ var PageTaskExecutor = class {
593
594
  return locateResult;
594
595
  }
595
596
  });
596
- (0, import_node_assert2.default)(element, `Element not found: ${param.prompt}`);
597
597
  if (locateResult) {
598
598
  this.taskCache.saveCache({
599
599
  type: "locate",
@@ -605,6 +605,12 @@ var PageTaskExecutor = class {
605
605
  response: locateResult
606
606
  });
607
607
  }
608
+ if (!element) {
609
+ task.log = {
610
+ dump: insightDump
611
+ };
612
+ throw new Error(`Element not found: ${param.prompt}`);
613
+ }
608
614
  return {
609
615
  output: {
610
616
  element
package/dist/lib/playwright.js CHANGED
@@ -566,7 +566,8 @@ var PageTaskExecutor = class {
566
566
  type: "Insight",
567
567
  subType: "Locate",
568
568
  param: plan2.param,
569
- executor: async (param) => {
569
+ executor: async (param, taskContext) => {
570
+ const { task } = taskContext;
570
571
  let insightDump;
571
572
  const dumpCollector = (dump) => {
572
573
  insightDump = dump;
@@ -591,7 +592,6 @@ var PageTaskExecutor = class {
591
592
  return locateResult;
592
593
  }
593
594
  });
594
- (0, import_node_assert2.default)(element, `Element not found: ${param.prompt}`);
595
595
  if (locateResult) {
596
596
  this.taskCache.saveCache({
597
597
  type: "locate",
@@ -603,6 +603,12 @@ var PageTaskExecutor = class {
603
603
  response: locateResult
604
604
  });
605
605
  }
606
+ if (!element) {
607
+ task.log = {
608
+ dump: insightDump
609
+ };
610
+ throw new Error(`Element not found: ${param.prompt}`);
611
+ }
606
612
  return {
607
613
  output: {
608
614
  element
package/dist/lib/puppeteer.js CHANGED
@@ -563,7 +563,8 @@ var PageTaskExecutor = class {
563
563
  type: "Insight",
564
564
  subType: "Locate",
565
565
  param: plan2.param,
566
- executor: async (param) => {
566
+ executor: async (param, taskContext) => {
567
+ const { task } = taskContext;
567
568
  let insightDump;
568
569
  const dumpCollector = (dump) => {
569
570
  insightDump = dump;
@@ -588,7 +589,6 @@ var PageTaskExecutor = class {
588
589
  return locateResult;
589
590
  }
590
591
  });
591
- (0, import_node_assert2.default)(element, `Element not found: ${param.prompt}`);
592
592
  if (locateResult) {
593
593
  this.taskCache.saveCache({
594
594
  type: "locate",
@@ -600,6 +600,12 @@ var PageTaskExecutor = class {
600
600
  response: locateResult
601
601
  });
602
602
  }
603
+ if (!element) {
604
+ task.log = {
605
+ dump: insightDump
606
+ };
607
+ throw new Error(`Element not found: ${param.prompt}`);
608
+ }
603
609
  return {
604
610
  output: {
605
611
  element
package/dist/script/htmlElement.js CHANGED
@@ -602,6 +602,9 @@ var midscene_element_inspector = (() => {
602
602
  function isTextElement(node) {
603
603
  return node.nodeName.toLowerCase() === "#text";
604
604
  }
605
+ function isWidgetElement(node) {
606
+ return node instanceof HTMLElement && (node.hasAttribute("aria-label") || node.hasAttribute("aria-controls") || node.hasAttribute("aria-labelledby"));
607
+ }
605
608
 
606
609
  // src/extractor/util.ts
607
610
  var import_js_sha256 = __toESM(require_sha256());
@@ -734,7 +737,7 @@ var midscene_element_inspector = (() => {
734
737
  window.generateHash = generateHash;
735
738
 
736
739
  // src/extractor/extractor.ts
737
- var container = document.body;
740
+ var container = document.body || document;
738
741
  function generateId(numberId) {
739
742
  return `${numberId}`;
740
743
  }
@@ -881,6 +884,29 @@ var midscene_element_inspector = (() => {
881
884
  });
882
885
  return;
883
886
  }
887
+ if (isWidgetElement(node)) {
888
+ const attributes = getNodeAttributes(node);
889
+ const nodeHashId = generateHash("", rect);
890
+ const selector = setDataForNode(node, nodeHashId);
891
+ elementInfoArray.push({
892
+ id: nodeHashId,
893
+ indexId: generateId(nodeIndex++),
894
+ nodeHashId,
895
+ nodeType: NodeType.FORM_ITEM,
896
+ locator: selector,
897
+ attributes: __spreadProps(__spreadValues({}, attributes), {
898
+ nodeType: NodeType.FORM_ITEM
899
+ }),
900
+ content: "",
901
+ rect,
902
+ center: [
903
+ Math.round(rect.left + rect.width / 2),
904
+ Math.round(rect.top + rect.height / 2)
905
+ ],
906
+ htmlNode: debugMode2 ? node : null
907
+ });
908
+ return true;
909
+ }
884
910
  return true;
885
911
  }
886
912
  dfs(initNode, nodeMapTree);
package/dist/script/htmlElementDebug.js CHANGED
@@ -591,6 +591,9 @@ var midscene_element_inspector = (() => {
591
591
  function isTextElement(node) {
592
592
  return node.nodeName.toLowerCase() === "#text";
593
593
  }
594
+ function isWidgetElement(node) {
595
+ return node instanceof HTMLElement && (node.hasAttribute("aria-label") || node.hasAttribute("aria-controls") || node.hasAttribute("aria-labelledby"));
596
+ }
594
597
 
595
598
  // src/extractor/util.ts
596
599
  var import_js_sha256 = __toESM(require_sha256());
@@ -723,7 +726,7 @@ var midscene_element_inspector = (() => {
723
726
  window.generateHash = generateHash;
724
727
 
725
728
  // src/extractor/extractor.ts
726
- var container = document.body;
729
+ var container = document.body || document;
727
730
  function generateId(numberId) {
728
731
  return `${numberId}`;
729
732
  }
@@ -870,6 +873,29 @@ var midscene_element_inspector = (() => {
870
873
  });
871
874
  return;
872
875
  }
876
+ if (isWidgetElement(node)) {
877
+ const attributes = getNodeAttributes(node);
878
+ const nodeHashId = generateHash("", rect);
879
+ const selector = setDataForNode(node, nodeHashId);
880
+ elementInfoArray.push({
881
+ id: nodeHashId,
882
+ indexId: generateId(nodeIndex++),
883
+ nodeHashId,
884
+ nodeType: NodeType.FORM_ITEM,
885
+ locator: selector,
886
+ attributes: __spreadProps(__spreadValues({}, attributes), {
887
+ nodeType: NodeType.FORM_ITEM
888
+ }),
889
+ content: "",
890
+ rect,
891
+ center: [
892
+ Math.round(rect.left + rect.width / 2),
893
+ Math.round(rect.top + rect.height / 2)
894
+ ],
895
+ htmlNode: debugMode2 ? node : null
896
+ });
897
+ return true;
898
+ }
873
899
  return true;
874
900
  }
875
901
  dfs(initNode, nodeMapTree);
@@ -879,6 +905,7 @@ var midscene_element_inspector = (() => {
879
905
  // src/extractor/debug.ts
880
906
  console.log(extractTextWithPosition(document.body, true));
881
907
  console.log(JSON.stringify(extractTextWithPosition(document.body, false)));
908
+ window.extractTextWithPosition = extractTextWithPosition;
882
909
  })();
883
910
  /*! Bundled license information:
884
911
 
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@midscene/web",
3
3
  "description": "Web integration for Midscene.js",
4
- "version": "0.4.0",
4
+ "version": "0.4.1-beta-20240827150150.0",
5
5
  "jsnext:source": "./src/index.ts",
6
6
  "main": "./dist/lib/index.js",
7
7
  "module": "./dist/es/index.js",
@@ -75,8 +75,8 @@
75
75
  "dependencies": {
76
76
  "openai": "4.47.1",
77
77
  "inquirer": "10.1.5",
78
- "@midscene/core": "0.4.0",
79
- "@midscene/shared": "0.4.0"
78
+ "@midscene/core": "0.4.1-beta-20240827150150.0",
79
+ "@midscene/shared": "0.4.1-beta-20240827150150.0"
80
80
  },
81
81
  "devDependencies": {
82
82
  "@modern-js/module-tools": "2.58.2",