@intuned/browser-dev 2.2.3-unify-sdks.20

package/dist/optimized-extractors/listExtractionHelpers/__tests__/dynamicListExtractor.spec.js

@@ -0,0 +1,312 @@
+"use strict";
+
+var _extendedTest = require("../../../common/extendedTest");
+var _dynamicListExtractor = require("../dynamicListExtractor");
+var _uuid = require("uuid");
+var _dotenv = require("dotenv");
+var _neverthrow = require("neverthrow");
+var _Logger = require("../../../common/Logger");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+(0, _dotenv.config)();
+const booksTemplate = `
+<div class="books-list">
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">The Great Gatsby</h1>
+      <div class="price-container">
+        <span class="price">$15.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A classic novel about the American Dream</p>
+      </div>
+    </div>
+  </div>
+
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">1984</h1>
+      <div class="price-container">
+        <span class="price">$12.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A dystopian social science fiction novel</p>
+      </div>
+    </div>
+  </div>
+
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">To Kill a Mockingbird</h1>
+      <div class="price-container">
+        <span class="price">$14.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A story of racial injustice and loss of innocence</p>
+      </div>
+    </div>
+  </div>
+</div>
+`;
+(0, _extendedTest.describe)("Dynamic List Extractor Caching Tests", () => {
+  (0, _extendedTest.describe)("DOM Changes and Cache Behavior", () => {
+    (0, _extendedTest.test)("should demonstrate caching behavior with different types of DOM changes", async ({
+      page
+    }) => {
+      const testLabel = `books-cache-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const entitySchema = {
+        type: "object",
+        required: ["name"],
+        properties: {
+          name: {
+            type: "string",
+            description: "book name",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "book price"
+          },
+          description: {
+            type: "string",
+            description: "book description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "book",
+        label: testLabel,
+        itemEntitySchema: entitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      await page.setContent(booksTemplate);
+      const firstResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(firstResult.isOk()).toBe(true);
+      const books = firstResult._unsafeUnwrap();
+      const expectedResult = [{
+        name: {
+          matchText: "The Great Gatsby",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[1]/div[1]/h1[1]"
+        },
+        price: {
+          matchText: "$15.99",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/span[1]"
+        },
+        description: {
+          matchText: "A classic novel about the American Dream",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[1]/div[1]/div[2]/p[1]"
+        }
+      }, {
+        name: {
+          matchText: "1984",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[2]/div[1]/h1[1]"
+        },
+        price: {
+          matchText: "$12.99",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[2]/div[1]/div[1]/span[1]"
+        },
+        description: {
+          matchText: "A dystopian social science fiction novel",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[2]/div[1]/div[2]/p[1]"
+        }
+      }, {
+        name: {
+          matchText: "To Kill a Mockingbird",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[3]/div[1]/h1[1]"
+        },
+        price: {
+          matchText: "$14.99",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[3]/div[1]/div[1]/span[1]"
+        },
+        description: {
+          matchText: "A story of racial injustice and loss of innocence",
+          matchType: "direct-text",
+          matchXpath: "html[1]/body[1]/div[1]/div[3]/div[1]/div[2]/p[1]"
+        }
+      }];
+      (0, _extendedTest.expect)(books).toHaveLength(3);
+      (0, _extendedTest.expect)(books[0]).toEqual(expectedResult[0]);
+      (0, _extendedTest.expect)(books[0]).toHaveProperty("price", expectedResult[0].price);
+      (0, _extendedTest.expect)(books[0]).toHaveProperty("description", expectedResult[0].description);
+      (0, _extendedTest.expect)(books[1]).toEqual(expectedResult[1]);
+      (0, _extendedTest.expect)(books[1]).toHaveProperty("price", expectedResult[1].price);
+      (0, _extendedTest.expect)(books[1]).toHaveProperty("description", expectedResult[1].description);
+      (0, _extendedTest.expect)(books[2]).toEqual(expectedResult[2]);
+      const irrelevantChangeTemplate = booksTemplate.replace('class="books-list"', 'class="books-list featured-books"');
+      await page.setContent(irrelevantChangeTemplate);
+      const secondResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(secondResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(secondResult._unsafeUnwrap()).toEqual(books);
+      const modifiedTemplate = booksTemplate.replace("The Great Gatsby", "The Great Gatsby (Deluxe Edition)").replace("$15.99", "$19.99");
+      await page.setContent(modifiedTemplate);
+      const thirdResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(thirdResult.isOk()).toBe(true);
+      const modifiedBooks = thirdResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(modifiedBooks).not.toEqual(books);
+      (0, _extendedTest.expect)(modifiedBooks[0]).toHaveProperty("name", {
+        matchText: "The Great Gatsby (Deluxe Edition)",
+        matchType: "direct-text",
+        matchXpath: "html[1]/body[1]/div[1]/div[1]/div[1]/h1[1]"
+      });
+      (0, _extendedTest.expect)(modifiedBooks[0]).toHaveProperty("price", {
+        matchText: "$19.99",
+        matchType: "direct-text",
+        matchXpath: "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/span[1]"
+      });
+      const insertedTemplate = `
+        <div class="books-list">
+          <div class="featured-section">
+            <h2>Featured Books</h2>
+          </div>
+          ${booksTemplate.split('<div class="books-list">')[1]}
+      `;
+      await page.setContent(insertedTemplate);
+      const fourthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(fourthResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(fourthResult._unsafeUnwrap()).not.toEqual(modifiedBooks);
+      const appendedTemplate = insertedTemplate.replace("</div>\n</div>", `</div>
+            <div class="book-page">
+                <div class="book-info">
+                    <h1 class="book-title">Dune</h1>
+                    <div class="price-container">
+                        <span class="price">$16.99</span>
+                    </div>
+                    <div class="description">
+                        <p class="book-description">A science fiction masterpiece</p>
+                    </div>
+                </div>
+            </div>
+        </div>`);
+      await page.setContent(appendedTemplate);
+      const fifthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(fifthResult.isOk()).toBe(true);
+      const appendedBooks = fifthResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(appendedBooks).toHaveLength(4);
+      console.log("All cache behavior tests completed successfully!");
+      const outsideTemplate = appendedTemplate + `
+        <div class="outside-books-list">
+          <div class="book-page">
+            <div class="book-info">
+              <h1 class="book-title">Dune</h1>
+            </div>
+          </div>
+        </div>
+      `;
+      await page.setContent(outsideTemplate);
+      const sixthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(sixthResult.isOk()).toBe(true);
+      const outsideBooks = sixthResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(outsideBooks).toHaveLength(4);
+      console.log("All cache behavior tests completed successfully!");
+    });
+    (0, _extendedTest.test)("should handle cache size limit correctly", async ({
+      page
+    }) => {
+      const testLabel = `books-cache-size-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const simpleTemplate = `
+        <div class="books-list">
+          <div class="book-page">
+            <div class="book-info">
+              <h1 class="book-title">Test Book</h1>
+              <div class="price-container">
+                <span class="price">$15.99</span>
+              </div>
+              <div class="description">
+                <p class="book-description">A test book</p>
+              </div>
+            </div>
+          </div>
+        </div>
+      `;
+      const entitySchema = {
+        type: "object",
+        required: ["name"],
+        properties: {
+          name: {
+            type: "string",
+            description: "book name",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "book price"
+          },
+          description: {
+            type: "string",
+            description: "book description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "book",
+        label: testLabel,
+        itemEntitySchema: entitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      const largeMockData = [];
+      for (let i = 0; i < 1000; i++) {
+        largeMockData.push({
+          rowIndex: i,
+          result: {
+            name: {
+              matchText: `Book Title ${i} - This is a very long book title with extensive descriptive text to make the cached data structure large enough to exceed the 380KB limit when we have many items like this in the response`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/h1",
+              matchType: "all-text"
+            },
+            price: {
+              matchText: `$${(15 + i % 50).toFixed(2)}`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/div[1]/span",
+              matchType: "all-text"
+            },
+            description: {
+              matchText: `A very detailed description of book ${i} with extensive content to make the cached data structure large enough to exceed the 380KB limit. This description contains multiple sentences and detailed information about the plot, characters, themes, and critical reception of the book. The description is intentionally verbose to increase the JSON size when serialized for caching purposes. Additional padding text to ensure we reach the size limit for testing cache overflow behavior.`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/div[2]/p",
+              matchType: "all-text"
+            }
+          }
+        });
+      }
+      const runAiExtractionModule = await Promise.resolve().then(() => _interopRequireWildcard(require("../runAiExtraction")));
+      const runAiExtractionSpy = _extendedTest.vi.spyOn(runAiExtractionModule, "runAiExtraction").mockResolvedValue((0, _neverthrow.ok)({
+        resultValues: largeMockData,
+        containerPath: "/html/body/div[1]",
+        matches: new Map()
+      }));
+      const consoleSpy = _extendedTest.vi.spyOn(_Logger.logger, "debug");
+      const consoleInfoSpy = _extendedTest.vi.spyOn(_Logger.logger, "info");
+      const consoleWarnSpy = _extendedTest.vi.spyOn(_Logger.logger, "warn");
+      await page.setContent(simpleTemplate);
+      const firstResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(firstResult.isOk()).toBe(true);
+      const books = firstResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(books.length).toBe(1000);
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("Results exceed cache limit"));
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("skipping caching"));
+      const secondResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(secondResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("exceeds cache limit and is not cacheable"));
+      consoleSpy.mockRestore();
+      runAiExtractionSpy.mockRestore();
+      console.log("Cache size limit test completed successfully!");
+    });
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/findSetOfXpathsToCreateAnArrayExtractor.test.js

@@ -0,0 +1,22 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("find xpaths to create an array", () => {
+  (0, _vitest.it)("basic case | 1 element in smallest", () => {
+    const input = [["html[1]/li[2]/article[1]/h3[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title"], ["html[1]/li[1]/article[1]/h3[1]/a[1]/@title"], ["html[1]/li[4]/article[1]/h3[1]", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/text()"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"]);
+  });
+  (0, _vitest.it)("basic case | 2 elements in smallest", () => {
+    const input = [["html[1]/li[1]/article[1]/h3[1]", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"], ["html[1]/li[2]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/li[3]/article[1]/h3[1]/a[1]/@href", "html[1]/li[3]/article[1]/h3[1]/a[1]/text()", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/@href", "html[1]/li[3]/article[1]/h3[1]/a[1]/@href"]);
+  });
+  (0, _vitest.it)("2 sets | 2 items different depth", () => {
+    const input = [["html[1]/li[1]/article[1]/h3[1]", "html[1]/li[1]/article[1]/h3[1]/a[1]"], ["html[1]/li[2]/article[1]/h3[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]"]);
+  });
+  (0, _vitest.it)("Real world", () => {
+    const input = [["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[5]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[8]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[9]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[11]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[14]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[15]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[18]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/text()"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[5]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[8]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[9]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[11]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[14]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[15]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[18]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title"]);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/getContainerElement.test.js

@@ -0,0 +1,21 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("getContainerElement", () => {
+  (0, _vitest.it)("basic case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"])).toBe("html[1]");
+  });
+  (0, _vitest.it)("real case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title"])).toBe("html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]");
+  });
+  (0, _vitest.it)("empty array", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)([])).toBeNull();
+  });
+  (0, _vitest.it)("no common parent", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[2]/div[1]/span[1]/@text"])).toBeNull();
+  });
+  (0, _vitest.it)("should return the container element even if the list starts at 2", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/body[1]/main[1]/div[3]/div[1]/div[4]/div[2]", "html[1]/body[1]/main[1]/div[3]/div[1]/div[7]/div[2]", "html[1]/body[1]/main[1]/div[3]/div[1]/div[10]/div[2]"])).toBe("html[1]/body[1]/main[1]/div[3]/div[1]");
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/partOfSameArrayXpath.test.js

@@ -0,0 +1,42 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("XPath Matcher", () => {
+  (0, _vitest.it)("basic case", () => {
+    const xpaths = ["html[1]/li[11]/article[1]/h3[1]/a[1]/@href", "html[1]/li[22]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("different structure case", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("multiple numbers in a segment", () => {
+    const xpaths = ["html[1]/li[3]/article[2]/h3[1]/a[1]/@href", "html[1]/li[3]/article[2]/h3[1]/a[2]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("two numeric differences", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[2]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("non-numeric differences", () => {
+    const xpaths = ["html[1]/ul[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("one numeric difference, same structure", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("exact same xpath", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("non-numeric differences in numeric segment", () => {
+    const xpaths = ["html[1]/li[1a]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1b]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("different lengths", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/verifyThatAllXpathsArePartOfSameArray.test.js

@@ -0,0 +1,9 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("verifyThatAllXpathsArePartOfSameArray", () => {
+  (0, _vitest.it)("basic case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.verifyThatAllXpathsArePartOfSameArray)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"])).toBe(true);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/dynamicListExtractor.js

@@ -0,0 +1,152 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.dynamicListExtractor = dynamicListExtractor;
+var _neverthrow = require("neverthrow");
+var _captureSnapshot = require("../objectExtractionHelpers/captureSnapshot");
+var _checksumUtils = require("../objectExtractionHelpers/checksumUtils");
+var _runAiExtraction = require("./runAiExtraction");
+var _validateOptions = require("./utils/validateOptions");
+var _cache = require("../../intunedServices/cache");
+var _xpathMapping = require("../../common/xpathMapping");
+var _Logger = require("../../common/Logger");
+async function dynamicListExtractor(page, identifier, options) {
+  const inputValidation = await (0, _validateOptions.validateDynamicListExtractorOptions)(page, identifier, options);
+  if (inputValidation.isErr()) {
+    return inputValidation;
+  }
+  const {
+    pageUrl,
+    searchRegion,
+    searchRegionHandler,
+    invalidate: _invalidate,
+    itemEntityName,
+    itemEntitySchema,
+    variantKey,
+    primaryProperty,
+    hasSearchRegionContainer,
+    label: _label,
+    searchRegionKey,
+    searchRegionXpath: _searchRegionXpath,
+    strategy,
+    prompt,
+    apiKey
+  } = inputValidation.value;
+  const extractorInputHash = (0, _checksumUtils.hashObject)({
+    itemEntityName,
+    itemEntitySchema,
+    variantKey,
+    currentPageUrl: pageUrl,
+    strategy,
+    searchRegionKey,
+    prompt
+  }, true);
+  const pageAndSearchRegion = {
+    page,
+    searchRegion,
+    searchRegionHandler: searchRegionHandler
+  };
+  const snapshot = await (0, _captureSnapshot.captureSnapshot)(pageAndSearchRegion);
+  if (snapshot.isErr()) {
+    return (0, _neverthrow.err)(snapshot.error);
+  }
+  _Logger.logger.info("Looking for value in the cache...");
+  const cachedResult = await _cache.cache.get(extractorInputHash);
+  if (cachedResult) {
+    _Logger.logger.info("Found value in cache");
+    if (cachedResult.exceedsLimit) {
+      _Logger.logger.warn(`Cache key ${extractorInputHash} exceeds cache limit and is not cacheable`);
+    } else {
+      const isValid = await (0, _xpathMapping.validateXPathMapping)(page, cachedResult.matchesMapping, cachedResult.containerPath);
+      if (isValid) {
+        const nonRelatedChildrenCount = cachedResult.nonRelatedChildrenCount;
+        const currentChildrenCount = await page.evaluate(containerPath => {
+          if (!containerPath) return 0;
+          const containerResult = document.evaluate(containerPath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+          const containerElement = containerResult.singleNodeValue;
+          if (!containerElement) return 0;
+          return containerElement.children.length;
+        }, cachedResult.containerPath);
+        if (currentChildrenCount - nonRelatedChildrenCount === cachedResult.result.length) {
+          _Logger.logger.info("The values in the cache are the same as the current page, returning the cached result");
+          return (0, _neverthrow.ok)(cachedResult.result);
+        }
+        _Logger.logger.info("The values in the cache are different from the current page, running AI extraction");
+      }
+      _Logger.logger.info("The values in the cache are different from the current page, running AI extraction");
+    }
+  } else {
+    _Logger.logger.info("No value found in the cache, running AI extraction");
+  }
+  const aiExtractionResult = await (0, _runAiExtraction.runAiExtraction)({
+    pageAndSearchRegion,
+    itemEntityName,
+    itemEntitySchema,
+    primaryProperty,
+    hasSearchRegionContainer,
+    strategy,
+    identifier,
+    prompt,
+    examples: [],
+    apiKey
+  });
+  if (aiExtractionResult.isErr()) {
+    return (0, _neverthrow.err)(aiExtractionResult.error);
+  }
+  const xpathsMapping = buildXpathsMapping(aiExtractionResult.value);
+  const resultsToReturn = getResultToReturn(aiExtractionResult.value.resultValues);
+  const allContainerChildrenCount = await page.evaluate(containerPath => {
+    if (!containerPath) return 0;
+    const containerResult = document.evaluate(containerPath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
+    const containerElement = containerResult.singleNodeValue;
+    if (!containerElement) return 0;
+    return containerElement.children.length;
+  }, aiExtractionResult.value.containerPath);
+  const nonRelatedChildrenCount = allContainerChildrenCount - resultsToReturn.length;
+  const resultsToCache = {
+    result: resultsToReturn,
+    matchesMapping: xpathsMapping,
+    containerPath: aiExtractionResult.value.containerPath,
+    nonRelatedChildrenCount
+  };
+  const cacheDataSize = JSON.stringify(resultsToCache).length;
+  const CACHE_SIZE_LIMIT = 380 * 1024;
+  if (cacheDataSize > CACHE_SIZE_LIMIT) {
+    _Logger.logger.warn(`Results exceed cache limit (${cacheDataSize} bytes > ${CACHE_SIZE_LIMIT} bytes), skipping caching`);
+    await _cache.cache.set(extractorInputHash, {
+      exceedsLimit: true
+    });
+  } else {
+    _Logger.logger.debug("Caching results...");
+    await _cache.cache.set(extractorInputHash, resultsToCache);
+    _Logger.logger.debug("Results cached");
+  }
+  return (0, _neverthrow.ok)(resultsToReturn);
+}
+function buildXpathsMapping(results) {
+  const containerXpath = results.containerPath;
+  const xpathsMapping = {};
+  for (const result of results.resultValues) {
+    for (const [_key, valueObj] of Object.entries(result.result)) {
+      const value = valueObj;
+      const matchedXpath = value.matchXpath;
+      if (matchedXpath) {
+        const relativePath = matchedXpath.replace(containerXpath + "/", "");
+        const xpathEntry = {
+          xpath: relativePath,
+          matchType: value.matchType
+        };
+        if (!xpathsMapping[value.matchText]) {
+          xpathsMapping[value.matchText] = [];
+        }
+        xpathsMapping[value.matchText].push(xpathEntry);
+      }
+    }
+  }
+  return xpathsMapping;
+}
+function getResultToReturn(extractionResult) {
+  return extractionResult.map(result => result.result);
+}

package/dist/optimized-extractors/listExtractionHelpers/errors.js

@@ -0,0 +1,46 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.requiredPropertyNotExtracted = exports.other = exports.invalidSearchRegion = exports.invalidList = exports.invalidInput = exports.invalidExtractionResult = exports.invalidAddressUrl = exports.insufficientAiCredits = void 0;
+const other = (context, error) => ({
+  type: "Other",
+  context,
+  error
+});
+exports.other = other;
+const invalidSearchRegion = () => ({
+  type: "InvalidSearchRegion"
+});
+exports.invalidSearchRegion = invalidSearchRegion;
+const invalidList = () => ({
+  type: "InvalidList",
+  context: "Failed to get container path, please try to choose another primary property and make sure all list items are direct child of a common container"
+});
+exports.invalidList = invalidList;
+const invalidInput = context => ({
+  type: "InvalidInput",
+  context
+});
+exports.invalidInput = invalidInput;
+const invalidExtractionResult = context => ({
+  type: "InvalidExtractionResult",
+  context
+});
+exports.invalidExtractionResult = invalidExtractionResult;
+const invalidAddressUrl = context => ({
+  type: "InvalidAddressUrl",
+  context
+});
+exports.invalidAddressUrl = invalidAddressUrl;
+const requiredPropertyNotExtracted = context => ({
+  type: "RequiredPropertyNotExtracted",
+  context
+});
+exports.requiredPropertyNotExtracted = requiredPropertyNotExtracted;
+const insufficientAiCredits = context => ({
+  type: "InsufficientAiCredits",
+  context
+});
+exports.insufficientAiCredits = insufficientAiCredits;

package/dist/optimized-extractors/listExtractionHelpers/getListMatches.js

@@ -0,0 +1,14 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getListMatches = getListMatches;
+var _noEmpty = _interopRequireDefault(require("../../common/noEmpty"));
+var _findDomMatches = require("../objectExtractionHelpers/findDomMatches");
+function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
+async function getListMatches(page, searchRegionHandler, results) {
+  const allExtractedValues = results.map(v => Object.values(v.result)).filter(_noEmpty.default).flat();
+  const allExtractedValuesDomMatches = await (0, _findDomMatches.getDomMatches)(page, searchRegionHandler, allExtractedValues);
+  return allExtractedValuesDomMatches;
+}