@intuned/browser-dev 0.1.4-dev.0

package/dist/optimized-extractors/listExtractionHelpers/__tests__/dynamicListExtractor.spec.js

@@ -0,0 +1,269 @@
+"use strict";
+
+var _extendedTest = require("../../../common/extendedTest");
+var _dynamicListExtractor = require("../dynamicListExtractor");
+var _uuid = require("uuid");
+var _dotenv = require("dotenv");
+var _neverthrow = require("neverthrow");
+var _Logger = require("../../../common/Logger");
+function _interopRequireWildcard(e, t) { if ("function" == typeof WeakMap) var r = new WeakMap(), n = new WeakMap(); return (_interopRequireWildcard = function (e, t) { if (!t && e && e.__esModule) return e; var o, i, f = { __proto__: null, default: e }; if (null === e || "object" != typeof e && "function" != typeof e) return f; if (o = t ? n : r) { if (o.has(e)) return o.get(e); o.set(e, f); } for (const t in e) "default" !== t && {}.hasOwnProperty.call(e, t) && ((i = (o = Object.defineProperty) && Object.getOwnPropertyDescriptor(e, t)) && (i.get || i.set) ? o(f, t, i) : f[t] = e[t]); return f; })(e, t); }
+(0, _dotenv.config)();
+const booksTemplate = `
+<div class="books-list">
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">The Great Gatsby</h1>
+      <div class="price-container">
+        <span class="price">$15.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A classic novel about the American Dream</p>
+      </div>
+    </div>
+  </div>
+
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">1984</h1>
+      <div class="price-container">
+        <span class="price">$12.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A dystopian social science fiction novel</p>
+      </div>
+    </div>
+  </div>
+
+  <div class="book-page">
+    <div class="book-info">
+      <h1 class="book-title">To Kill a Mockingbird</h1>
+      <div class="price-container">
+        <span class="price">$14.99</span>
+      </div>
+      <div class="description">
+        <p class="book-description">A story of racial injustice and loss of innocence</p>
+      </div>
+    </div>
+  </div>
+</div>
+`;
+_extendedTest.describe.skip("Dynamic List Extractor Caching Tests", () => {
+  (0, _extendedTest.describe)("DOM Changes and Cache Behavior", () => {
+    (0, _extendedTest.test)("should demonstrate caching behavior with different types of DOM changes", async ({
+      page
+    }) => {
+      const testLabel = `books-cache-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const entitySchema = {
+        type: "object",
+        required: ["name"],
+        properties: {
+          name: {
+            type: "string",
+            description: "book name",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "book price"
+          },
+          description: {
+            type: "string",
+            description: "book description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "book",
+        label: testLabel,
+        itemEntitySchema: entitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      await page.setContent(booksTemplate);
+      const firstResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(firstResult.isOk()).toBe(true);
+      const books = firstResult._unsafeUnwrap();
+      const expectedResult = [{
+        name: "The Great Gatsby",
+        price: "$15.99",
+        description: "A classic novel about the American Dream"
+      }, {
+        name: "1984",
+        price: "$12.99",
+        description: "A dystopian social science fiction novel"
+      }, {
+        name: "To Kill a Mockingbird",
+        price: "$14.99",
+        description: "A story of racial injustice and loss of innocence"
+      }];
+      (0, _extendedTest.expect)(books).toHaveLength(3);
+      (0, _extendedTest.expect)(books[0]).toEqual(expectedResult[0]);
+      (0, _extendedTest.expect)(books[0]).toHaveProperty("price", expectedResult[0].price);
+      (0, _extendedTest.expect)(books[0]).toHaveProperty("description", expectedResult[0].description);
+      (0, _extendedTest.expect)(books[1]).toEqual(expectedResult[1]);
+      (0, _extendedTest.expect)(books[1]).toHaveProperty("price", expectedResult[1].price);
+      (0, _extendedTest.expect)(books[1]).toHaveProperty("description", expectedResult[1].description);
+      (0, _extendedTest.expect)(books[2]).toEqual(expectedResult[2]);
+      const irrelevantChangeTemplate = booksTemplate.replace('class="books-list"', 'class="books-list featured-books"');
+      await page.setContent(irrelevantChangeTemplate);
+      const secondResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(secondResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(secondResult._unsafeUnwrap()).toEqual(books);
+      const modifiedTemplate = booksTemplate.replace("The Great Gatsby", "The Great Gatsby (Deluxe Edition)").replace("$15.99", "$19.99");
+      await page.setContent(modifiedTemplate);
+      const thirdResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(thirdResult.isOk()).toBe(true);
+      const modifiedBooks = thirdResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(modifiedBooks).not.toEqual(books);
+      (0, _extendedTest.expect)(modifiedBooks[0]).toHaveProperty("name", "The Great Gatsby (Deluxe Edition)");
+      (0, _extendedTest.expect)(modifiedBooks[0]).toHaveProperty("price", "$19.99");
+      const insertedTemplate = `
+        <div class="books-list">
+          <div class="featured-section">
+            <h2>Featured Books</h2>
+          </div>
+          ${booksTemplate.split('<div class="books-list">')[1]}
+      `;
+      await page.setContent(insertedTemplate);
+      const fourthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(fourthResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(fourthResult._unsafeUnwrap()).not.toEqual(modifiedBooks);
+      const appendedTemplate = insertedTemplate.replace("</div>\n</div>", `</div>
+            <div class="book-page">
+                <div class="book-info">
+                    <h1 class="book-title">Dune</h1>
+                    <div class="price-container">
+                        <span class="price">$16.99</span>
+                    </div>
+                    <div class="description">
+                        <p class="book-description">A science fiction masterpiece</p>
+                    </div>
+                </div>
+            </div>
+        </div>`);
+      await page.setContent(appendedTemplate);
+      const fifthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(fifthResult.isOk()).toBe(true);
+      const appendedBooks = fifthResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(appendedBooks).toHaveLength(4);
+      _Logger.logger.info("All cache behavior tests completed successfully!");
+      const outsideTemplate = appendedTemplate + `
+        <div class="outside-books-list">
+          <div class="book-page">
+            <div class="book-info">
+              <h1 class="book-title">Dune</h1>
+            </div>
+          </div>
+        </div>
+      `;
+      await page.setContent(outsideTemplate);
+      const sixthResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(sixthResult.isOk()).toBe(true);
+      const outsideBooks = sixthResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(outsideBooks).toHaveLength(4);
+      _Logger.logger.info("All cache behavior tests completed successfully!");
+    });
+    (0, _extendedTest.test)("should handle cache size limit correctly", async ({
+      page
+    }) => {
+      const testLabel = `books-cache-size-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const simpleTemplate = `
+        <div class="books-list">
+          <div class="book-page">
+            <div class="book-info">
+              <h1 class="book-title">Test Book</h1>
+              <div class="price-container">
+                <span class="price">$15.99</span>
+              </div>
+              <div class="description">
+                <p class="book-description">A test book</p>
+              </div>
+            </div>
+          </div>
+        </div>
+      `;
+      const entitySchema = {
+        type: "object",
+        required: ["name"],
+        properties: {
+          name: {
+            type: "string",
+            description: "book name",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "book price"
+          },
+          description: {
+            type: "string",
+            description: "book description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "book",
+        label: testLabel,
+        itemEntitySchema: entitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      const largeMockData = [];
+      for (let i = 0; i < 1000; i++) {
+        largeMockData.push({
+          rowIndex: i,
+          result: {
+            name: {
+              matchText: `Book Title ${i} - This is a very long book title with extensive descriptive text to make the cached data structure large enough to exceed the 380KB limit when we have many items like this in the response`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/h1",
+              matchType: "direct-text"
+            },
+            price: {
+              matchText: `$${(15 + i % 50).toFixed(2)}`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/div[1]/span",
+              matchType: "direct-text"
+            },
+            description: {
+              matchText: `A very detailed description of book ${i} with extensive content to make the cached data structure large enough to exceed the 380KB limit. This description contains multiple sentences and detailed information about the plot, characters, themes, and critical reception of the book. The description is intentionally verbose to increase the JSON size when serialized for caching purposes. Additional padding text to ensure we reach the size limit for testing cache overflow behavior.`,
+              matchXpath: "/html/body/div[1]/div[1]/div[1]/div[2]/p",
+              matchType: "direct-text"
+            }
+          }
+        });
+      }
+      const runAiExtractionModule = await Promise.resolve().then(() => _interopRequireWildcard(require("../runAiExtraction")));
+      const runAiExtractionSpy = _extendedTest.vi.spyOn(runAiExtractionModule, "runAiExtraction").mockResolvedValue((0, _neverthrow.ok)({
+        resultValues: largeMockData,
+        containerPath: "/html/body/div[1]",
+        matches: new Map(),
+        fullContainerXpath: "/html/body/div[1]"
+      }));
+      const consoleSpy = _extendedTest.vi.spyOn(_Logger.logger, "debug");
+      const consoleInfoSpy = _extendedTest.vi.spyOn(_Logger.logger, "info");
+      const consoleWarnSpy = _extendedTest.vi.spyOn(_Logger.logger, "warn");
+      await page.setContent(simpleTemplate);
+      const firstResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(firstResult.isOk()).toBe(true);
+      const books = firstResult._unsafeUnwrap();
+      (0, _extendedTest.expect)(books.length).toBe(1000);
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("Results exceed cache limit"));
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("skipping caching"));
+      const secondResult = await (0, _dynamicListExtractor.dynamicListExtractor)(page, ".books-list", extractionOptions);
+      (0, _extendedTest.expect)(secondResult.isOk()).toBe(true);
+      (0, _extendedTest.expect)(consoleWarnSpy).toHaveBeenCalledWith(_extendedTest.expect.stringContaining("exceeds cache limit and is not cacheable"));
+      consoleSpy.mockRestore();
+      runAiExtractionSpy.mockRestore();
+      console.log("Cache size limit test completed successfully!");
+    });
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/findSetOfXpathsToCreateAnArrayExtractor.test.js

@@ -0,0 +1,22 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("find xpaths to create an array", () => {
+  (0, _vitest.it)("basic case | 1 element in smallest", () => {
+    const input = [["html[1]/li[2]/article[1]/h3[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title"], ["html[1]/li[1]/article[1]/h3[1]/a[1]/@title"], ["html[1]/li[4]/article[1]/h3[1]", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/text()"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"]);
+  });
+  (0, _vitest.it)("basic case | 2 elements in smallest", () => {
+    const input = [["html[1]/li[1]/article[1]/h3[1]", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"], ["html[1]/li[2]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/li[3]/article[1]/h3[1]/a[1]/@href", "html[1]/li[3]/article[1]/h3[1]/a[1]/text()", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/@href", "html[1]/li[3]/article[1]/h3[1]/a[1]/@href"]);
+  });
+  (0, _vitest.it)("2 sets | 2 items different depth", () => {
+    const input = [["html[1]/li[1]/article[1]/h3[1]", "html[1]/li[1]/article[1]/h3[1]/a[1]"], ["html[1]/li[2]/article[1]/h3[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/li[1]/article[1]/h3[1]/a[1]", "html[1]/li[2]/article[1]/h3[1]/a[1]"]);
+  });
+  (0, _vitest.it)("Real world", () => {
+    const input = [["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[5]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[8]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[9]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[11]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[14]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[15]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[18]/article[1]/h3[1]/a[1]/@title"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/text()"], ["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/text()"]];
+    (0, _vitest.expect)((0, _getListContainerXpath.findSetOfXpathsToCreateAnArrayExtractor)(input)).toEqual(["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[5]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[6]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[8]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[9]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[11]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[12]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[14]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[15]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[17]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[18]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[20]/article[1]/h3[1]/a[1]/@title"]);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/getContainerElement.test.js

@@ -0,0 +1,21 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("getContainerElement", () => {
+  (0, _vitest.it)("basic case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/li[2]/article[1]/h3[1]/a[1]/@title", "html[1]/li[3]/article[1]/h3[1]/a[1]/@title", "html[1]/li[4]/article[1]/h3[1]/a[1]/@title"])).toBe("html[1]");
+  });
+  (0, _vitest.it)("real case", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[4]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[7]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[10]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[13]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[16]/article[1]/h3[1]/a[1]/@title", "html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]/li[19]/article[1]/h3[1]/a[1]/@title"])).toBe("html[1]/body[1]/div[1]/div[1]/div[1]/div[1]/section[1]/div[2]/ol[1]");
+  });
+  (0, _vitest.it)("empty array", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)([])).toBeNull();
+  });
+  (0, _vitest.it)("no common parent", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/li[1]/article[1]/h3[1]/a[1]/@title", "html[2]/div[1]/span[1]/@text"])).toBeNull();
+  });
+  (0, _vitest.it)("should return the container element even if the list starts at 2", () => {
+    (0, _vitest.expect)((0, _getListContainerXpath.getContainerElement)(["html[1]/body[1]/main[1]/div[3]/div[1]/div[4]/div[2]", "html[1]/body[1]/main[1]/div[3]/div[1]/div[7]/div[2]", "html[1]/body[1]/main[1]/div[3]/div[1]/div[10]/div[2]"])).toBe("html[1]/body[1]/main[1]/div[3]/div[1]");
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/partOfSameArrayXpath.test.js

@@ -0,0 +1,42 @@
+"use strict";
+
+var _getListContainerXpath = require("../utils/getListContainerXpath");
+var _vitest = require("vitest");
+(0, _vitest.describe)("XPath Matcher", () => {
+  (0, _vitest.it)("basic case", () => {
+    const xpaths = ["html[1]/li[11]/article[1]/h3[1]/a[1]/@href", "html[1]/li[22]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("different structure case", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/text()"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("multiple numbers in a segment", () => {
+    const xpaths = ["html[1]/li[3]/article[2]/h3[1]/a[1]/@href", "html[1]/li[3]/article[2]/h3[1]/a[2]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("two numeric differences", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[2]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("non-numeric differences", () => {
+    const xpaths = ["html[1]/ul[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("one numeric difference, same structure", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[2]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(true);
+  });
+  (0, _vitest.it)("exact same xpath", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("non-numeric differences in numeric segment", () => {
+    const xpaths = ["html[1]/li[1a]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1b]/article[1]/h3[1]/a[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+  (0, _vitest.it)("different lengths", () => {
+    const xpaths = ["html[1]/li[1]/article[1]/h3[1]/a[1]/@href", "html[1]/li[1]/article[1]/h3[1]/@href"];
+    (0, _vitest.expect)((0, _getListContainerXpath.partOfSameArrayXpath)(xpaths[0], xpaths[1])).toBe(false);
+  });
+});

package/dist/optimized-extractors/listExtractionHelpers/__tests__/testArrayExtractorFromLocator.spec.js

@@ -0,0 +1,146 @@
+"use strict";
+
+var _extendedTest = require("../../../common/extendedTest");
+var _ = require("../..");
+var _uuid = require("uuid");
+const productListTemplate = `
+<div class="products-container">
+  <div class="product-item">
+    <h2 class="product-title">iPhone 14 Pro</h2>
+    <div class="price-wrapper">
+      <span class="price">$999</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Latest iPhone with advanced camera system</p>
+    </div>
+  </div>
+  <div class="product-item">
+    <h2 class="product-title">MacBook Air M2</h2>
+    <div class="price-wrapper">
+      <span class="price">$1199</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Thin and light laptop with M2 chip</p>
+    </div>
+  </div>
+  <div class="product-item">
+    <h2 class="product-title">AirPods Pro</h2>
+    <div class="price-wrapper">
+      <span class="price">$249</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Active noise cancellation earbuds</p>
+    </div>
+  </div>
+  <div class="additional-info">
+    <div class="shipping-notice">Free shipping on all orders</div>
+    <div class="return-policy">30-day return policy</div>
+  </div>
+</div>
+`;
+_extendedTest.describe.skip("Array Extractor Caching Tests", () => {
+  (0, _extendedTest.describe)("DOM Changes and Cache Behavior", () => {
+    (0, _extendedTest.test)("should demonstrate caching behavior with different types of DOM changes", async ({
+      page
+    }) => {
+      const testLabel = `product-list-cache-test-${(0, _uuid.v4)()}`;
+      const variantKey = testLabel;
+      const itemEntitySchema = {
+        type: "object",
+        required: ["title", "price"],
+        properties: {
+          title: {
+            type: "string",
+            description: "Product title",
+            primary: true
+          },
+          price: {
+            type: "string",
+            description: "Product price"
+          },
+          description: {
+            type: "string",
+            description: "Product description"
+          }
+        }
+      };
+      const extractionOptions = {
+        itemEntityName: "product",
+        label: testLabel,
+        itemEntitySchema,
+        strategy: {
+          model: "claude-3-5-sonnet-20240620",
+          type: "HTML"
+        },
+        variantKey: variantKey,
+        apiKey: process.env.ANTHROPIC_API_KEY
+      };
+      await page.setContent(productListTemplate);
+      const firstResult = await (0, _.extractArrayFromLocator)(page.locator(".products-container"), extractionOptions);
+      console.log("First extraction result:", firstResult);
+      (0, _extendedTest.expect)(firstResult).toHaveLength(3);
+      (0, _extendedTest.expect)(firstResult[0]).toHaveProperty("title", "iPhone 14 Pro");
+      (0, _extendedTest.expect)(firstResult[0]).toHaveProperty("price", "$999");
+      (0, _extendedTest.expect)(firstResult[1]).toHaveProperty("title", "MacBook Air M2");
+      (0, _extendedTest.expect)(firstResult[1]).toHaveProperty("price", "$1199");
+      (0, _extendedTest.expect)(firstResult[2]).toHaveProperty("title", "AirPods Pro");
+      (0, _extendedTest.expect)(firstResult[2]).toHaveProperty("price", "$249");
+      const modifiedTemplate = productListTemplate.replace("iPhone 14 Pro", "iPhone 15 Pro Max").replace("$999", "$1099").replace("MacBook Air M2", "MacBook Pro M3").replace("$1199", "$1999");
+      await page.setContent(modifiedTemplate);
+      const secondResult = await (0, _.extractArrayFromLocator)(page.locator(".products-container"), extractionOptions);
+      console.log("Second extraction result (after relevant change):", secondResult);
+      (0, _extendedTest.expect)(secondResult).not.toEqual(firstResult);
+      (0, _extendedTest.expect)(secondResult).toHaveLength(3);
+      (0, _extendedTest.expect)(secondResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(secondResult[0]).toHaveProperty("price", "$1099");
+      (0, _extendedTest.expect)(secondResult[1]).toHaveProperty("title", "MacBook Pro M3");
+      (0, _extendedTest.expect)(secondResult[1]).toHaveProperty("price", "$1999");
+      (0, _extendedTest.expect)(secondResult[2]).toHaveProperty("title", "AirPods Pro");
+      (0, _extendedTest.expect)(secondResult[2]).toHaveProperty("price", "$249");
+      const irrelevantChangeTemplate = modifiedTemplate.replace("Free shipping on all orders", "Express shipping available").replace("30-day return policy", "60-day return policy");
+      await page.setContent(irrelevantChangeTemplate);
+      const thirdResult = await (0, _.extractArrayFromLocator)(page.locator(".products-container"), extractionOptions);
+      console.log("Third extraction result (after irrelevant change):", thirdResult);
+      (0, _extendedTest.expect)(thirdResult).toEqual(secondResult);
+      (0, _extendedTest.expect)(thirdResult).toHaveLength(3);
+      (0, _extendedTest.expect)(thirdResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(thirdResult[0]).toHaveProperty("price", "$1099");
+      const appendedTemplate = irrelevantChangeTemplate.replace('<div class="return-policy">60-day return policy</div>', `<div class="return-policy">60-day return policy</div>
+    <div class="newly-added-section">
+      <div class="customer-service">
+        <h3>Customer Support</h3>
+        <p>24/7 support available</p>
+      </div>
+      <div class="social-media">
+        <button class="share-facebook">Share on Facebook</button>
+        <button class="share-twitter">Share on Twitter</button>
+      </div>
+    </div>`);
+      await page.setContent(appendedTemplate);
+      const fourthResult = await (0, _.extractArrayFromLocator)(page.locator(".products-container"), extractionOptions);
+      console.log("Fourth extraction result (after appending content):", fourthResult);
+      (0, _extendedTest.expect)(fourthResult).toEqual(thirdResult);
+      (0, _extendedTest.expect)(fourthResult).toHaveLength(3);
+      (0, _extendedTest.expect)(fourthResult[0]).toHaveProperty("title", "iPhone 15 Pro Max");
+      (0, _extendedTest.expect)(fourthResult[0]).toHaveProperty("price", "$1099");
+      const newItemTemplate = appendedTemplate.replace('<div class="additional-info">', `<div class="product-item">
+    <h2 class="product-title">iPad Pro</h2>
+    <div class="price-wrapper">
+      <span class="price">$799</span>
+    </div>
+    <div class="details">
+      <p class="product-description">Powerful tablet with M2 chip</p>
+    </div>
+  </div>
+  <div class="additional-info">`);
+      await page.setContent(newItemTemplate);
+      const fifthResult = await (0, _.extractArrayFromLocator)(page.locator(".products-container"), extractionOptions);
+      console.log("Fifth extraction result (after adding new item):", fifthResult);
+      (0, _extendedTest.expect)(fifthResult).not.toEqual(fourthResult);
+      (0, _extendedTest.expect)(fifthResult).toHaveLength(4);
+      (0, _extendedTest.expect)(fifthResult[3]).toHaveProperty("title", "iPad Pro");
+      (0, _extendedTest.expect)(fifthResult[3]).toHaveProperty("price", "$799");
+      console.log("All cache behavior tests completed successfully!");
+    });
+  });
+});