@cleaver/breakcheck-core 0.2.4 → 0.3.0

package/dist/core/compare/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,IAAI;QACvB,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;KACrD,CAAC;IACF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAE9D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAe,SAAS,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9E,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAwB,EACxB,kBAAsC,EACtC,oBAA0C,EAC1C,WAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,sBAAsB;IACtB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,MAAM,CAAC,gBAAgB,CACxB,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACzD,MAAM,CAAC,eAAe,CACvB,CAAC;IAEF,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,IAAI,UAAU,CAAC;IAChD,MAAM,mBAAmB,GACvB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;IAE7E,4BAA4B;IAC5B,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpE,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,2BAA2B;IAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,wCAAwC;QAE3E,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,eAAe,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,sBAAsB,IAAI,qBAAqB,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,MAAM,WAAW,CAC5B,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAClD,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,qBAAqB,EAAE,CACjD,CAAC;YACF,MAAM,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,eAAe,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,kBAAkB,EAAE,CAAC;IAEpE,0BAA0B;IAC1B,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,aAAa,EACX,UAAU,CAAC,QAAQ,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAChE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,mBAAmB;QACnB,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;QACxC,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;QAClD,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,oBAAoB;QAC9D,eAAe;QACf,OAAO;QACP,WAAW;QACX,uBAAuB,EAAE,EAAE;QAC3B,YAAY,EAAE,MAAM,CAAC,cAAc;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC;QACrD,WAAW;KACZ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,KAAK;QACxB,WAAW,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;KAChD,CAAC;IACF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAE9D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAe,SAAS,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC,GAAG,CACxE,CAAC,MAAM,EAAE,EAAE;QACT,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,eAAe,EAAE,GAAG,MAAM,CAAC;QACtD,OAAO;YACL,GAAG,eAAe;YAClB,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC;IACJ,CAAC,CACF,CAAC;IACF,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9E,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAwB,EACxB,kBAAsC,EACtC,oBAA0C,EAC1C,WAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,sBAAsB;IACtB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,MAAM,CAAC,gBAAgB,CACxB,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACzD,MAAM,CAAC,eAAe,CACvB,CAAC;IAEF,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,IAAI,UAAU,CAAC;IAChD,MAAM,aAAa,GACjB,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC/D,MAAM,mBAAmB,GACvB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;IAE7E,4BAA4B;IAC5B,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC;QACzB,CAAC,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAC1D,CAAC;IACF,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CACnC,CAAC,GAAG,EAAE,EAAE,CACN,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;QACnB,CAAC,aAAa,KAAK,SAAS,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAC1D,CAAC;IAEF,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,2BAA2B;IAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,wCAAwC;QAE3E,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,8BAA8B;QAC7E,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,eAAe,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,sBAAsB,IAAI,qBAAqB,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,MAAM,WAAW,CAC5B,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAClD,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,qBAAqB,EAAE,CACjD,CAAC;YACF,MAAM,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,eAAe,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,kBAAkB,EAAE,CAAC;IAEpE,0BAA0B;IAC1B,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,aAAa,EACX,UAAU,CAAC,QAAQ,CAAC,oBAAoB,GAAG,CAAC;YAC5C,OAAO,CAAC,MAAM,GAAG,CAAC;YAClB,WAAW,CAAC,MAAM,GAAG,CAAC;YACpB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,MAAM;QACZ,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,mBAAmB;QACnB,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;QACxC,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;QAClD,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,oBAAoB;QAC9D,eAAe;QACf,OAAO;QACP,WAAW;QACX,uBAAuB,EAAE,EAAE;QAC3B,YAAY,EAAE,MAAM,CAAC,cAAc;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC;QACrD,WAAW;KACZ,CAAC;AACJ,CAAC"}

package/dist/core/crawler/implementations/cheerio.d.ts

@@ -1,3 +1,3 @@
 import type { CrawlerConfig } from "../../../types/api.js";
 import type { CrawlerInstance } from "../../../types/crawler.js";
-export declare function createCheerioCrawler(config: CrawlerConfig): CrawlerInstance;
+export declare function createCheerioCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;

package/dist/core/crawler/implementations/cheerio.js

@@ -1,5 +1,5 @@
 import { CheerioCrawler, Dataset } from "crawlee";
-export function createCheerioCrawler(config) {
+export function createCheerioCrawler(config, followLinks = true) {
     const { maxDepth, includePatterns, excludePatterns } = config;
     let datasetPromise = Dataset.open();
     return new CheerioCrawler({
@@ -23,7 +23,7 @@ export function createCheerioCrawler(config) {
             const dataset = await datasetPromise;
             await dataset.pushData(pageSnapshot);
             const depth = request.userData.depth || 0;
-            if (depth < (maxDepth || Infinity)) {
+            if (followLinks && depth < (maxDepth || Infinity)) {
                 await enqueueLinks({
                     globs: includePatterns || [],
                     exclude: excludePatterns || [],

package/dist/core/crawler/implementations/cheerio.js.map

@@ -1 +1 @@
-{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,CAAC,MAAqB;IACxD,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,cAAc,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE;YAC/D,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YACrD,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;gBACrC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;oBACrD,GAAG;oBACH,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;iBACtD,CAAC,CACH;gBACD,KAAK;aACN,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBACnC,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,CAClC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,cAAc,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE;YAC/D,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YACrD,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;gBACrC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;oBACrD,GAAG;oBACH,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;iBACtD,CAAC,CACH;gBACD,KAAK;aACN,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/core/crawler/implementations/playwright.d.ts

@@ -1,3 +1,3 @@
 import type { CrawlerConfig } from "../../../types/api.js";
 import type { CrawlerInstance } from "../../../types/crawler.js";
-export declare function createPlaywrightCrawler(config: CrawlerConfig): CrawlerInstance;
+export declare function createPlaywrightCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;

package/dist/core/crawler/implementations/playwright.js

@@ -1,5 +1,5 @@
 import { Dataset, PlaywrightCrawler } from "crawlee";
-export function createPlaywrightCrawler(config) {
+export function createPlaywrightCrawler(config, followLinks = true) {
     const { maxDepth, includePatterns, excludePatterns } = config;
     let datasetPromise = Dataset.open();
     return new PlaywrightCrawler({
@@ -21,7 +21,7 @@ export function createPlaywrightCrawler(config) {
             const dataset = await datasetPromise;
             await dataset.pushData(pageSnapshot);
             const depth = request.userData.depth || 0;
-            if (depth < (maxDepth || Infinity)) {
+            if (followLinks && depth < (maxDepth || Infinity)) {
                 await enqueueLinks({
                     globs: includePatterns || [],
                     exclude: excludePatterns || [],

package/dist/core/crawler/implementations/playwright.js.map

@@ -1 +1 @@
-{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB;IAErB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,CAAC,GAAG;gBACxC,OAAO;gBACP,KAAK;gBACL,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;gBACnC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;aACnC,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBACnC,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,CAAC,GAAG;gBACxC,OAAO;gBACP,KAAK;gBACL,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;gBACnC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;aACnC,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/core/crawler/index.d.ts

@@ -1,10 +1,21 @@
 import type { CrawlerConfig } from "../../types/api.js";
 import type { CrawlResult } from "../../types/crawler.js";
+interface CrawlerRuntime {
+    run(startUrls: string[]): Promise<unknown>;
+    teardown(): Promise<void>;
+}
 export declare class BreakcheckCrawler {
     private config;
     private crawler;
     private errors;
-    constructor(config: CrawlerConfig);
+    private readonly startUrls;
+    private readonly followLinks;
+    constructor(config: CrawlerConfig, options?: {
+        startUrls?: string[];
+        followLinks?: boolean;
+        crawler?: CrawlerRuntime;
+    });
     private createCrawler;
-    crawl(): Promise<CrawlResult>;
+    crawl(signal?: AbortSignal): Promise<CrawlResult>;
 }
+export {};

package/dist/core/crawler/index.js

@@ -6,26 +6,40 @@ export class BreakcheckCrawler {
     config;
     crawler;
     errors = [];
-    constructor(config) {
+    startUrls;
+    followLinks;
+    constructor(config, options = {}) {
         this.config = config;
-        this.crawler = this.createCrawler();
+        this.startUrls = options.startUrls ?? [config.baseUrl];
+        this.followLinks = options.followLinks ?? true;
+        this.crawler = options.crawler ?? this.createCrawler();
     }
     createCrawler() {
         switch (this.config.crawlerType) {
             case "cheerio":
-                return createCheerioCrawler(this.config);
+                return createCheerioCrawler(this.config, this.followLinks);
             case "playwright":
-                return createPlaywrightCrawler(this.config);
+                return createPlaywrightCrawler(this.config, this.followLinks);
             default:
                 const _exhaustiveCheck = this.config.crawlerType;
                 throw new Error(`Unsupported crawler type: ${this.config.crawlerType}`);
         }
     }
-    async crawl() {
+    async crawl(signal) {
+        signal?.throwIfAborted();
+        let teardownPromise;
+        const teardown = () => {
+            teardownPromise ??= this.crawler.teardown();
+            void teardownPromise.catch(() => undefined);
+        };
+        const onAbort = () => teardown();
+        signal?.addEventListener("abort", onAbort, { once: true });
         try {
             await purgeDefaultStorages();
+            signal?.throwIfAborted();
             this.errors = [];
-            await this.crawler.run([this.config.baseUrl]);
+            await this.crawler.run(this.startUrls);
+            signal?.throwIfAborted();
             // Open the dataset and scan for error items only
             const dataset = await Dataset.open();
             await dataset.forEach((item) => {
@@ -56,6 +70,11 @@ export class BreakcheckCrawler {
             }, "Crawling failed");
             throw new Error(`Crawling failed: ${error instanceof Error ? error.message : String(error)}`);
         }
+        finally {
+            signal?.removeEventListener("abort", onAbort);
+            if (teardownPromise !== undefined)
+                await teardownPromise;
+        }
     }
 }
 //# sourceMappingURL=index.js.map

package/dist/core/crawler/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAkB;IACzB,MAAM,GAAiB,EAAE,CAAC;IAElC,YAAY,MAAqB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC;IAEO,aAAa;QACnB,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChC,KAAK,SAAS;gBACZ,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,KAAK,YAAY;gBACf,OAAO,uBAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9C;gBACE,MAAM,gBAAgB,GAAU,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,oBAAoB,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9C,iDAAiD;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;gBAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;wBACf,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,OAAO,EAAE,IAAI,CAAC,KAAK;wBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC5C,IAAI,iBAAiB,GACnB,WAAW,EAAE,IAAI;gBACjB,aAAa,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,iBAAiB,GAAG,EAAE,CAAC;YAC/C,OAAO;gBACL,WAAW,EAAE,iBAAiB;gBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV;gBACE,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,EACD,iBAAiB,CAClB,CAAC;YACF,MAAM,IAAI,KAAK,CACb,oBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAO1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAiB;IACxB,MAAM,GAAiB,EAAE,CAAC;IACjB,SAAS,CAAW;IACpB,WAAW,CAAU;IAEtC,YACE,MAAqB,EACrB,OAAO,GAIH,EAAE;QAEN,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;IACzD,CAAC;IAEO,aAAa;QACnB,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChC,KAAK,SAAS;gBACZ,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7D,KAAK,YAAY;gBACf,OAAO,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChE;gBACE,MAAM,gBAAgB,GAAU,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAoB;QAC9B,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,IAAI,eAA0C,CAAC;QAC/C,MAAM,QAAQ,GAAG,GAAG,EAAE;YACpB,eAAe,KAAK,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC5C,KAAK,eAAe,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9C,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC;QACjC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAE3D,IAAI,CAAC;YACH,MAAM,oBAAoB,EAAE,CAAC;YAC7B,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,iDAAiD;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;gBAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;wBACf,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,OAAO,EAAE,IAAI,CAAC,KAAK;wBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC5C,IAAI,iBAAiB,GACnB,WAAW,EAAE,IAAI;gBACjB,aAAa,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,iBAAiB,GAAG,EAAE,CAAC;YAC/C,OAAO;gBACL,WAAW,EAAE,iBAAiB;gBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV;gBACE,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,EACD,iBAAiB,CAClB,CAAC;YACF,MAAM,IAAI,KAAK,CACb,oBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,IAAI,eAAe,KAAK,SAAS;gBAAE,MAAM,eAAe,CAAC;QAC3D,CAAC;IACH,CAAC;CACF"}

package/dist/core/crawler/url-paths.d.ts

@@ -0,0 +1,18 @@
+export type UrlPathIssue = {
+    type: "manifest";
+    message: string;
+} | {
+    type: "base";
+    value: string;
+    message: string;
+} | {
+    type: "entry";
+    index: number;
+    value: string;
+    message: string;
+};
+export interface UrlPathResolution {
+    paths: string[];
+    issues: UrlPathIssue[];
+}
+export declare function resolveUrlPaths(baseUrl: string, paths: readonly string[]): UrlPathResolution;

package/dist/core/crawler/url-paths.js

@@ -0,0 +1,113 @@
+const ROOT_RELATIVE_PATH_MESSAGE = "Path must be root-relative and begin with exactly one '/'";
+function isHttpUrl(url) {
+    return url.protocol === "http:" || url.protocol === "https:";
+}
+function toPath(url) {
+    return `${url.pathname}${url.search}${url.hash}`;
+}
+export function resolveUrlPaths(baseUrl, paths) {
+    const issues = [];
+    const resolvedPaths = [];
+    const seenUrls = new Set();
+    let base;
+    try {
+        base = new URL(baseUrl);
+    }
+    catch {
+        return {
+            paths: [],
+            issues: [
+                {
+                    type: "base",
+                    value: baseUrl,
+                    message: "Base URL must be a valid HTTP(S) URL",
+                },
+            ],
+        };
+    }
+    if (!isHttpUrl(base)) {
+        return {
+            paths: [],
+            issues: [
+                {
+                    type: "base",
+                    value: baseUrl,
+                    message: "Base URL must be a valid HTTP(S) URL",
+                },
+            ],
+        };
+    }
+    if (paths.length === 0) {
+        issues.push({
+            type: "manifest",
+            message: "URL manifest must contain at least one path",
+        });
+    }
+    paths.forEach((path, index) => {
+        const value = path.trim();
+        if (value === "" || !value.startsWith("/") || value.startsWith("//")) {
+            issues.push({
+                type: "entry",
+                index,
+                value: path,
+                message: value.startsWith("//")
+                    ? "Cross-origin or scheme-relative paths are not allowed"
+                    : ROOT_RELATIVE_PATH_MESSAGE,
+            });
+            return;
+        }
+        if (/\s/.test(value)) {
+            issues.push({
+                type: "entry",
+                index,
+                value: path,
+                message: "Path must not contain whitespace",
+            });
+            return;
+        }
+        if (value.includes("\\")) {
+            issues.push({
+                type: "entry",
+                index,
+                value: path,
+                message: ROOT_RELATIVE_PATH_MESSAGE,
+            });
+            return;
+        }
+        if (/%(?![0-9A-Fa-f]{2})/.test(value)) {
+            issues.push({
+                type: "entry",
+                index,
+                value: path,
+                message: "Path contains an invalid percent escape",
+            });
+            return;
+        }
+        try {
+            const url = new URL(value, base);
+            if (!isHttpUrl(url) || url.origin !== base.origin) {
+                issues.push({
+                    type: "entry",
+                    index,
+                    value: path,
+                    message: "Cross-origin paths are not allowed",
+                });
+                return;
+            }
+            if (!seenUrls.has(url.href)) {
+                seenUrls.add(url.href);
+                resolvedPaths.push(toPath(url));
+            }
+        }
+        catch {
+            issues.push({
+                type: "entry",
+                index,
+                value: path,
+                message: "Path is not a valid URL path",
+            });
+        }
+    });
+    return { paths: resolvedPaths, issues };
+}
+//# sourceMappingURL=url-paths.js.map

package/dist/core/crawler/url-paths.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-paths.js","sourceRoot":"","sources":["../../../src/core/crawler/url-paths.ts"],"names":[],"mappings":"AAsBA,MAAM,0BAA0B,GAC9B,2DAA2D,CAAC;AAE9D,SAAS,SAAS,CAAC,GAAQ;IACzB,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;AAC/D,CAAC;AAED,SAAS,MAAM,CAAC,GAAQ;IACtB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,KAAwB;IAExB,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,6CAA6C;SACvD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE1B,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;oBAC7B,CAAC,CAAC,uDAAuD;oBACzD,CAAC,CAAC,0BAA0B;aAC/B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,kCAAkC;aAC5C,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,0BAA0B;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,KAAK;oBACL,KAAK,EAAE,IAAI;oBACX,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,8BAA8B;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC"}