@cleaver/breakcheck-core 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -1
- package/dist/core/compare/index.js +2 -2
- package/dist/core/compare/index.js.map +1 -1
- package/dist/core/crawler/implementations/cheerio.d.ts +1 -1
- package/dist/core/crawler/implementations/cheerio.js +2 -2
- package/dist/core/crawler/implementations/cheerio.js.map +1 -1
- package/dist/core/crawler/implementations/playwright.d.ts +1 -1
- package/dist/core/crawler/implementations/playwright.js +2 -2
- package/dist/core/crawler/implementations/playwright.js.map +1 -1
- package/dist/core/crawler/index.d.ts +6 -1
- package/dist/core/crawler/index.js +8 -4
- package/dist/core/crawler/index.js.map +1 -1
- package/dist/core/crawler/url-paths.d.ts +18 -0
- package/dist/core/crawler/url-paths.js +113 -0
- package/dist/core/crawler/url-paths.js.map +1 -0
- package/dist/core/rules/RulesDsl.js +273 -56
- package/dist/core/rules/RulesDsl.js.map +1 -1
- package/dist/core/rules/RulesEngine.js +56 -17
- package/dist/core/rules/RulesEngine.js.map +1 -1
- package/dist/core/snapshot/index.d.ts +1 -1
- package/dist/core/snapshot/index.js +36 -1
- package/dist/core/snapshot/index.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/crawlee-logger.d.ts +2 -0
- package/dist/lib/crawlee-logger.js +43 -0
- package/dist/lib/crawlee-logger.js.map +1 -0
- package/dist/lib/logger.d.ts +5 -1
- package/dist/lib/logger.js +11 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/types/api.d.ts +5 -0
- package/dist/types/rules.js.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAU,mBAAmB,EAAW,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAU,mBAAmB,EAAW,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAoBhD,SAAS,WAAW,CAAC,KAAc,EAAE,OAAe;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,qBAAqB,CAAC,CAAC;IACnD,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CACd,SAAkC,EAClC,OAA0B,EAC1B,OAAe;IAEf,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,8BAA8B,KAAK,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,cAAc,CACrB,SAAkC,EAClC,GAAW,EACX,OAAe,EACf,QAAQ,GAAG,KAAK;IAEhB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QAC/D,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,aAAa,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,KAAK,GAAG,aAAa,CAClF,CAAC;IACJ,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,OAAe,EACf,OAAe,EACf,QAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,gBAAgB,QAAQ,WAAW,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACrG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,KAAc,EACd,QAAgB,EAChB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,SAAS,WAAW,WAAW,EAAE,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,OAAO,GAAG,QAAQ,SAAS,cAAc,QAAQ,YAAY,WAAW,EAAE,CAAC;IACjF,MAAM,SAAS,GACb,MAAM,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,OAAO,YAAY,CAAC,CAAC;IAE5D,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,cAAc,GAAG,SAAS,CAAC,aAAa,CAAC;YAC/C,IAAI,cAAc,KAAK,SAAS,IAAI,OAAO,cAAc,KAAK,QAAQ;gBACpE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,6CAA6C,CACxD,CAAC;YACJ,OAAO;gBACL,MAAM,EAAE,UAAU;gBAClB,GAAG,CAAC,OAAO,cAAc,KAAK,QAAQ;oBACpC,CAAC,CAAC;wBACE,YAAY,EAAE,YAAY,CACxB,cAAc,EACd,OAAO,EACP,eAAe,CAChB;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;aACvD,CAAC;QACJ,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,cAAc;gBACtB,IAAI;gBACJ,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACvB,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,4BAA4B,MAAM,CAAC,UAAU,CAAC,IAAI,CAC7D,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAiB,EAAE,OAAe;IAC1D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,iCAAiC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACxG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,QAAQ,SAAS,cAAc,QAAQ,iCAAiC,CACzE,CAAC;QACJ,OAAO;YACL,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAChD,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CACxD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC5E,IACE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAEtC,MAAM,IAAI,KAAK,CACb,wBAAwB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAChG,CAAC;QACJ,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,OAAO,WAAW;IACe;IAArC,YAAqC,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;IAAG,CAAC;IAE1D,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,uBAA0C;QAE1C,MAAM,OAAO,GACX,OAAO,uBAAuB,KAAK,QAAQ;YACzC,CAAC,CAAC,MAAM,eAAe,CAAC,uBAAuB,CAAC;YAChD,CAAC,CAAC,CAAC,uBAAuB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,IAAI,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,CAAC;IAEM,OAAO,CAAC,IAAY;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACnC,IACE,OAAO,CAAC,IAAI,KAAK,KAAK;oBACtB,OAAO,CAAC,IAAI,KAAK,QAAQ;oBACzB,OAAO,CAAC,IAAI,KAAK,OAAO;oBAExB,OAAO;gBACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,gBAAgB,CAAC,CAAqB;QAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CACzB,2CAA2C,EAC3C,EAAE,EACF,KAAK,CACN,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC7D,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CACpC,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK;oBAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,CAAC,MAAM;gBAClB,IAAI,CAAC,MAAM,CACT,MAAM,CAAC,mBAAmB,CAAC;qBACxB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;qBACzB,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC9B,CAAC;QACN,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAEO,WAAW,CACjB,CAAqB,EACrB,OAAwB,EACxB,MAAsB;QAEtB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC5B,IACE,cAAc,IAAI,MAAM;YACxB,MAAM,CAAC,YAAY;YACnB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE1C,OAAO;QACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,SAAS;gBACZ,OAAO;YACT,KAAK,SAAS;gBACZ,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO;YACT,KAAK,aAAa;gBAChB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjC,OAAO;YACT,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,OAAO,KAAK,SAAS;oBACvB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,IAAI,EACX,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAC9C,CAAC;gBACJ,OAAO;YACT,CAAC;YACD,KAAK,iBAAiB;gBACpB,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,gBAAgB,CACtB,CAAqB,EACrB,OAAwB,EACxB,KAAa,EACb,OAAe;QAEf,CAAC,CAAC,OAAO,CAAC;aACP,QAAQ,EAAE;aACV,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YACjB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;gBACvB,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;iBAC7C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAC3B,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACP,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SnapshotConfig, SnapshotResult } from "../../types/api.js";
|
|
1
|
+
import type { SnapshotConfig, SnapshotResult } from "../../types/api.js";
|
|
2
2
|
import { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
3
3
|
export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
|
|
4
4
|
export { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Dataset } from "crawlee";
|
|
2
2
|
import { logger } from "../../lib/logger.js";
|
|
3
|
+
import { resolveUrlPaths } from "../crawler/url-paths.js";
|
|
3
4
|
import { BreakcheckCrawler } from "../crawler/index.js";
|
|
4
5
|
export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
|
|
5
6
|
export { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
@@ -20,7 +21,27 @@ export async function createSnapshot(config, snapshotRepository) {
|
|
|
20
21
|
logger.error({ config }, "Snapshot creation failed: name is required");
|
|
21
22
|
throw new Error("name is required");
|
|
22
23
|
}
|
|
23
|
-
const
|
|
24
|
+
const manifest = config.urlPaths === undefined
|
|
25
|
+
? undefined
|
|
26
|
+
: resolveUrlPaths(config.baseUrl, config.urlPaths);
|
|
27
|
+
if (manifest?.issues.length) {
|
|
28
|
+
throw new Error(formatUrlPathIssues(manifest.issues));
|
|
29
|
+
}
|
|
30
|
+
const startUrls = manifest?.paths.map((urlPath) => new URL(urlPath, config.baseUrl).href);
|
|
31
|
+
if (startUrls &&
|
|
32
|
+
config.crawlSettings.maxRequests !== undefined &&
|
|
33
|
+
config.crawlSettings.maxRequests < startUrls.length) {
|
|
34
|
+
throw new Error(`maxRequests (${config.crawlSettings.maxRequests}) must be at least the number of unique manifest paths (${startUrls.length})`);
|
|
35
|
+
}
|
|
36
|
+
if (startUrls &&
|
|
37
|
+
(config.crawlSettings.includePatterns?.length ||
|
|
38
|
+
config.crawlSettings.excludePatterns?.length)) {
|
|
39
|
+
throw new Error("includePatterns and excludePatterns cannot be used with an exact URL manifest");
|
|
40
|
+
}
|
|
41
|
+
const crawler = new BreakcheckCrawler(config.crawlSettings, {
|
|
42
|
+
startUrls,
|
|
43
|
+
followLinks: config.urlPaths === undefined,
|
|
44
|
+
});
|
|
24
45
|
const { datasetName, errors: crawlErrors } = await crawler.crawl();
|
|
25
46
|
errors.push(...crawlErrors);
|
|
26
47
|
const dataset = await Dataset.open(datasetName);
|
|
@@ -64,4 +85,18 @@ export async function createSnapshot(config, snapshotRepository) {
|
|
|
64
85
|
};
|
|
65
86
|
}
|
|
66
87
|
}
|
|
88
|
+
function formatUrlPathIssues(issues) {
|
|
89
|
+
return issues
|
|
90
|
+
.map((issue) => {
|
|
91
|
+
switch (issue.type) {
|
|
92
|
+
case "base":
|
|
93
|
+
return `${issue.message}: ${issue.value}`;
|
|
94
|
+
case "entry":
|
|
95
|
+
return `Manifest path ${issue.index + 1} (${issue.value}): ${issue.message}`;
|
|
96
|
+
case "manifest":
|
|
97
|
+
return issue.message;
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
.join("; ");
|
|
101
|
+
}
|
|
67
102
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,eAAe,EAAqB,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS;YAC3B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,CACnC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CACnD,CAAC;QAEF,IACE,SAAS;YACT,MAAM,CAAC,aAAa,CAAC,WAAW,KAAK,SAAS;YAC9C,MAAM,CAAC,aAAa,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gBAAgB,MAAM,CAAC,aAAa,CAAC,WAAW,2DAA2D,SAAS,CAAC,MAAM,GAAG,CAC/H,CAAC;QACJ,CAAC;QAED,IACE,SAAS;YACT,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM;gBAC3C,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM,CAAC,EAC/C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,EAAE;YAC1D,SAAS;YACT,WAAW,EAAE,MAAM,CAAC,QAAQ,KAAK,SAAS;SAC3C,CAAC,CAAC;QAEH,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnE,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAE5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnE,OAAO;YACP,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC;SACF,CAAC,CAAC;QAEH,IAAI,WAA+B,CAAC;QACpC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACpD,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,WAAW,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAExC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS;YACT,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,UAAU,EAAE,QAAQ;aACrB;YACD,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,OAAO;YACnB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACpE,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;YACnE,MAAM;SACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAsB;IACjD,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5C,KAAK,OAAO;gBACV,OAAO,iBAAiB,KAAK,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/E,KAAK,UAAU;gBACb,OAAO,KAAK,CAAC,OAAO,CAAC;QACzB,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
export * from "./api/index.js";
|
|
2
|
-
export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
|
|
2
|
+
export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
|
|
3
3
|
export * from "./core/view/index.js";
|
|
4
|
+
export * from "./core/crawler/url-paths.js";
|
|
4
5
|
export * from "./lib/logger.js";
|
|
5
6
|
export { createLogger, logger } from "./lib/logger.js";
|
|
6
7
|
export * from "./types/api.js";
|
|
7
|
-
export type { ComparisonConfig, SnapshotConfig, SnapshotResult } from "./types/api.js";
|
|
8
|
+
export type { ComparisonConfig, SnapshotConfig, SnapshotResult, } from "./types/api.js";
|
|
8
9
|
export * from "./types/compare.js";
|
|
9
10
|
export * from "./types/crawler.js";
|
|
10
11
|
export type { CrawlError } from "./types/crawler.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export * from "./api/index.js";
|
|
2
2
|
// Explicit exports for CLI and other consumers
|
|
3
|
-
export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
|
|
3
|
+
export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
|
|
4
4
|
export * from "./core/view/index.js";
|
|
5
|
+
export * from "./core/crawler/url-paths.js";
|
|
5
6
|
export * from "./lib/logger.js";
|
|
6
7
|
export { createLogger, logger } from "./lib/logger.js";
|
|
7
8
|
export * from "./types/api.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,GACd,MAAM,gBAAgB,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACvD,cAAc,gBAAgB,CAAC;AAM/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AAEnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Logger as CrawleeLogger, log as crawleeLog, LogLevel } from "crawlee";
|
|
2
|
+
function isRecord(value) {
|
|
3
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4
|
+
}
|
|
5
|
+
class PinoCrawleeLogger extends CrawleeLogger {
|
|
6
|
+
pinoLogger;
|
|
7
|
+
constructor(pinoLogger) {
|
|
8
|
+
super({});
|
|
9
|
+
this.pinoLogger = pinoLogger;
|
|
10
|
+
}
|
|
11
|
+
_log(level, message, data, exception, opts = {}) {
|
|
12
|
+
const component = typeof opts.prefix === "string" ? opts.prefix : undefined;
|
|
13
|
+
const prefix = component ? `${component}: ` : "";
|
|
14
|
+
const suffix = typeof opts.suffix === "string" ? ` ${opts.suffix}` : "";
|
|
15
|
+
const fields = {
|
|
16
|
+
...(isRecord(data) ? data : {}),
|
|
17
|
+
...(component ? { component } : {}),
|
|
18
|
+
...(exception !== undefined ? { err: exception } : {}),
|
|
19
|
+
};
|
|
20
|
+
const logMessage = `${prefix}${message}${suffix}`;
|
|
21
|
+
switch (level) {
|
|
22
|
+
case LogLevel.ERROR:
|
|
23
|
+
this.pinoLogger.error(fields, logMessage);
|
|
24
|
+
break;
|
|
25
|
+
case LogLevel.SOFT_FAIL:
|
|
26
|
+
case LogLevel.WARNING:
|
|
27
|
+
this.pinoLogger.warn(fields, logMessage);
|
|
28
|
+
break;
|
|
29
|
+
case LogLevel.DEBUG:
|
|
30
|
+
case LogLevel.PERF:
|
|
31
|
+
this.pinoLogger.debug(fields, logMessage);
|
|
32
|
+
break;
|
|
33
|
+
case LogLevel.INFO:
|
|
34
|
+
default:
|
|
35
|
+
this.pinoLogger.info(fields, logMessage);
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export function configureCrawleeLogger(pinoLogger) {
|
|
41
|
+
crawleeLog.setOptions({ logger: new PinoCrawleeLogger(pinoLogger) });
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=crawlee-logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawlee-logger.js","sourceRoot":"","sources":["../../src/lib/crawlee-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,GAAG,IAAI,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAG/E,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,iBAAkB,SAAQ,aAAa;IACd;IAA7B,YAA6B,UAAsB;QACjD,KAAK,CAAC,EAAE,CAAC,CAAC;QADiB,eAAU,GAAV,UAAU,CAAY;IAEnD,CAAC;IAEQ,IAAI,CACX,KAAe,EACf,OAAe,EACf,IAAc,EACd,SAAmB,EACnB,OAAgC,EAAE;QAElC,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;QACF,MAAM,UAAU,GAAG,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC;QAElD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,QAAQ,CAAC,KAAK;gBACjB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,SAAS,CAAC;YACxB,KAAK,QAAQ,CAAC,OAAO;gBACnB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;YACR,KAAK,QAAQ,CAAC,KAAK,CAAC;YACpB,KAAK,QAAQ,CAAC,IAAI;gBAChB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,IAAI,CAAC;YACnB;gBACE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;QACV,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAsB;IAC3D,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;AACvE,CAAC"}
|
package/dist/lib/logger.d.ts
CHANGED
|
@@ -4,5 +4,9 @@ export interface LoggerConfig {
|
|
|
4
4
|
level?: LogLevel;
|
|
5
5
|
}
|
|
6
6
|
export declare function createLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Configure the shared Breakcheck logger and the Crawlee logger together.
|
|
9
|
+
*/
|
|
10
|
+
export declare function configureLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
|
|
11
|
+
export declare let logger: import("pino").Logger<never, boolean>;
|
|
8
12
|
export {};
|
package/dist/lib/logger.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { pino } from "pino";
|
|
2
|
+
import { configureCrawleeLogger } from "./crawlee-logger.js";
|
|
2
3
|
// Determine the log level from the environment variable, defaulting to 'info'
|
|
3
4
|
const logLevel = process.env.LOG_LEVEL || "info";
|
|
4
5
|
// Create a logger with the specified configuration
|
|
@@ -20,7 +21,16 @@ export function createLogger(config = {}) {
|
|
|
20
21
|
transport,
|
|
21
22
|
});
|
|
22
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Configure the shared Breakcheck logger and the Crawlee logger together.
|
|
26
|
+
*/
|
|
27
|
+
export function configureLogger(config = {}) {
|
|
28
|
+
logger = createLogger(config);
|
|
29
|
+
configureCrawleeLogger(logger);
|
|
30
|
+
return logger;
|
|
31
|
+
}
|
|
23
32
|
// Create the default shared logger instance (for backward compatibility)
|
|
24
33
|
// This will use pretty logging by default for CLI, but can be overridden
|
|
25
|
-
export
|
|
34
|
+
export let logger = createLogger({ useJsonLogs: false });
|
|
35
|
+
configureCrawleeLogger(logger);
|
|
26
36
|
//# sourceMappingURL=logger.js.map
|
package/dist/lib/logger.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAK7D,8EAA8E;AAC9E,MAAM,QAAQ,GAAc,OAAO,CAAC,GAAG,CAAC,SAAsB,IAAI,MAAM,CAAC;AAQzE,mDAAmD;AACnD,MAAM,UAAU,YAAY,CAAC,SAAuB,EAAE;IACpD,MAAM,EAAE,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;IAEzD,gDAAgD;IAChD,MAAM,SAAS,GAAG,WAAW;QAC3B,CAAC,CAAC,SAAS,CAAC,4BAA4B;QACxC,CAAC,CAAC;YACE,MAAM,EAAE,aAAa;YACrB,OAAO,EAAE;gBACP,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,cAAc;gBAC7B,MAAM,EAAE,cAAc;aACvB;SACF,CAAC;IAEN,OAAO,IAAI,CAAC;QACV,KAAK;QACL,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,SAAuB,EAAE;IACvD,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,CAAC,IAAI,MAAM,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;AACzD,sBAAsB,CAAC,MAAM,CAAC,CAAC"}
|
package/dist/types/api.d.ts
CHANGED
|
@@ -34,6 +34,11 @@ export interface SnapshotConfig {
|
|
|
34
34
|
name: string;
|
|
35
35
|
/** Crawler configuration to use */
|
|
36
36
|
crawlSettings: CrawlerConfig;
|
|
37
|
+
/**
|
|
38
|
+
* Optional root-relative paths to crawl exactly. When provided, link
|
|
39
|
+
* discovery is disabled and the base URL is not added automatically.
|
|
40
|
+
*/
|
|
41
|
+
urlPaths?: string[];
|
|
37
42
|
/** Optional path to generate a URL list file */
|
|
38
43
|
urlListPath?: string;
|
|
39
44
|
}
|
package/dist/types/rules.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,0BAA0B,CAAC"}
|