@cleaver/breakcheck-core 0.2.3 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +42 -1
  2. package/dist/core/compare/index.js +2 -2
  3. package/dist/core/compare/index.js.map +1 -1
  4. package/dist/core/crawler/implementations/cheerio.d.ts +1 -1
  5. package/dist/core/crawler/implementations/cheerio.js +2 -2
  6. package/dist/core/crawler/implementations/cheerio.js.map +1 -1
  7. package/dist/core/crawler/implementations/playwright.d.ts +1 -1
  8. package/dist/core/crawler/implementations/playwright.js +2 -2
  9. package/dist/core/crawler/implementations/playwright.js.map +1 -1
  10. package/dist/core/crawler/index.d.ts +6 -1
  11. package/dist/core/crawler/index.js +8 -4
  12. package/dist/core/crawler/index.js.map +1 -1
  13. package/dist/core/crawler/url-paths.d.ts +18 -0
  14. package/dist/core/crawler/url-paths.js +113 -0
  15. package/dist/core/crawler/url-paths.js.map +1 -0
  16. package/dist/core/rules/RulesDsl.js +253 -139
  17. package/dist/core/rules/RulesDsl.js.map +1 -1
  18. package/dist/core/rules/RulesEngine.d.ts +2 -2
  19. package/dist/core/rules/RulesEngine.js +168 -82
  20. package/dist/core/rules/RulesEngine.js.map +1 -1
  21. package/dist/core/snapshot/index.d.ts +1 -1
  22. package/dist/core/snapshot/index.js +36 -1
  23. package/dist/core/snapshot/index.js.map +1 -1
  24. package/dist/index.d.ts +3 -2
  25. package/dist/index.js +2 -1
  26. package/dist/index.js.map +1 -1
  27. package/dist/lib/crawlee-logger.d.ts +2 -0
  28. package/dist/lib/crawlee-logger.js +43 -0
  29. package/dist/lib/crawlee-logger.js.map +1 -0
  30. package/dist/lib/logger.d.ts +5 -1
  31. package/dist/lib/logger.js +11 -1
  32. package/dist/lib/logger.js.map +1 -1
  33. package/dist/types/api.d.ts +5 -0
  34. package/dist/types/rules.d.ts +18 -10
  35. package/dist/types/rules.js.map +1 -1
  36. package/package.json +1 -1
@@ -1,116 +1,202 @@
1
1
  import * as cheerio from "cheerio";
2
- import { REGION_NAME_PATTERN, } from "../../types/rules.js";
2
+ import { REGION_NAME_PATTERN } from "../../types/rules.js";
3
3
  import { processRulesDsl } from "./RulesDsl.js";
4
+ function objectValue(value, context) {
5
+ if (typeof value !== "object" || value === null || Array.isArray(value))
6
+ throw new Error(`${context} must be an object.`);
7
+ return value;
8
+ }
9
+ function keysAre(modifiers, allowed, context) {
10
+ const extra = Object.keys(modifiers).find((key) => !allowed.includes(key));
11
+ if (extra)
12
+ throw new Error(`${context} has unsupported modifier "${extra}".`);
13
+ }
14
+ function requiredString(modifiers, key, context, nonEmpty = false) {
15
+ const value = modifiers[key];
16
+ if (typeof value !== "string" || (nonEmpty && value.length === 0))
17
+ throw new Error(`${context} requires ${nonEmpty ? "a non-empty" : "a string"} "${key}" modifier.`);
18
+ return value;
19
+ }
20
+ function compileRegex(pattern, context, modifier) {
21
+ try {
22
+ return new RegExp(pattern);
23
+ }
24
+ catch (error) {
25
+ throw new Error(`${context} has invalid ${modifier} regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
26
+ }
27
+ }
28
+ function compileAction(value, selector, ruleIndex, actionIndex) {
29
+ const action = objectValue(value, `Rule ${ruleIndex} action ${actionIndex}`);
30
+ const actionName = action.action;
31
+ const context = `Rule ${ruleIndex} selector "${selector}" action ${actionIndex}`;
32
+ const modifiers = action.modifiers === undefined
33
+ ? {}
34
+ : objectValue(action.modifiers, `${context} modifiers`);
35
+ switch (actionName) {
36
+ case "include":
37
+ case "exclude": {
38
+ keysAre(modifiers, ["content_regex"], context);
39
+ const contentPattern = modifiers.content_regex;
40
+ if (contentPattern !== undefined && typeof contentPattern !== "string")
41
+ throw new Error(`${context} modifier "content_regex" must be a string.`);
42
+ return {
43
+ action: actionName,
44
+ ...(typeof contentPattern === "string"
45
+ ? {
46
+ contentRegex: compileRegex(contentPattern, context, "content_regex"),
47
+ }
48
+ : {}),
49
+ };
50
+ }
51
+ case "remove_attr":
52
+ keysAre(modifiers, ["attr"], context);
53
+ return {
54
+ action: "remove_attr",
55
+ attr: requiredString(modifiers, "attr", context, true),
56
+ };
57
+ case "rewrite_attr": {
58
+ keysAre(modifiers, ["attr", "regex", "replace"], context);
59
+ const attr = requiredString(modifiers, "attr", context, true);
60
+ const pattern = requiredString(modifiers, "regex", context);
61
+ return {
62
+ action: "rewrite_attr",
63
+ attr,
64
+ regex: compileRegex(pattern, context, "regex"),
65
+ replace: requiredString(modifiers, "replace", context),
66
+ };
67
+ }
68
+ case "rewrite_content": {
69
+ keysAre(modifiers, ["regex", "replace"], context);
70
+ const pattern = requiredString(modifiers, "regex", context);
71
+ return {
72
+ action: "rewrite_content",
73
+ regex: compileRegex(pattern, context, "regex"),
74
+ replace: requiredString(modifiers, "replace", context),
75
+ };
76
+ }
77
+ default:
78
+ throw new Error(`${context} has unsupported action "${String(actionName)}".`);
79
+ }
80
+ }
81
+ function validateSelector(selector, context) {
82
+ if (typeof selector !== "string" || selector.trim().length === 0)
83
+ throw new Error(`${context} requires a non-empty selector.`);
84
+ const cloned = selector.slice();
85
+ try {
86
+ cheerio.load("", {}, false)(cloned);
87
+ }
88
+ catch (error) {
89
+ throw new Error(`${context} has invalid selector "${cloned}": ${error instanceof Error ? error.message : String(error)}`, { cause: error });
90
+ }
91
+ return cloned;
92
+ }
93
+ function compileRuleset(value) {
94
+ const input = objectValue(value, "Ruleset");
95
+ if (!Array.isArray(input.rules))
96
+ throw new Error("Ruleset rules must be an array.");
97
+ const rules = input.rules.map((candidate, ruleIndex) => {
98
+ const rule = objectValue(candidate, `Rule ${ruleIndex}`);
99
+ const selector = validateSelector(rule.selector, `Rule ${ruleIndex}`);
100
+ if (!Array.isArray(rule.actions) || rule.actions.length === 0)
101
+ throw new Error(`Rule ${ruleIndex} selector "${selector}" requires at least one action.`);
102
+ return {
103
+ selector,
104
+ actions: rule.actions.map((action, actionIndex) => compileAction(action, selector, ruleIndex, actionIndex)),
105
+ };
106
+ });
107
+ const regionsValue = input.regions ?? [];
108
+ if (!Array.isArray(regionsValue))
109
+ throw new Error("Ruleset regions must be an array.");
110
+ const names = new Set();
111
+ const regions = regionsValue.map((candidate, regionIndex) => {
112
+ const region = objectValue(candidate, `Region ${regionIndex}`);
113
+ const selector = validateSelector(region.selector, `Region ${regionIndex}`);
114
+ if (typeof region.name !== "string" ||
115
+ !REGION_NAME_PATTERN.test(region.name))
116
+ throw new Error(`Invalid region name "${String(region.name)}". Region names must match [A-Za-z_][A-Za-z0-9_]*.`);
117
+ if (names.has(region.name))
118
+ throw new Error(`Duplicate region name: ${region.name}`);
119
+ names.add(region.name);
120
+ return { selector, name: region.name.slice() };
121
+ });
122
+ return { rules, regions };
123
+ }
4
124
  export class RulesEngine {
5
125
  ruleset;
6
126
  constructor(ruleset) {
7
- const regions = ruleset.regions ?? [];
8
- this.validateRegions(regions);
9
- this.ruleset = { ...ruleset, regions };
127
+ this.ruleset = ruleset;
10
128
  }
11
129
  static async create(rulesDirectoryOrRuleset) {
12
130
  const ruleset = typeof rulesDirectoryOrRuleset === "string"
13
131
  ? await processRulesDsl(rulesDirectoryOrRuleset)
14
- : rulesDirectoryOrRuleset ?? { name: "none", rules: [] };
15
- return new RulesEngine(ruleset);
132
+ : (rulesDirectoryOrRuleset ?? { name: "none", rules: [] });
133
+ return new RulesEngine(compileRuleset(ruleset));
16
134
  }
17
135
  process(html) {
18
136
  const $ = cheerio.load(html);
19
- // Apply ordinary rules in declaration order before selecting regions.
20
137
  for (const rule of this.ruleset.rules) {
21
- const elements = $(rule.selector);
22
- elements.each((_, element) => {
23
- if (element.type === "tag") {
24
- for (const action of rule.actions) {
25
- this.applyAction($, element, action);
26
- }
27
- }
138
+ $(rule.selector).each((_, element) => {
139
+ if (element.type !== "tag" &&
140
+ element.type !== "script" &&
141
+ element.type !== "style")
142
+ return;
143
+ for (const action of rule.actions)
144
+ this.applyAction($, element, action);
28
145
  });
29
146
  }
30
- const regions = this.ruleset.regions ?? [];
31
- if (regions.length > 0) {
32
- return this.serializeRegions($, regions);
33
- }
34
- return $.html();
147
+ return this.ruleset.regions.length ? this.serializeRegions($) : $.html();
35
148
  }
36
- validateRegions(regions) {
37
- const names = new Set();
38
- for (const region of regions) {
39
- if (!REGION_NAME_PATTERN.test(region.name)) {
40
- throw new Error(`Invalid region name "${region.name}". Region names must match [A-Za-z_][A-Za-z0-9_]*.`);
41
- }
42
- if (names.has(region.name)) {
43
- throw new Error(`Duplicate region name: ${region.name}`);
44
- }
45
- names.add(region.name);
46
- }
47
- }
48
- serializeRegions($, regions) {
149
+ serializeRegions($) {
49
150
  const output = cheerio.load("<breakcheck-regions></breakcheck-regions>", {}, false);
50
151
  const root = output("breakcheck-regions");
51
- const sortedRegions = [...regions].sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0);
52
- for (const region of sortedRegions) {
152
+ const regions = [...this.ruleset.regions].sort((left, right) => left.name.localeCompare(right.name));
153
+ for (const region of regions) {
53
154
  const fragments = [];
54
155
  $(region.selector).each((_, element) => {
55
- if (element.type === "tag") {
156
+ if (element.type === "tag")
56
157
  fragments.push($.html(element));
57
- }
58
158
  });
59
- if (fragments.length === 0) {
60
- continue;
61
- }
62
- const wrapper = output("<region></region>").attr("name", region.name);
63
- wrapper.append(fragments.join(""));
64
- root.append(wrapper);
159
+ if (fragments.length)
160
+ root.append(output("<region></region>")
161
+ .attr("name", region.name)
162
+ .append(fragments.join("")));
65
163
  }
66
164
  return output.html();
67
165
  }
68
166
  applyAction($, element, action) {
69
- const $element = $(element);
70
- // Check content regex if specified
71
- if (action.modifiers?.content_regex) {
72
- const content = $element.text();
73
- const regex = new RegExp(action.modifiers.content_regex);
74
- if (!regex.test(content)) {
75
- return;
76
- }
77
- }
167
+ const selected = $(element);
168
+ if ("contentRegex" in action &&
169
+ action.contentRegex &&
170
+ !action.contentRegex.test(selected.text()))
171
+ return;
78
172
  switch (action.action) {
79
- case "exclude":
80
- $element.remove();
81
- break;
82
173
  case "include":
83
- // No action needed for include
84
- break;
174
+ return;
175
+ case "exclude":
176
+ selected.remove();
177
+ return;
85
178
  case "remove_attr":
86
- if (action.modifiers && "attr" in action.modifiers) {
87
- $element.removeAttr(action.modifiers.attr);
88
- }
89
- break;
90
- case "rewrite_attr":
91
- if (action.modifiers &&
92
- "attr" in action.modifiers &&
93
- "regex" in action.modifiers &&
94
- "replace" in action.modifiers) {
95
- const currentValue = $element.attr(action.modifiers.attr);
96
- if (currentValue) {
97
- const regex = new RegExp(action.modifiers.regex);
98
- const newValue = currentValue.replace(regex, action.modifiers.replace);
99
- $element.attr(action.modifiers.attr, newValue);
100
- }
101
- }
102
- break;
179
+ selected.removeAttr(action.attr);
180
+ return;
181
+ case "rewrite_attr": {
182
+ const current = selected.attr(action.attr);
183
+ if (current !== undefined)
184
+ selected.attr(action.attr, current.replace(action.regex, action.replace));
185
+ return;
186
+ }
103
187
  case "rewrite_content":
104
- if (action.modifiers &&
105
- "regex" in action.modifiers &&
106
- "replace" in action.modifiers) {
107
- const content = $element.text();
108
- const regex = new RegExp(action.modifiers.regex);
109
- const newContent = content.replace(regex, action.modifiers.replace);
110
- $element.text(newContent);
111
- }
112
- break;
188
+ this.rewriteTextNodes($, element, action.regex, action.replace);
113
189
  }
114
190
  }
191
+ rewriteTextNodes($, element, regex, replace) {
192
+ $(element)
193
+ .contents()
194
+ .each((_, child) => {
195
+ if (child.type === "text")
196
+ child.data = child.data.replace(regex, replace);
197
+ else if (child.type === "tag")
198
+ this.rewriteTextNodes($, child, regex, replace);
199
+ });
200
+ }
115
201
  }
116
202
  //# sourceMappingURL=RulesEngine.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAEL,mBAAmB,GAGpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,OAAO,WAAW;IACd,OAAO,CAAU;IAEzB,YAAoB,OAAgB;QAClC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,CAAC;IACzC,CAAC;IAEM,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,uBAA0C;QAE1C,MAAM,OAAO,GACX,OAAO,uBAAuB,KAAK,QAAQ;YACzC,CAAC,CAAC,MAAM,eAAe,CAAC,uBAAuB,CAAC;YAChD,CAAC,CAAC,uBAAuB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QAE7D,OAAO,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAEM,OAAO,CAAC,IAAY;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,sEAAsE;QACtE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBAC3B,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBAC3B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBAClC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;oBACvC,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QAC3C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAED,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IAClB,CAAC;IAEO,eAAe,CAAC,OAAiB;QACvC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CACb,wBAAwB,MAAM,CAAC,IAAI,oDAAoD,CACxF,CAAC;YACJ,CAAC;YACD,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3D,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,CAAC;IACH,CAAC;IAEO,gBAAgB,CACtB,CAAqB,EACrB,OAAiB;QAEjB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CACzB,2CAA2C,EAC3C,EAAE,EACF,KAAK,CACN,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,aAAa,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACtD,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7D,CAAC;QAEF,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACnC,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;oBAC3B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;gBAClC,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YACtE,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACnC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAEO,WAAW,CACjB,CAAqB,EACrB,OAAwB,EACxB,MAAc;QAEd,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAE5B,mCAAmC;QACnC,IAAI,MAAM,CAAC,SAAS,EAAE,aAAa,EAAE,CAAC;YACpC,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;YACzD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzB,OAAO;YACT,CAAC;QACH,CAAC;QAED,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,SAAS;gBACZ,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,MAAM;YAER,KAAK,SAAS;gBACZ,+BAA+B;gBAC/B,MAAM;YAER,KAAK,aAAa;gBAChB,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;oBACnD,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC7C,CAAC;gBACD,MAAM;YAER,KAAK,cAAc;gBACjB,IACE,MAAM,CAAC,SAAS;oBAChB,MAAM,IAAI,MAAM,CAAC,SAAS;oBAC1B,OAAO,IAAI,MAAM,CAAC,SAAS;oBAC3B,SAAS,IAAI,MAAM,CAAC,SAAS,EAC7B,CAAC;oBACD,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;oBAC1D,IAAI,YAAY,EAAE,CAAC;wBACjB,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;wBACjD,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,CACnC,KAAK,EACL,MAAM,CAAC,SAAS,CAAC,OAAO,CACzB,CAAC;wBACF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;oBACjD,CAAC;gBACH,CAAC;gBACD,MAAM;YAER,KAAK,iBAAiB;gBACpB,IACE,MAAM,CAAC,SAAS;oBAChB,OAAO,IAAI,MAAM,CAAC,SAAS;oBAC3B,SAAS,IAAI,MAAM,CAAC,SAAS,EAC7B,CAAC;oBACD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;oBACjD,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;oBACpE,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM;QACV,CAAC;IACH,CAAC;CACF"}
1
+ {"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAU,mBAAmB,EAAW,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAoBhD,SAAS,WAAW,CAAC,KAAc,EAAE,OAAe;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,qBAAqB,CAAC,CAAC;IACnD,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CACd,SAAkC,EAClC,OAA0B,EAC1B,OAAe;IAEf,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,8BAA8B,KAAK,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,cAAc,CACrB,SAAkC,EAClC,GAAW,EACX,OAAe,EACf,QAAQ,GAAG,KAAK;IAEhB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QAC/D,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,aAAa,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,KAAK,GAAG,aAAa,CAClF,CAAC;IACJ,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,OAAe,EACf,OAAe,EACf,QAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,gBAAgB,QAAQ,WAAW,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACrG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,KAAc,EACd,QAAgB,EAChB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,SAAS,WAAW,WAAW,EAAE,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,OAAO,GAAG,QAAQ,SAAS,cAAc,QAAQ,YAAY,WAAW,EAAE,CAAC;IACjF,MAAM,SAAS,GACb,MAAM,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,OAAO,YAAY,CAAC,CAAC;IAE5D,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,cAAc,GAAG,SAAS,CAAC,aAAa,CAAC;YAC/C,IAAI,cAAc,KAAK,SAAS,IAAI,OAAO,cAAc,KAAK,QAAQ;gBACpE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,6CAA6C,CACxD,CAAC;YACJ,OAAO;gBACL,MAAM,EAAE,UAAU;gBAClB,GAAG,CAAC,OAAO,cAAc,KAAK,QAAQ;oBACpC,CAAC,CAAC;wBACE,YAAY,EAAE,YAAY,CACxB,cAAc,EACd,OAAO,EACP,eAAe,CAChB;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;aACvD,CAAC;QACJ,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,cAAc;gBACtB,IAAI;gBACJ,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACvB,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,4BAA4B,MAAM,CAAC,UAAU,CAAC,IAAI,CAC7D,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAiB,EAAE,OAAe;IAC1D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,iCAAiC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACxG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,QAAQ,SAAS,cAAc,QAAQ,iCAAiC,CACzE,CAAC;QACJ,OAAO;YACL,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAChD,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CACxD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC5E,IACE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAEtC,MAAM,IAAI,KAAK,CACb,wBAAwB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAChG,CAAC;QACJ,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,OAAO,WAAW;IACe;IAArC,YAAqC,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;IAAG,CAAC;IAE1D,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,uBAA0C;QAE1C,MAAM,OAAO,GACX,OAAO,uBAAuB,KAAK,QAAQ;YACzC,CAAC,CAAC,MAAM,eAAe,CAAC,uBAAuB,CAAC;YAChD,CAAC,CAAC,CAAC,uBAAuB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,IAAI,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,CAAC;IAEM,OAAO,CAAC,IAAY;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACnC,IACE,OAAO,CAAC,IAAI,KAAK,KAAK;oBACtB,OAAO,CAAC,IAAI,KAAK,QAAQ;oBACzB,OAAO,CAAC,IAAI,KAAK,OAAO;oBAExB,OAAO;gBACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,gBAAgB,CAAC,CAAqB;QAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CACzB,2CAA2C,EAC3C,EAAE,EACF,KAAK,CACN,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC7D,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CACpC,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK;oBAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,CAAC,MAAM;gBAClB,IAAI,CAAC,MAAM,CACT,MAAM,CAAC,mBAAmB,CAAC;qBACxB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;qBACzB,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC9B,CAAC;QACN,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAEO,WAAW,CACjB,CAAqB,EACrB,OAAwB,EACxB,MAAsB;QAEtB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC5B,IACE,cAAc,IAAI,MAAM;YACxB,MAAM,CAAC,YAAY;YACnB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE1C,OAAO;QACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,SAAS;gBACZ,OAAO;YACT,KAAK,SAAS;gBACZ,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO;YACT,KAAK,aAAa;gBAChB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjC,OAAO;YACT,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,OAAO,KAAK,SAAS;oBACvB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,IAAI,EACX,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAC9C,CAAC;gBACJ,OAAO;YACT,CAAC;YACD,KAAK,iBAAiB;gBACpB,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,gBAAgB,CACtB,CAAqB,EACrB,OAAwB,EACxB,KAAa,EACb,OAAe;QAEf,CAAC,CAAC,OAAO,CAAC;aACP,QAAQ,EAAE;aACV,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YACjB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;gBACvB,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;iBAC7C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAC3B,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACP,CAAC;CACF"}
@@ -1,4 +1,4 @@
1
- import { SnapshotConfig, SnapshotResult } from "../../types/api.js";
1
+ import type { SnapshotConfig, SnapshotResult } from "../../types/api.js";
2
2
  import { SnapshotRepository } from "./classes/SnapshotRepository.js";
3
3
  export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
4
4
  export { SnapshotRepository } from "./classes/SnapshotRepository.js";
@@ -1,5 +1,6 @@
1
1
  import { Dataset } from "crawlee";
2
2
  import { logger } from "../../lib/logger.js";
3
+ import { resolveUrlPaths } from "../crawler/url-paths.js";
3
4
  import { BreakcheckCrawler } from "../crawler/index.js";
4
5
  export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
5
6
  export { SnapshotRepository } from "./classes/SnapshotRepository.js";
@@ -20,7 +21,27 @@ export async function createSnapshot(config, snapshotRepository) {
20
21
  logger.error({ config }, "Snapshot creation failed: name is required");
21
22
  throw new Error("name is required");
22
23
  }
23
- const crawler = new BreakcheckCrawler(config.crawlSettings);
24
+ const manifest = config.urlPaths === undefined
25
+ ? undefined
26
+ : resolveUrlPaths(config.baseUrl, config.urlPaths);
27
+ if (manifest?.issues.length) {
28
+ throw new Error(formatUrlPathIssues(manifest.issues));
29
+ }
30
+ const startUrls = manifest?.paths.map((urlPath) => new URL(urlPath, config.baseUrl).href);
31
+ if (startUrls &&
32
+ config.crawlSettings.maxRequests !== undefined &&
33
+ config.crawlSettings.maxRequests < startUrls.length) {
34
+ throw new Error(`maxRequests (${config.crawlSettings.maxRequests}) must be at least the number of unique manifest paths (${startUrls.length})`);
35
+ }
36
+ if (startUrls &&
37
+ (config.crawlSettings.includePatterns?.length ||
38
+ config.crawlSettings.excludePatterns?.length)) {
39
+ throw new Error("includePatterns and excludePatterns cannot be used with an exact URL manifest");
40
+ }
41
+ const crawler = new BreakcheckCrawler(config.crawlSettings, {
42
+ startUrls,
43
+ followLinks: config.urlPaths === undefined,
44
+ });
24
45
  const { datasetName, errors: crawlErrors } = await crawler.crawl();
25
46
  errors.push(...crawlErrors);
26
47
  const dataset = await Dataset.open(datasetName);
@@ -64,4 +85,18 @@ export async function createSnapshot(config, snapshotRepository) {
64
85
  };
65
86
  }
66
87
  }
88
+ function formatUrlPathIssues(issues) {
89
+ return issues
90
+ .map((issue) => {
91
+ switch (issue.type) {
92
+ case "base":
93
+ return `${issue.message}: ${issue.value}`;
94
+ case "entry":
95
+ return `Manifest path ${issue.index + 1} (${issue.value}): ${issue.message}`;
96
+ case "manifest":
97
+ return issue.message;
98
+ }
99
+ })
100
+ .join("; ");
101
+ }
67
102
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAE5D,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnE,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAE5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnE,OAAO;YACP,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC;SACF,CAAC,CAAC;QAEH,IAAI,WAA+B,CAAC;QACpC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACpD,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,WAAW,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAExC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS;YACT,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,UAAU,EAAE,QAAQ;aACrB;YACD,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,OAAO;YACnB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACpE,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;YACnE,MAAM;SACP,CAAC;IACJ,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,eAAe,EAAqB,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS;YAC3B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,CACnC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CACnD,CAAC;QAEF,IACE,SAAS;YACT,MAAM,CAAC,aAAa,CAAC,WAAW,KAAK,SAAS;YAC9C,MAAM,CAAC,aAAa,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gBAAgB,MAAM,CAAC,aAAa,CAAC,WAAW,2DAA2D,SAAS,CAAC,MAAM,GAAG,CAC/H,CAAC;QACJ,CAAC;QAED,IACE,SAAS;YACT,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM;gBAC3C,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM,CAAC,EAC/C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,EAAE;YAC1D,SAAS;YACT,WAAW,EAAE,MAAM,CAAC,QAAQ,KAAK,SAAS;SAC3C,CAAC,CAAC;QAEH,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnE,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAE5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnE,OAAO;YACP,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC;SACF,CAAC,CAAC;QAEH,IAAI,WAA+B,CAAC;QACpC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACpD,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,WAAW,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAExC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS;YACT,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,UAAU,EAAE,QAAQ;aACrB;YACD,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,OAAO;YACnB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACpE,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;YACnE,MAAM;SACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAsB;IACjD,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5C,KAAK,OAAO;gBACV,OAAO,iBAAiB,KAAK,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/E,KAAK,UAAU;gBACb,OAAO,KAAK,CAAC,OAAO,CAAC;QACzB,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
package/dist/index.d.ts CHANGED
@@ -1,10 +1,11 @@
1
1
  export * from "./api/index.js";
2
- export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
2
+ export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
3
3
  export * from "./core/view/index.js";
4
+ export * from "./core/crawler/url-paths.js";
4
5
  export * from "./lib/logger.js";
5
6
  export { createLogger, logger } from "./lib/logger.js";
6
7
  export * from "./types/api.js";
7
- export type { ComparisonConfig, SnapshotConfig, SnapshotResult } from "./types/api.js";
8
+ export type { ComparisonConfig, SnapshotConfig, SnapshotResult, } from "./types/api.js";
8
9
  export * from "./types/compare.js";
9
10
  export * from "./types/crawler.js";
10
11
  export type { CrawlError } from "./types/crawler.js";
package/dist/index.js CHANGED
@@ -1,7 +1,8 @@
1
1
  export * from "./api/index.js";
2
2
  // Explicit exports for CLI and other consumers
3
- export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
3
+ export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
4
4
  export * from "./core/view/index.js";
5
+ export * from "./core/crawler/url-paths.js";
5
6
  export * from "./lib/logger.js";
6
7
  export { createLogger, logger } from "./lib/logger.js";
7
8
  export * from "./types/api.js";
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,EACH,wBAAwB,EACxB,aAAa,EACb,aAAa,EAChB,MAAM,gBAAgB,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACvD,cAAc,gBAAgB,CAAC;AAM/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AAEnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,GACd,MAAM,gBAAgB,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACvD,cAAc,gBAAgB,CAAC;AAM/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AAEnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC"}
@@ -0,0 +1,2 @@
1
+ import type { Logger as PinoLogger } from "pino";
2
+ export declare function configureCrawleeLogger(pinoLogger: PinoLogger): void;
@@ -0,0 +1,43 @@
1
+ import { Logger as CrawleeLogger, log as crawleeLog, LogLevel } from "crawlee";
2
+ function isRecord(value) {
3
+ return typeof value === "object" && value !== null && !Array.isArray(value);
4
+ }
5
+ class PinoCrawleeLogger extends CrawleeLogger {
6
+ pinoLogger;
7
+ constructor(pinoLogger) {
8
+ super({});
9
+ this.pinoLogger = pinoLogger;
10
+ }
11
+ _log(level, message, data, exception, opts = {}) {
12
+ const component = typeof opts.prefix === "string" ? opts.prefix : undefined;
13
+ const prefix = component ? `${component}: ` : "";
14
+ const suffix = typeof opts.suffix === "string" ? ` ${opts.suffix}` : "";
15
+ const fields = {
16
+ ...(isRecord(data) ? data : {}),
17
+ ...(component ? { component } : {}),
18
+ ...(exception !== undefined ? { err: exception } : {}),
19
+ };
20
+ const logMessage = `${prefix}${message}${suffix}`;
21
+ switch (level) {
22
+ case LogLevel.ERROR:
23
+ this.pinoLogger.error(fields, logMessage);
24
+ break;
25
+ case LogLevel.SOFT_FAIL:
26
+ case LogLevel.WARNING:
27
+ this.pinoLogger.warn(fields, logMessage);
28
+ break;
29
+ case LogLevel.DEBUG:
30
+ case LogLevel.PERF:
31
+ this.pinoLogger.debug(fields, logMessage);
32
+ break;
33
+ case LogLevel.INFO:
34
+ default:
35
+ this.pinoLogger.info(fields, logMessage);
36
+ break;
37
+ }
38
+ }
39
+ }
40
+ export function configureCrawleeLogger(pinoLogger) {
41
+ crawleeLog.setOptions({ logger: new PinoCrawleeLogger(pinoLogger) });
42
+ }
43
+ //# sourceMappingURL=crawlee-logger.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"crawlee-logger.js","sourceRoot":"","sources":["../../src/lib/crawlee-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,GAAG,IAAI,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAG/E,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,iBAAkB,SAAQ,aAAa;IACd;IAA7B,YAA6B,UAAsB;QACjD,KAAK,CAAC,EAAE,CAAC,CAAC;QADiB,eAAU,GAAV,UAAU,CAAY;IAEnD,CAAC;IAEQ,IAAI,CACX,KAAe,EACf,OAAe,EACf,IAAc,EACd,SAAmB,EACnB,OAAgC,EAAE;QAElC,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;QACF,MAAM,UAAU,GAAG,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC;QAElD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,QAAQ,CAAC,KAAK;gBACjB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,SAAS,CAAC;YACxB,KAAK,QAAQ,CAAC,OAAO;gBACnB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;YACR,KAAK,QAAQ,CAAC,KAAK,CAAC;YACpB,KAAK,QAAQ,CAAC,IAAI;gBAChB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,IAAI,CAAC;YACnB;gBACE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;QACV,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAsB;IAC3D,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;AACvE,CAAC"}
@@ -4,5 +4,9 @@ export interface LoggerConfig {
4
4
  level?: LogLevel;
5
5
  }
6
6
  export declare function createLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
7
- export declare const logger: import("pino").Logger<never, boolean>;
7
+ /**
8
+ * Configure the shared Breakcheck logger and the Crawlee logger together.
9
+ */
10
+ export declare function configureLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
11
+ export declare let logger: import("pino").Logger<never, boolean>;
8
12
  export {};
@@ -1,4 +1,5 @@
1
1
  import { pino } from "pino";
2
+ import { configureCrawleeLogger } from "./crawlee-logger.js";
2
3
  // Determine the log level from the environment variable, defaulting to 'info'
3
4
  const logLevel = process.env.LOG_LEVEL || "info";
4
5
  // Create a logger with the specified configuration
@@ -20,7 +21,16 @@ export function createLogger(config = {}) {
20
21
  transport,
21
22
  });
22
23
  }
24
+ /**
25
+ * Configure the shared Breakcheck logger and the Crawlee logger together.
26
+ */
27
+ export function configureLogger(config = {}) {
28
+ logger = createLogger(config);
29
+ configureCrawleeLogger(logger);
30
+ return logger;
31
+ }
23
32
  // Create the default shared logger instance (for backward compatibility)
24
33
  // This will use pretty logging by default for CLI, but can be overridden
25
- export const logger = createLogger({ useJsonLogs: false });
34
+ export let logger = createLogger({ useJsonLogs: false });
35
+ configureCrawleeLogger(logger);
26
36
  //# sourceMappingURL=logger.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAK5B,8EAA8E;AAC9E,MAAM,QAAQ,GAAc,OAAO,CAAC,GAAG,CAAC,SAAsB,IAAI,MAAM,CAAC;AAQzE,mDAAmD;AACnD,MAAM,UAAU,YAAY,CAAC,SAAuB,EAAE;IACpD,MAAM,EAAE,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;IAEzD,gDAAgD;IAChD,MAAM,SAAS,GAAG,WAAW;QAC3B,CAAC,CAAC,SAAS,CAAC,4BAA4B;QACxC,CAAC,CAAC;YACE,MAAM,EAAE,aAAa;YACrB,OAAO,EAAE;gBACP,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,cAAc;gBAC7B,MAAM,EAAE,cAAc;aACvB;SACF,CAAC;IAEN,OAAO,IAAI,CAAC;QACV,KAAK;QACL,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,CAAC,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC"}
1
+ {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAK7D,8EAA8E;AAC9E,MAAM,QAAQ,GAAc,OAAO,CAAC,GAAG,CAAC,SAAsB,IAAI,MAAM,CAAC;AAQzE,mDAAmD;AACnD,MAAM,UAAU,YAAY,CAAC,SAAuB,EAAE;IACpD,MAAM,EAAE,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;IAEzD,gDAAgD;IAChD,MAAM,SAAS,GAAG,WAAW;QAC3B,CAAC,CAAC,SAAS,CAAC,4BAA4B;QACxC,CAAC,CAAC;YACE,MAAM,EAAE,aAAa;YACrB,OAAO,EAAE;gBACP,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,cAAc;gBAC7B,MAAM,EAAE,cAAc;aACvB;SACF,CAAC;IAEN,OAAO,IAAI,CAAC;QACV,KAAK;QACL,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,SAAuB,EAAE;IACvD,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,CAAC,IAAI,MAAM,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;AACzD,sBAAsB,CAAC,MAAM,CAAC,CAAC"}
@@ -34,6 +34,11 @@ export interface SnapshotConfig {
34
34
  name: string;
35
35
  /** Crawler configuration to use */
36
36
  crawlSettings: CrawlerConfig;
37
+ /**
38
+ * Optional root-relative paths to crawl exactly. When provided, link
39
+ * discovery is disabled and the base URL is not added automatically.
40
+ */
41
+ urlPaths?: string[];
37
42
  /** Optional path to generate a URL list file */
38
43
  urlListPath?: string;
39
44
  }
@@ -13,19 +13,19 @@ export declare const REGION_NAME_PATTERN: RegExp;
13
13
  /**
14
14
  * Base interface for action modifiers
15
15
  */
16
- interface BaseActionModifiers {
16
+ export interface ContentFilterModifiers {
17
17
  content_regex?: string;
18
18
  }
19
19
  /**
20
20
  * Modifiers for remove_attr action
21
21
  */
22
- interface RemoveAttrModifiers extends BaseActionModifiers {
22
+ export interface RemoveAttrModifiers {
23
23
  attr: string;
24
24
  }
25
25
  /**
26
26
  * Modifiers for rewrite_attr action
27
27
  */
28
- interface RewriteAttrModifiers extends BaseActionModifiers {
28
+ export interface RewriteAttrModifiers {
29
29
  attr: string;
30
30
  regex: string;
31
31
  replace: string;
@@ -33,21 +33,30 @@ interface RewriteAttrModifiers extends BaseActionModifiers {
33
33
  /**
34
34
  * Modifiers for rewrite_content action
35
35
  */
36
- interface RewriteContentModifiers extends BaseActionModifiers {
36
+ export interface RewriteContentModifiers {
37
37
  regex: string;
38
38
  replace: string;
39
39
  }
40
40
  /**
41
41
  * Union type for all possible action modifiers
42
42
  */
43
- export type ActionModifiers = RemoveAttrModifiers | RewriteAttrModifiers | RewriteContentModifiers | BaseActionModifiers;
43
+ export type ActionModifiers = RemoveAttrModifiers | RewriteAttrModifiers | RewriteContentModifiers | ContentFilterModifiers;
44
44
  /**
45
45
  * Represents a single action to be performed
46
46
  */
47
- export interface Action {
48
- action: ActionType;
49
- modifiers?: ActionModifiers;
50
- }
47
+ export type Action = {
48
+ action: "include" | "exclude";
49
+ modifiers?: ContentFilterModifiers;
50
+ } | {
51
+ action: "remove_attr";
52
+ modifiers: RemoveAttrModifiers;
53
+ } | {
54
+ action: "rewrite_attr";
55
+ modifiers: RewriteAttrModifiers;
56
+ } | {
57
+ action: "rewrite_content";
58
+ modifiers: RewriteContentModifiers;
59
+ };
51
60
  /**
52
61
  * Represents a single rule with a selector and its actions
53
62
  */
@@ -71,4 +80,3 @@ export interface Ruleset {
71
80
  /** Named regions are optional for backwards-compatible inline rulesets. */
72
81
  regions?: Region[];
73
82
  }
74
- export {};
@@ -1 +1 @@
1
- {"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAYH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,0BAA0B,CAAC"}
1
+ {"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,0BAA0B,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cleaver/breakcheck-core",
3
- "version": "0.2.3",
3
+ "version": "0.2.5",
4
4
  "type": "module",
5
5
  "engines": {
6
6
  "node": ">=22"