@cleaver/breakcheck-core 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/dist/core/compare/index.js +2 -2
- package/dist/core/compare/index.js.map +1 -1
- package/dist/core/crawler/implementations/cheerio.d.ts +1 -1
- package/dist/core/crawler/implementations/cheerio.js +2 -2
- package/dist/core/crawler/implementations/cheerio.js.map +1 -1
- package/dist/core/crawler/implementations/playwright.d.ts +1 -1
- package/dist/core/crawler/implementations/playwright.js +2 -2
- package/dist/core/crawler/implementations/playwright.js.map +1 -1
- package/dist/core/crawler/index.d.ts +6 -1
- package/dist/core/crawler/index.js +8 -4
- package/dist/core/crawler/index.js.map +1 -1
- package/dist/core/crawler/url-paths.d.ts +18 -0
- package/dist/core/crawler/url-paths.js +113 -0
- package/dist/core/crawler/url-paths.js.map +1 -0
- package/dist/core/rules/RulesDsl.js +253 -139
- package/dist/core/rules/RulesDsl.js.map +1 -1
- package/dist/core/rules/RulesEngine.d.ts +2 -2
- package/dist/core/rules/RulesEngine.js +168 -82
- package/dist/core/rules/RulesEngine.js.map +1 -1
- package/dist/core/snapshot/index.d.ts +1 -1
- package/dist/core/snapshot/index.js +36 -1
- package/dist/core/snapshot/index.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/crawlee-logger.d.ts +2 -0
- package/dist/lib/crawlee-logger.js +43 -0
- package/dist/lib/crawlee-logger.js.map +1 -0
- package/dist/lib/logger.d.ts +5 -1
- package/dist/lib/logger.js +11 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/types/api.d.ts +5 -0
- package/dist/types/rules.d.ts +18 -10
- package/dist/types/rules.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,116 +1,202 @@
|
|
|
1
1
|
import * as cheerio from "cheerio";
|
|
2
|
-
import { REGION_NAME_PATTERN
|
|
2
|
+
import { REGION_NAME_PATTERN } from "../../types/rules.js";
|
|
3
3
|
import { processRulesDsl } from "./RulesDsl.js";
|
|
4
|
+
function objectValue(value, context) {
|
|
5
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
6
|
+
throw new Error(`${context} must be an object.`);
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
9
|
+
function keysAre(modifiers, allowed, context) {
|
|
10
|
+
const extra = Object.keys(modifiers).find((key) => !allowed.includes(key));
|
|
11
|
+
if (extra)
|
|
12
|
+
throw new Error(`${context} has unsupported modifier "${extra}".`);
|
|
13
|
+
}
|
|
14
|
+
function requiredString(modifiers, key, context, nonEmpty = false) {
|
|
15
|
+
const value = modifiers[key];
|
|
16
|
+
if (typeof value !== "string" || (nonEmpty && value.length === 0))
|
|
17
|
+
throw new Error(`${context} requires ${nonEmpty ? "a non-empty" : "a string"} "${key}" modifier.`);
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
function compileRegex(pattern, context, modifier) {
|
|
21
|
+
try {
|
|
22
|
+
return new RegExp(pattern);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
throw new Error(`${context} has invalid ${modifier} regex: ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function compileAction(value, selector, ruleIndex, actionIndex) {
|
|
29
|
+
const action = objectValue(value, `Rule ${ruleIndex} action ${actionIndex}`);
|
|
30
|
+
const actionName = action.action;
|
|
31
|
+
const context = `Rule ${ruleIndex} selector "${selector}" action ${actionIndex}`;
|
|
32
|
+
const modifiers = action.modifiers === undefined
|
|
33
|
+
? {}
|
|
34
|
+
: objectValue(action.modifiers, `${context} modifiers`);
|
|
35
|
+
switch (actionName) {
|
|
36
|
+
case "include":
|
|
37
|
+
case "exclude": {
|
|
38
|
+
keysAre(modifiers, ["content_regex"], context);
|
|
39
|
+
const contentPattern = modifiers.content_regex;
|
|
40
|
+
if (contentPattern !== undefined && typeof contentPattern !== "string")
|
|
41
|
+
throw new Error(`${context} modifier "content_regex" must be a string.`);
|
|
42
|
+
return {
|
|
43
|
+
action: actionName,
|
|
44
|
+
...(typeof contentPattern === "string"
|
|
45
|
+
? {
|
|
46
|
+
contentRegex: compileRegex(contentPattern, context, "content_regex"),
|
|
47
|
+
}
|
|
48
|
+
: {}),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
case "remove_attr":
|
|
52
|
+
keysAre(modifiers, ["attr"], context);
|
|
53
|
+
return {
|
|
54
|
+
action: "remove_attr",
|
|
55
|
+
attr: requiredString(modifiers, "attr", context, true),
|
|
56
|
+
};
|
|
57
|
+
case "rewrite_attr": {
|
|
58
|
+
keysAre(modifiers, ["attr", "regex", "replace"], context);
|
|
59
|
+
const attr = requiredString(modifiers, "attr", context, true);
|
|
60
|
+
const pattern = requiredString(modifiers, "regex", context);
|
|
61
|
+
return {
|
|
62
|
+
action: "rewrite_attr",
|
|
63
|
+
attr,
|
|
64
|
+
regex: compileRegex(pattern, context, "regex"),
|
|
65
|
+
replace: requiredString(modifiers, "replace", context),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
case "rewrite_content": {
|
|
69
|
+
keysAre(modifiers, ["regex", "replace"], context);
|
|
70
|
+
const pattern = requiredString(modifiers, "regex", context);
|
|
71
|
+
return {
|
|
72
|
+
action: "rewrite_content",
|
|
73
|
+
regex: compileRegex(pattern, context, "regex"),
|
|
74
|
+
replace: requiredString(modifiers, "replace", context),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
default:
|
|
78
|
+
throw new Error(`${context} has unsupported action "${String(actionName)}".`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function validateSelector(selector, context) {
|
|
82
|
+
if (typeof selector !== "string" || selector.trim().length === 0)
|
|
83
|
+
throw new Error(`${context} requires a non-empty selector.`);
|
|
84
|
+
const cloned = selector.slice();
|
|
85
|
+
try {
|
|
86
|
+
cheerio.load("", {}, false)(cloned);
|
|
87
|
+
}
|
|
88
|
+
catch (error) {
|
|
89
|
+
throw new Error(`${context} has invalid selector "${cloned}": ${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
90
|
+
}
|
|
91
|
+
return cloned;
|
|
92
|
+
}
|
|
93
|
+
function compileRuleset(value) {
|
|
94
|
+
const input = objectValue(value, "Ruleset");
|
|
95
|
+
if (!Array.isArray(input.rules))
|
|
96
|
+
throw new Error("Ruleset rules must be an array.");
|
|
97
|
+
const rules = input.rules.map((candidate, ruleIndex) => {
|
|
98
|
+
const rule = objectValue(candidate, `Rule ${ruleIndex}`);
|
|
99
|
+
const selector = validateSelector(rule.selector, `Rule ${ruleIndex}`);
|
|
100
|
+
if (!Array.isArray(rule.actions) || rule.actions.length === 0)
|
|
101
|
+
throw new Error(`Rule ${ruleIndex} selector "${selector}" requires at least one action.`);
|
|
102
|
+
return {
|
|
103
|
+
selector,
|
|
104
|
+
actions: rule.actions.map((action, actionIndex) => compileAction(action, selector, ruleIndex, actionIndex)),
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
const regionsValue = input.regions ?? [];
|
|
108
|
+
if (!Array.isArray(regionsValue))
|
|
109
|
+
throw new Error("Ruleset regions must be an array.");
|
|
110
|
+
const names = new Set();
|
|
111
|
+
const regions = regionsValue.map((candidate, regionIndex) => {
|
|
112
|
+
const region = objectValue(candidate, `Region ${regionIndex}`);
|
|
113
|
+
const selector = validateSelector(region.selector, `Region ${regionIndex}`);
|
|
114
|
+
if (typeof region.name !== "string" ||
|
|
115
|
+
!REGION_NAME_PATTERN.test(region.name))
|
|
116
|
+
throw new Error(`Invalid region name "${String(region.name)}". Region names must match [A-Za-z_][A-Za-z0-9_]*.`);
|
|
117
|
+
if (names.has(region.name))
|
|
118
|
+
throw new Error(`Duplicate region name: ${region.name}`);
|
|
119
|
+
names.add(region.name);
|
|
120
|
+
return { selector, name: region.name.slice() };
|
|
121
|
+
});
|
|
122
|
+
return { rules, regions };
|
|
123
|
+
}
|
|
4
124
|
export class RulesEngine {
|
|
5
125
|
ruleset;
|
|
6
126
|
constructor(ruleset) {
|
|
7
|
-
|
|
8
|
-
this.validateRegions(regions);
|
|
9
|
-
this.ruleset = { ...ruleset, regions };
|
|
127
|
+
this.ruleset = ruleset;
|
|
10
128
|
}
|
|
11
129
|
static async create(rulesDirectoryOrRuleset) {
|
|
12
130
|
const ruleset = typeof rulesDirectoryOrRuleset === "string"
|
|
13
131
|
? await processRulesDsl(rulesDirectoryOrRuleset)
|
|
14
|
-
: rulesDirectoryOrRuleset ?? { name: "none", rules: [] };
|
|
15
|
-
return new RulesEngine(ruleset);
|
|
132
|
+
: (rulesDirectoryOrRuleset ?? { name: "none", rules: [] });
|
|
133
|
+
return new RulesEngine(compileRuleset(ruleset));
|
|
16
134
|
}
|
|
17
135
|
process(html) {
|
|
18
136
|
const $ = cheerio.load(html);
|
|
19
|
-
// Apply ordinary rules in declaration order before selecting regions.
|
|
20
137
|
for (const rule of this.ruleset.rules) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
138
|
+
$(rule.selector).each((_, element) => {
|
|
139
|
+
if (element.type !== "tag" &&
|
|
140
|
+
element.type !== "script" &&
|
|
141
|
+
element.type !== "style")
|
|
142
|
+
return;
|
|
143
|
+
for (const action of rule.actions)
|
|
144
|
+
this.applyAction($, element, action);
|
|
28
145
|
});
|
|
29
146
|
}
|
|
30
|
-
|
|
31
|
-
if (regions.length > 0) {
|
|
32
|
-
return this.serializeRegions($, regions);
|
|
33
|
-
}
|
|
34
|
-
return $.html();
|
|
147
|
+
return this.ruleset.regions.length ? this.serializeRegions($) : $.html();
|
|
35
148
|
}
|
|
36
|
-
|
|
37
|
-
const names = new Set();
|
|
38
|
-
for (const region of regions) {
|
|
39
|
-
if (!REGION_NAME_PATTERN.test(region.name)) {
|
|
40
|
-
throw new Error(`Invalid region name "${region.name}". Region names must match [A-Za-z_][A-Za-z0-9_]*.`);
|
|
41
|
-
}
|
|
42
|
-
if (names.has(region.name)) {
|
|
43
|
-
throw new Error(`Duplicate region name: ${region.name}`);
|
|
44
|
-
}
|
|
45
|
-
names.add(region.name);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
serializeRegions($, regions) {
|
|
149
|
+
serializeRegions($) {
|
|
49
150
|
const output = cheerio.load("<breakcheck-regions></breakcheck-regions>", {}, false);
|
|
50
151
|
const root = output("breakcheck-regions");
|
|
51
|
-
const
|
|
52
|
-
for (const region of
|
|
152
|
+
const regions = [...this.ruleset.regions].sort((left, right) => left.name.localeCompare(right.name));
|
|
153
|
+
for (const region of regions) {
|
|
53
154
|
const fragments = [];
|
|
54
155
|
$(region.selector).each((_, element) => {
|
|
55
|
-
if (element.type === "tag")
|
|
156
|
+
if (element.type === "tag")
|
|
56
157
|
fragments.push($.html(element));
|
|
57
|
-
}
|
|
58
158
|
});
|
|
59
|
-
if (fragments.length
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
wrapper.append(fragments.join(""));
|
|
64
|
-
root.append(wrapper);
|
|
159
|
+
if (fragments.length)
|
|
160
|
+
root.append(output("<region></region>")
|
|
161
|
+
.attr("name", region.name)
|
|
162
|
+
.append(fragments.join("")));
|
|
65
163
|
}
|
|
66
164
|
return output.html();
|
|
67
165
|
}
|
|
68
166
|
applyAction($, element, action) {
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if (!regex.test(content)) {
|
|
75
|
-
return;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
167
|
+
const selected = $(element);
|
|
168
|
+
if ("contentRegex" in action &&
|
|
169
|
+
action.contentRegex &&
|
|
170
|
+
!action.contentRegex.test(selected.text()))
|
|
171
|
+
return;
|
|
78
172
|
switch (action.action) {
|
|
79
|
-
case "exclude":
|
|
80
|
-
$element.remove();
|
|
81
|
-
break;
|
|
82
173
|
case "include":
|
|
83
|
-
|
|
84
|
-
|
|
174
|
+
return;
|
|
175
|
+
case "exclude":
|
|
176
|
+
selected.remove();
|
|
177
|
+
return;
|
|
85
178
|
case "remove_attr":
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
"replace" in action.modifiers) {
|
|
95
|
-
const currentValue = $element.attr(action.modifiers.attr);
|
|
96
|
-
if (currentValue) {
|
|
97
|
-
const regex = new RegExp(action.modifiers.regex);
|
|
98
|
-
const newValue = currentValue.replace(regex, action.modifiers.replace);
|
|
99
|
-
$element.attr(action.modifiers.attr, newValue);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
break;
|
|
179
|
+
selected.removeAttr(action.attr);
|
|
180
|
+
return;
|
|
181
|
+
case "rewrite_attr": {
|
|
182
|
+
const current = selected.attr(action.attr);
|
|
183
|
+
if (current !== undefined)
|
|
184
|
+
selected.attr(action.attr, current.replace(action.regex, action.replace));
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
103
187
|
case "rewrite_content":
|
|
104
|
-
|
|
105
|
-
"regex" in action.modifiers &&
|
|
106
|
-
"replace" in action.modifiers) {
|
|
107
|
-
const content = $element.text();
|
|
108
|
-
const regex = new RegExp(action.modifiers.regex);
|
|
109
|
-
const newContent = content.replace(regex, action.modifiers.replace);
|
|
110
|
-
$element.text(newContent);
|
|
111
|
-
}
|
|
112
|
-
break;
|
|
188
|
+
this.rewriteTextNodes($, element, action.regex, action.replace);
|
|
113
189
|
}
|
|
114
190
|
}
|
|
191
|
+
rewriteTextNodes($, element, regex, replace) {
|
|
192
|
+
$(element)
|
|
193
|
+
.contents()
|
|
194
|
+
.each((_, child) => {
|
|
195
|
+
if (child.type === "text")
|
|
196
|
+
child.data = child.data.replace(regex, replace);
|
|
197
|
+
else if (child.type === "tag")
|
|
198
|
+
this.rewriteTextNodes($, child, regex, replace);
|
|
199
|
+
});
|
|
200
|
+
}
|
|
115
201
|
}
|
|
116
202
|
//# sourceMappingURL=RulesEngine.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,
|
|
1
|
+
{"version":3,"file":"RulesEngine.js","sourceRoot":"","sources":["../../../src/core/rules/RulesEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAU,mBAAmB,EAAW,MAAM,sBAAsB,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAoBhD,SAAS,WAAW,CAAC,KAAc,EAAE,OAAe;IAClD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QACrE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,qBAAqB,CAAC,CAAC;IACnD,OAAO,KAAgC,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CACd,SAAkC,EAClC,OAA0B,EAC1B,OAAe;IAEf,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,8BAA8B,KAAK,IAAI,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,cAAc,CACrB,SAAkC,EAClC,GAAW,EACX,OAAe,EACf,QAAQ,GAAG,KAAK;IAEhB,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QAC/D,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,aAAa,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,UAAU,KAAK,GAAG,aAAa,CAClF,CAAC;IACJ,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CACnB,OAAe,EACf,OAAe,EACf,QAAgB;IAEhB,IAAI,CAAC;QACH,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,gBAAgB,QAAQ,WAAW,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACrG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CACpB,KAAc,EACd,QAAgB,EAChB,SAAiB,EACjB,WAAmB;IAEnB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,EAAE,QAAQ,SAAS,WAAW,WAAW,EAAE,CAAC,CAAC;IAC7E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;IACjC,MAAM,OAAO,GAAG,QAAQ,SAAS,cAAc,QAAQ,YAAY,WAAW,EAAE,CAAC;IACjF,MAAM,SAAS,GACb,MAAM,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,CAAC,EAAE;QACJ,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,OAAO,YAAY,CAAC,CAAC;IAE5D,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,SAAS,CAAC,CAAC,CAAC;YACf,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,CAAC;YAC/C,MAAM,cAAc,GAAG,SAAS,CAAC,aAAa,CAAC;YAC/C,IAAI,cAAc,KAAK,SAAS,IAAI,OAAO,cAAc,KAAK,QAAQ;gBACpE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,6CAA6C,CACxD,CAAC;YACJ,OAAO;gBACL,MAAM,EAAE,UAAU;gBAClB,GAAG,CAAC,OAAO,cAAc,KAAK,QAAQ;oBACpC,CAAC,CAAC;wBACE,YAAY,EAAE,YAAY,CACxB,cAAc,EACd,OAAO,EACP,eAAe,CAChB;qBACF;oBACH,CAAC,CAAC,EAAE,CAAC;aACR,CAAC;QACJ,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YACtC,OAAO;gBACL,MAAM,EAAE,aAAa;gBACrB,IAAI,EAAE,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;aACvD,CAAC;QACJ,KAAK,cAAc,CAAC,CAAC,CAAC;YACpB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAC1D,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,cAAc;gBACtB,IAAI;gBACJ,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD,KAAK,iBAAiB,CAAC,CAAC,CAAC;YACvB,OAAO,CAAC,SAAS,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,OAAO;gBACL,MAAM,EAAE,iBAAiB;gBACzB,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;gBAC9C,OAAO,EAAE,cAAc,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC;aACvD,CAAC;QACJ,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,4BAA4B,MAAM,CAAC,UAAU,CAAC,IAAI,CAC7D,CAAC;IACN,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAiB,EAAE,OAAe;IAC1D,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,GAAG,OAAO,iCAAiC,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,0BAA0B,MAAM,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EACxG,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE;QACrD,MAAM,IAAI,GAAG,WAAW,CAAC,SAAS,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;QACtE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,QAAQ,SAAS,cAAc,QAAQ,iCAAiC,CACzE,CAAC;QACJ,OAAO;YACL,QAAQ;YACR,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAChD,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,CACxD;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,IAAI,EAAE,CAAC;IACzC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,YAAY,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,WAAW,CAAC,SAAS,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC/D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,WAAW,EAAE,CAAC,CAAC;QAC5E,IACE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ;YAC/B,CAAC,mBAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAEtC,MAAM,IAAI,KAAK,CACb,wBAAwB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,oDAAoD,CAChG,CAAC;QACJ,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,0BAA0B,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACvB,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,OAAO,WAAW;IACe;IAArC,YAAqC,OAAwB;QAAxB,YAAO,GAAP,OAAO,CAAiB;IAAG,CAAC;IAE1D,MAAM,CAAC,KAAK,CAAC,MAAM,CACxB,uBAA0C;QAE1C,MAAM,OAAO,GACX,OAAO,uBAAuB,KAAK,QAAQ;YACzC,CAAC,CAAC,MAAM,eAAe,CAAC,uBAAuB,CAAC;YAChD,CAAC,CAAC,CAAC,uBAAuB,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,IAAI,WAAW,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;IAClD,CAAC;IAEM,OAAO,CAAC,IAAY;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACtC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACnC,IACE,OAAO,CAAC,IAAI,KAAK,KAAK;oBACtB,OAAO,CAAC,IAAI,KAAK,QAAQ;oBACzB,OAAO,CAAC,IAAI,KAAK,OAAO;oBAExB,OAAO;gBACT,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO;oBAAE,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;YAC1E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,CAAC;IAEO,gBAAgB,CAAC,CAAqB;QAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CACzB,2CAA2C,EAC3C,EAAE,EACF,KAAK,CACN,CAAC;QACF,MAAM,IAAI,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAC7D,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CACpC,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAa,EAAE,CAAC;YAC/B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;gBACrC,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK;oBAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,CAAC,MAAM;gBAClB,IAAI,CAAC,MAAM,CACT,MAAM,CAAC,mBAAmB,CAAC;qBACxB,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC;qBACzB,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAC9B,CAAC;QACN,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IACvB,CAAC;IAEO,WAAW,CACjB,CAAqB,EACrB,OAAwB,EACxB,MAAsB;QAEtB,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;QAC5B,IACE,cAAc,IAAI,MAAM;YACxB,MAAM,CAAC,YAAY;YACnB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE1C,OAAO;QACT,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,SAAS;gBACZ,OAAO;YACT,KAAK,SAAS;gBACZ,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO;YACT,KAAK,aAAa;gBAChB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACjC,OAAO;YACT,KAAK,cAAc,CAAC,CAAC,CAAC;gBACpB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC3C,IAAI,OAAO,KAAK,SAAS;oBACvB,QAAQ,CAAC,IAAI,CACX,MAAM,CAAC,IAAI,EACX,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAC9C,CAAC;gBACJ,OAAO;YACT,CAAC;YACD,KAAK,iBAAiB;gBACpB,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAEO,gBAAgB,CACtB,CAAqB,EACrB,OAAwB,EACxB,KAAa,EACb,OAAe;QAEf,CAAC,CAAC,OAAO,CAAC;aACP,QAAQ,EAAE;aACV,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YACjB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;gBACvB,KAAK,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;iBAC7C,IAAI,KAAK,CAAC,IAAI,KAAK,KAAK;gBAC3B,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC,CAAC,CAAC;IACP,CAAC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { SnapshotConfig, SnapshotResult } from "../../types/api.js";
|
|
1
|
+
import type { SnapshotConfig, SnapshotResult } from "../../types/api.js";
|
|
2
2
|
import { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
3
3
|
export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
|
|
4
4
|
export { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Dataset } from "crawlee";
|
|
2
2
|
import { logger } from "../../lib/logger.js";
|
|
3
|
+
import { resolveUrlPaths } from "../crawler/url-paths.js";
|
|
3
4
|
import { BreakcheckCrawler } from "../crawler/index.js";
|
|
4
5
|
export { LoadedSnapshot } from "./classes/LoadedSnapshot.js";
|
|
5
6
|
export { SnapshotRepository } from "./classes/SnapshotRepository.js";
|
|
@@ -20,7 +21,27 @@ export async function createSnapshot(config, snapshotRepository) {
|
|
|
20
21
|
logger.error({ config }, "Snapshot creation failed: name is required");
|
|
21
22
|
throw new Error("name is required");
|
|
22
23
|
}
|
|
23
|
-
const
|
|
24
|
+
const manifest = config.urlPaths === undefined
|
|
25
|
+
? undefined
|
|
26
|
+
: resolveUrlPaths(config.baseUrl, config.urlPaths);
|
|
27
|
+
if (manifest?.issues.length) {
|
|
28
|
+
throw new Error(formatUrlPathIssues(manifest.issues));
|
|
29
|
+
}
|
|
30
|
+
const startUrls = manifest?.paths.map((urlPath) => new URL(urlPath, config.baseUrl).href);
|
|
31
|
+
if (startUrls &&
|
|
32
|
+
config.crawlSettings.maxRequests !== undefined &&
|
|
33
|
+
config.crawlSettings.maxRequests < startUrls.length) {
|
|
34
|
+
throw new Error(`maxRequests (${config.crawlSettings.maxRequests}) must be at least the number of unique manifest paths (${startUrls.length})`);
|
|
35
|
+
}
|
|
36
|
+
if (startUrls &&
|
|
37
|
+
(config.crawlSettings.includePatterns?.length ||
|
|
38
|
+
config.crawlSettings.excludePatterns?.length)) {
|
|
39
|
+
throw new Error("includePatterns and excludePatterns cannot be used with an exact URL manifest");
|
|
40
|
+
}
|
|
41
|
+
const crawler = new BreakcheckCrawler(config.crawlSettings, {
|
|
42
|
+
startUrls,
|
|
43
|
+
followLinks: config.urlPaths === undefined,
|
|
44
|
+
});
|
|
24
45
|
const { datasetName, errors: crawlErrors } = await crawler.crawl();
|
|
25
46
|
errors.push(...crawlErrors);
|
|
26
47
|
const dataset = await Dataset.open(datasetName);
|
|
@@ -64,4 +85,18 @@ export async function createSnapshot(config, snapshotRepository) {
|
|
|
64
85
|
};
|
|
65
86
|
}
|
|
66
87
|
}
|
|
88
|
+
function formatUrlPathIssues(issues) {
|
|
89
|
+
return issues
|
|
90
|
+
.map((issue) => {
|
|
91
|
+
switch (issue.type) {
|
|
92
|
+
case "base":
|
|
93
|
+
return `${issue.message}: ${issue.value}`;
|
|
94
|
+
case "entry":
|
|
95
|
+
return `Manifest path ${issue.index + 1} (${issue.value}): ${issue.message}`;
|
|
96
|
+
case "manifest":
|
|
97
|
+
return issue.message;
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
.join("; ");
|
|
101
|
+
}
|
|
67
102
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/snapshot/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAG7C,OAAO,EAAE,eAAe,EAAqB,MAAM,yBAAyB,CAAC;AAC7E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAErE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,MAAsB,EACtB,kBAAsC;IAEtC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,IAAI,CAAC;QACH,wBAAwB;QACxB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,+CAA+C,CAAC,CAAC;YAC1E,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,EAAE,4CAA4C,CAAC,CAAC;YACvE,MAAM,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACtC,CAAC;QAED,MAAM,QAAQ,GACZ,MAAM,CAAC,QAAQ,KAAK,SAAS;YAC3B,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEvD,IAAI,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,CACnC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CACnD,CAAC;QAEF,IACE,SAAS;YACT,MAAM,CAAC,aAAa,CAAC,WAAW,KAAK,SAAS;YAC9C,MAAM,CAAC,aAAa,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM,EACnD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,gBAAgB,MAAM,CAAC,aAAa,CAAC,WAAW,2DAA2D,SAAS,CAAC,MAAM,GAAG,CAC/H,CAAC;QACJ,CAAC;QAED,IACE,SAAS;YACT,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM;gBAC3C,MAAM,CAAC,aAAa,CAAC,eAAe,EAAE,MAAM,CAAC,EAC/C,CAAC;YACD,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,iBAAiB,CAAC,MAAM,CAAC,aAAa,EAAE;YAC1D,SAAS;YACT,WAAW,EAAE,MAAM,CAAC,QAAQ,KAAK,SAAS;SAC3C,CAAC,CAAC;QAEH,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QAEnE,MAAM,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;QAE5B,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEhD,MAAM,SAAS,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;YACnE,OAAO;YACP,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACnC,aAAa,EAAE,MAAM,CAAC,aAAa;aACpC;SACF,CAAC,CAAC;QAEH,IAAI,WAA+B,CAAC;QACpC,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,WAAW,GAAG,MAAM,kBAAkB,CAAC,eAAe,CACpD,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,WAAW,CACnB,CAAC;QACJ,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QAExC,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,SAAS;YACT,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE;gBACR,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,UAAU,EAAE,QAAQ;aACrB;YACD,WAAW;SACZ,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC;YACV,GAAG,EAAE,MAAM,CAAC,OAAO;YACnB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;SACpE,CAAC,CAAC;QAEH,OAAO;YACL,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,MAAM,CAAC,IAAI;YACvB,OAAO,EACL,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,wBAAwB;YACnE,MAAM;SACP,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAsB;IACjD,OAAO,MAAM;SACV,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,MAAM;gBACT,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5C,KAAK,OAAO;gBACV,OAAO,iBAAiB,KAAK,CAAC,KAAK,GAAG,CAAC,KAAK,KAAK,CAAC,KAAK,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;YAC/E,KAAK,UAAU;gBACb,OAAO,KAAK,CAAC,OAAO,CAAC;QACzB,CAAC;IACH,CAAC,CAAC;SACD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
export * from "./api/index.js";
|
|
2
|
-
export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
|
|
2
|
+
export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
|
|
3
3
|
export * from "./core/view/index.js";
|
|
4
|
+
export * from "./core/crawler/url-paths.js";
|
|
4
5
|
export * from "./lib/logger.js";
|
|
5
6
|
export { createLogger, logger } from "./lib/logger.js";
|
|
6
7
|
export * from "./types/api.js";
|
|
7
|
-
export type { ComparisonConfig, SnapshotConfig, SnapshotResult } from "./types/api.js";
|
|
8
|
+
export type { ComparisonConfig, SnapshotConfig, SnapshotResult, } from "./types/api.js";
|
|
8
9
|
export * from "./types/compare.js";
|
|
9
10
|
export * from "./types/crawler.js";
|
|
10
11
|
export type { CrawlError } from "./types/crawler.js";
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
export * from "./api/index.js";
|
|
2
2
|
// Explicit exports for CLI and other consumers
|
|
3
|
-
export { createSnapshotFromConfig, listSnapshots, runComparison } from "./api/index.js";
|
|
3
|
+
export { createSnapshotFromConfig, listSnapshots, runComparison, } from "./api/index.js";
|
|
4
4
|
export * from "./core/view/index.js";
|
|
5
|
+
export * from "./core/crawler/url-paths.js";
|
|
5
6
|
export * from "./lib/logger.js";
|
|
6
7
|
export { createLogger, logger } from "./lib/logger.js";
|
|
7
8
|
export * from "./types/api.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,gBAAgB,CAAC;AAC/B,+CAA+C;AAC/C,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,aAAa,GACd,MAAM,gBAAgB,CAAC;AACxB,cAAc,sBAAsB,CAAC;AACrC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,iBAAiB,CAAC;AAChC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACvD,cAAc,gBAAgB,CAAC;AAM/B,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AAEnC,cAAc,kBAAkB,CAAC;AACjC,cAAc,qBAAqB,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { Logger as CrawleeLogger, log as crawleeLog, LogLevel } from "crawlee";
|
|
2
|
+
function isRecord(value) {
|
|
3
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4
|
+
}
|
|
5
|
+
class PinoCrawleeLogger extends CrawleeLogger {
|
|
6
|
+
pinoLogger;
|
|
7
|
+
constructor(pinoLogger) {
|
|
8
|
+
super({});
|
|
9
|
+
this.pinoLogger = pinoLogger;
|
|
10
|
+
}
|
|
11
|
+
_log(level, message, data, exception, opts = {}) {
|
|
12
|
+
const component = typeof opts.prefix === "string" ? opts.prefix : undefined;
|
|
13
|
+
const prefix = component ? `${component}: ` : "";
|
|
14
|
+
const suffix = typeof opts.suffix === "string" ? ` ${opts.suffix}` : "";
|
|
15
|
+
const fields = {
|
|
16
|
+
...(isRecord(data) ? data : {}),
|
|
17
|
+
...(component ? { component } : {}),
|
|
18
|
+
...(exception !== undefined ? { err: exception } : {}),
|
|
19
|
+
};
|
|
20
|
+
const logMessage = `${prefix}${message}${suffix}`;
|
|
21
|
+
switch (level) {
|
|
22
|
+
case LogLevel.ERROR:
|
|
23
|
+
this.pinoLogger.error(fields, logMessage);
|
|
24
|
+
break;
|
|
25
|
+
case LogLevel.SOFT_FAIL:
|
|
26
|
+
case LogLevel.WARNING:
|
|
27
|
+
this.pinoLogger.warn(fields, logMessage);
|
|
28
|
+
break;
|
|
29
|
+
case LogLevel.DEBUG:
|
|
30
|
+
case LogLevel.PERF:
|
|
31
|
+
this.pinoLogger.debug(fields, logMessage);
|
|
32
|
+
break;
|
|
33
|
+
case LogLevel.INFO:
|
|
34
|
+
default:
|
|
35
|
+
this.pinoLogger.info(fields, logMessage);
|
|
36
|
+
break;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
export function configureCrawleeLogger(pinoLogger) {
|
|
41
|
+
crawleeLog.setOptions({ logger: new PinoCrawleeLogger(pinoLogger) });
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=crawlee-logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crawlee-logger.js","sourceRoot":"","sources":["../../src/lib/crawlee-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAE,GAAG,IAAI,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAG/E,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,MAAM,iBAAkB,SAAQ,aAAa;IACd;IAA7B,YAA6B,UAAsB;QACjD,KAAK,CAAC,EAAE,CAAC,CAAC;QADiB,eAAU,GAAV,UAAU,CAAY;IAEnD,CAAC;IAEQ,IAAI,CACX,KAAe,EACf,OAAe,EACf,IAAc,EACd,SAAmB,EACnB,OAAgC,EAAE;QAElC,MAAM,SAAS,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG;YACb,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/B,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;QACF,MAAM,UAAU,GAAG,GAAG,MAAM,GAAG,OAAO,GAAG,MAAM,EAAE,CAAC;QAElD,QAAQ,KAAK,EAAE,CAAC;YACd,KAAK,QAAQ,CAAC,KAAK;gBACjB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,SAAS,CAAC;YACxB,KAAK,QAAQ,CAAC,OAAO;gBACnB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;YACR,KAAK,QAAQ,CAAC,KAAK,CAAC;YACpB,KAAK,QAAQ,CAAC,IAAI;gBAChB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,QAAQ,CAAC,IAAI,CAAC;YACnB;gBACE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;gBACzC,MAAM;QACV,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAsB;IAC3D,UAAU,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;AACvE,CAAC"}
|
package/dist/lib/logger.d.ts
CHANGED
|
@@ -4,5 +4,9 @@ export interface LoggerConfig {
|
|
|
4
4
|
level?: LogLevel;
|
|
5
5
|
}
|
|
6
6
|
export declare function createLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
|
|
7
|
-
|
|
7
|
+
/**
|
|
8
|
+
* Configure the shared Breakcheck logger and the Crawlee logger together.
|
|
9
|
+
*/
|
|
10
|
+
export declare function configureLogger(config?: LoggerConfig): import("pino").Logger<never, boolean>;
|
|
11
|
+
export declare let logger: import("pino").Logger<never, boolean>;
|
|
8
12
|
export {};
|
package/dist/lib/logger.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { pino } from "pino";
|
|
2
|
+
import { configureCrawleeLogger } from "./crawlee-logger.js";
|
|
2
3
|
// Determine the log level from the environment variable, defaulting to 'info'
|
|
3
4
|
const logLevel = process.env.LOG_LEVEL || "info";
|
|
4
5
|
// Create a logger with the specified configuration
|
|
@@ -20,7 +21,16 @@ export function createLogger(config = {}) {
|
|
|
20
21
|
transport,
|
|
21
22
|
});
|
|
22
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Configure the shared Breakcheck logger and the Crawlee logger together.
|
|
26
|
+
*/
|
|
27
|
+
export function configureLogger(config = {}) {
|
|
28
|
+
logger = createLogger(config);
|
|
29
|
+
configureCrawleeLogger(logger);
|
|
30
|
+
return logger;
|
|
31
|
+
}
|
|
23
32
|
// Create the default shared logger instance (for backward compatibility)
|
|
24
33
|
// This will use pretty logging by default for CLI, but can be overridden
|
|
25
|
-
export
|
|
34
|
+
export let logger = createLogger({ useJsonLogs: false });
|
|
35
|
+
configureCrawleeLogger(logger);
|
|
26
36
|
//# sourceMappingURL=logger.js.map
|
package/dist/lib/logger.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/lib/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAK7D,8EAA8E;AAC9E,MAAM,QAAQ,GAAc,OAAO,CAAC,GAAG,CAAC,SAAsB,IAAI,MAAM,CAAC;AAQzE,mDAAmD;AACnD,MAAM,UAAU,YAAY,CAAC,SAAuB,EAAE;IACpD,MAAM,EAAE,WAAW,GAAG,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC;IAEzD,gDAAgD;IAChD,MAAM,SAAS,GAAG,WAAW;QAC3B,CAAC,CAAC,SAAS,CAAC,4BAA4B;QACxC,CAAC,CAAC;YACE,MAAM,EAAE,aAAa;YACrB,OAAO,EAAE;gBACP,QAAQ,EAAE,IAAI;gBACd,aAAa,EAAE,cAAc;gBAC7B,MAAM,EAAE,cAAc;aACvB;SACF,CAAC;IAEN,OAAO,IAAI,CAAC;QACV,KAAK;QACL,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,SAAuB,EAAE;IACvD,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,yEAAyE;AACzE,MAAM,CAAC,IAAI,MAAM,GAAG,YAAY,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;AACzD,sBAAsB,CAAC,MAAM,CAAC,CAAC"}
|
package/dist/types/api.d.ts
CHANGED
|
@@ -34,6 +34,11 @@ export interface SnapshotConfig {
|
|
|
34
34
|
name: string;
|
|
35
35
|
/** Crawler configuration to use */
|
|
36
36
|
crawlSettings: CrawlerConfig;
|
|
37
|
+
/**
|
|
38
|
+
* Optional root-relative paths to crawl exactly. When provided, link
|
|
39
|
+
* discovery is disabled and the base URL is not added automatically.
|
|
40
|
+
*/
|
|
41
|
+
urlPaths?: string[];
|
|
37
42
|
/** Optional path to generate a URL list file */
|
|
38
43
|
urlListPath?: string;
|
|
39
44
|
}
|
package/dist/types/rules.d.ts
CHANGED
|
@@ -13,19 +13,19 @@ export declare const REGION_NAME_PATTERN: RegExp;
|
|
|
13
13
|
/**
|
|
14
14
|
* Base interface for action modifiers
|
|
15
15
|
*/
|
|
16
|
-
interface
|
|
16
|
+
export interface ContentFilterModifiers {
|
|
17
17
|
content_regex?: string;
|
|
18
18
|
}
|
|
19
19
|
/**
|
|
20
20
|
* Modifiers for remove_attr action
|
|
21
21
|
*/
|
|
22
|
-
interface RemoveAttrModifiers
|
|
22
|
+
export interface RemoveAttrModifiers {
|
|
23
23
|
attr: string;
|
|
24
24
|
}
|
|
25
25
|
/**
|
|
26
26
|
* Modifiers for rewrite_attr action
|
|
27
27
|
*/
|
|
28
|
-
interface RewriteAttrModifiers
|
|
28
|
+
export interface RewriteAttrModifiers {
|
|
29
29
|
attr: string;
|
|
30
30
|
regex: string;
|
|
31
31
|
replace: string;
|
|
@@ -33,21 +33,30 @@ interface RewriteAttrModifiers extends BaseActionModifiers {
|
|
|
33
33
|
/**
|
|
34
34
|
* Modifiers for rewrite_content action
|
|
35
35
|
*/
|
|
36
|
-
interface RewriteContentModifiers
|
|
36
|
+
export interface RewriteContentModifiers {
|
|
37
37
|
regex: string;
|
|
38
38
|
replace: string;
|
|
39
39
|
}
|
|
40
40
|
/**
|
|
41
41
|
* Union type for all possible action modifiers
|
|
42
42
|
*/
|
|
43
|
-
export type ActionModifiers = RemoveAttrModifiers | RewriteAttrModifiers | RewriteContentModifiers |
|
|
43
|
+
export type ActionModifiers = RemoveAttrModifiers | RewriteAttrModifiers | RewriteContentModifiers | ContentFilterModifiers;
|
|
44
44
|
/**
|
|
45
45
|
* Represents a single action to be performed
|
|
46
46
|
*/
|
|
47
|
-
export
|
|
48
|
-
action:
|
|
49
|
-
modifiers?:
|
|
50
|
-
}
|
|
47
|
+
export type Action = {
|
|
48
|
+
action: "include" | "exclude";
|
|
49
|
+
modifiers?: ContentFilterModifiers;
|
|
50
|
+
} | {
|
|
51
|
+
action: "remove_attr";
|
|
52
|
+
modifiers: RemoveAttrModifiers;
|
|
53
|
+
} | {
|
|
54
|
+
action: "rewrite_attr";
|
|
55
|
+
modifiers: RewriteAttrModifiers;
|
|
56
|
+
} | {
|
|
57
|
+
action: "rewrite_content";
|
|
58
|
+
modifiers: RewriteContentModifiers;
|
|
59
|
+
};
|
|
51
60
|
/**
|
|
52
61
|
* Represents a single rule with a selector and its actions
|
|
53
62
|
*/
|
|
@@ -71,4 +80,3 @@ export interface Ruleset {
|
|
|
71
80
|
/** Named regions are optional for backwards-compatible inline rulesets. */
|
|
72
81
|
regions?: Region[];
|
|
73
82
|
}
|
|
74
|
-
export {};
|
package/dist/types/rules.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"rules.js","sourceRoot":"","sources":["../../src/types/rules.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,0BAA0B,CAAC"}
|