@cleaver/breakcheck-core 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -1
- package/dist/core/compare/index.js +2 -2
- package/dist/core/compare/index.js.map +1 -1
- package/dist/core/crawler/implementations/cheerio.d.ts +1 -1
- package/dist/core/crawler/implementations/cheerio.js +2 -2
- package/dist/core/crawler/implementations/cheerio.js.map +1 -1
- package/dist/core/crawler/implementations/playwright.d.ts +1 -1
- package/dist/core/crawler/implementations/playwright.js +2 -2
- package/dist/core/crawler/implementations/playwright.js.map +1 -1
- package/dist/core/crawler/index.d.ts +6 -1
- package/dist/core/crawler/index.js +8 -4
- package/dist/core/crawler/index.js.map +1 -1
- package/dist/core/crawler/url-paths.d.ts +18 -0
- package/dist/core/crawler/url-paths.js +113 -0
- package/dist/core/crawler/url-paths.js.map +1 -0
- package/dist/core/rules/RulesDsl.js +253 -139
- package/dist/core/rules/RulesDsl.js.map +1 -1
- package/dist/core/rules/RulesEngine.d.ts +2 -2
- package/dist/core/rules/RulesEngine.js +168 -82
- package/dist/core/rules/RulesEngine.js.map +1 -1
- package/dist/core/snapshot/index.d.ts +1 -1
- package/dist/core/snapshot/index.js +36 -1
- package/dist/core/snapshot/index.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/crawlee-logger.d.ts +2 -0
- package/dist/lib/crawlee-logger.js +43 -0
- package/dist/lib/crawlee-logger.js.map +1 -0
- package/dist/lib/logger.d.ts +5 -1
- package/dist/lib/logger.js +11 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/types/api.d.ts +5 -0
- package/dist/types/rules.d.ts +18 -10
- package/dist/types/rules.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -66,6 +66,45 @@ async function takeSnapshot() {
|
|
|
66
66
|
takeSnapshot();
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
+
#### Crawling an exact path list
|
|
70
|
+
|
|
71
|
+
Programmatic callers can set `SnapshotConfig.urlPaths` to supply an in-memory
|
|
72
|
+
manifest. It uses the same root-relative path format as `--url-file` and
|
|
73
|
+
switches off link discovery. Only the listed paths are requested; the base
|
|
74
|
+
path `/` must be included explicitly when needed.
|
|
75
|
+
|
|
76
|
+
```javascript
|
|
77
|
+
const snapshotConfig = {
|
|
78
|
+
baseUrl: "https://example.com",
|
|
79
|
+
name: "selected-pages",
|
|
80
|
+
urlPaths: ["/", "/about-me", "/blog/the-nephew-effect"],
|
|
81
|
+
crawlSettings: {
|
|
82
|
+
baseUrl: "https://example.com",
|
|
83
|
+
crawlerType: "cheerio",
|
|
84
|
+
maxConcurrency: 5,
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const result = await createSnapshotFromConfig(snapshotConfig);
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`urlPaths` is validated against `baseUrl`; entries must be root-relative and
|
|
92
|
+
same-origin, and duplicates are removed in first-seen order. An invalid or
|
|
93
|
+
empty manifest fails before crawling. File and stdin reading are CLI concerns;
|
|
94
|
+
the core API receives the already-created array.
|
|
95
|
+
|
|
96
|
+
### Configuring Logs
|
|
97
|
+
|
|
98
|
+
The core package uses pretty Pino logs by default. Call `configureLogger` before
|
|
99
|
+
starting a workflow to select JSON output or a different log level. Crawlee
|
|
100
|
+
messages emitted during snapshotting use the same configured logger.
|
|
101
|
+
|
|
102
|
+
```javascript
|
|
103
|
+
import { configureLogger } from "@cleaver/breakcheck-core";
|
|
104
|
+
|
|
105
|
+
configureLogger({ useJsonLogs: true });
|
|
106
|
+
```
|
|
107
|
+
|
|
69
108
|
### Listing Snapshots
|
|
70
109
|
|
|
71
110
|
You can get a list of all locally saved snapshots using the `listSnapshots` function.
|
|
@@ -84,7 +123,7 @@ async function showSnapshots() {
|
|
|
84
123
|
console.log("Available Snapshots:");
|
|
85
124
|
snapshots.forEach((snapshot) => {
|
|
86
125
|
console.log(
|
|
87
|
-
`- ${snapshot.name} (${snapshot.pageCount} pages, created on ${snapshot.date})
|
|
126
|
+
`- ${snapshot.name} (${snapshot.pageCount} pages, created on ${snapshot.date})`,
|
|
88
127
|
);
|
|
89
128
|
});
|
|
90
129
|
}
|
|
@@ -98,6 +137,8 @@ The core of the tool is comparing two snapshots. The `runComparison` function co
|
|
|
98
137
|
|
|
99
138
|
String `ruleset` values are directories containing `rules.breakcheck`. Relative paths resolve from `process.cwd()`, the directory where the consuming project invokes Breakcheck. In a monorepo, invoke it from the package that owns the rules or use an absolute path.
|
|
100
139
|
|
|
140
|
+
Rules may also be supplied inline as a typed `Ruleset`. `RulesEngine.create()` eagerly validates non-empty selectors and actions, Cheerio selector syntax, region names and uniqueness, supported modifier combinations, and JavaScript regular expressions. It compiles and clones the rules so later caller mutation cannot change processing. The DSL accepts full Cheerio-compatible CSS selectors on a single rule-header line; modifier values are double-quoted, with `\"` and `\\` decoded while regex escapes remain intact. `include` is an intentional compatibility no-op, and `rewrite_content` preserves nested markup by rewriting each descendant text node independently.
|
|
141
|
+
|
|
101
142
|
```javascript
|
|
102
143
|
import { runComparison } from "@cleaver/breakcheck-core";
|
|
103
144
|
|
|
@@ -9,8 +9,8 @@ export async function comparePage(before, after) {
|
|
|
9
9
|
const beautify = jsbeautify.html;
|
|
10
10
|
const beautifyOptions = {
|
|
11
11
|
indent_size: 2,
|
|
12
|
-
preserve_newlines:
|
|
13
|
-
unformatted: ["
|
|
12
|
+
preserve_newlines: false,
|
|
13
|
+
unformatted: ["span", "b", "strong", "i", "em"],
|
|
14
14
|
};
|
|
15
15
|
const beforeContent = beautify(before.content, beautifyOptions);
|
|
16
16
|
const afterContent = beautify(after.content, beautifyOptions);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,KAAK;QACxB,WAAW,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;KAChD,CAAC;IACF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAE9D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAe,SAAS,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9E,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAwB,EACxB,kBAAsC,EACtC,oBAA0C,EAC1C,WAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,sBAAsB;IACtB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,MAAM,CAAC,gBAAgB,CACxB,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACzD,MAAM,CAAC,eAAe,CACvB,CAAC;IAEF,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,IAAI,UAAU,CAAC;IAChD,MAAM,mBAAmB,GACvB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;IAE7E,4BAA4B;IAC5B,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpE,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,2BAA2B;IAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,wCAAwC;QAE3E,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,eAAe,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,sBAAsB,IAAI,qBAAqB,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,MAAM,WAAW,CAC5B,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAClD,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,qBAAqB,EAAE,CACjD,CAAC;YACF,MAAM,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,eAAe,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,kBAAkB,EAAE,CAAC;IAEpE,0BAA0B;IAC1B,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,aAAa,EACX,UAAU,CAAC,QAAQ,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAChE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,mBAAmB;QACnB,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;QACxC,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;QAClD,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,oBAAoB;QAC9D,eAAe;QACf,OAAO;QACP,WAAW;QACX,uBAAuB,EAAE,EAAE;QAC3B,YAAY,EAAE,MAAM,CAAC,cAAc;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC;QACrD,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CrawlerConfig } from "../../../types/api.js";
|
|
2
2
|
import type { CrawlerInstance } from "../../../types/crawler.js";
|
|
3
|
-
export declare function createCheerioCrawler(config: CrawlerConfig): CrawlerInstance;
|
|
3
|
+
export declare function createCheerioCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CheerioCrawler, Dataset } from "crawlee";
|
|
2
|
-
export function createCheerioCrawler(config) {
|
|
2
|
+
export function createCheerioCrawler(config, followLinks = true) {
|
|
3
3
|
const { maxDepth, includePatterns, excludePatterns } = config;
|
|
4
4
|
let datasetPromise = Dataset.open();
|
|
5
5
|
return new CheerioCrawler({
|
|
@@ -23,7 +23,7 @@ export function createCheerioCrawler(config) {
|
|
|
23
23
|
const dataset = await datasetPromise;
|
|
24
24
|
await dataset.pushData(pageSnapshot);
|
|
25
25
|
const depth = request.userData.depth || 0;
|
|
26
|
-
if (depth < (maxDepth || Infinity)) {
|
|
26
|
+
if (followLinks && depth < (maxDepth || Infinity)) {
|
|
27
27
|
await enqueueLinks({
|
|
28
28
|
globs: includePatterns || [],
|
|
29
29
|
exclude: excludePatterns || [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,CAClC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,cAAc,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE;YAC/D,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YACrD,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;gBACrC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;oBACrD,GAAG;oBACH,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;iBACtD,CAAC,CACH;gBACD,KAAK;aACN,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CrawlerConfig } from "../../../types/api.js";
|
|
2
2
|
import type { CrawlerInstance } from "../../../types/crawler.js";
|
|
3
|
-
export declare function createPlaywrightCrawler(config: CrawlerConfig): CrawlerInstance;
|
|
3
|
+
export declare function createPlaywrightCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Dataset, PlaywrightCrawler } from "crawlee";
|
|
2
|
-
export function createPlaywrightCrawler(config) {
|
|
2
|
+
export function createPlaywrightCrawler(config, followLinks = true) {
|
|
3
3
|
const { maxDepth, includePatterns, excludePatterns } = config;
|
|
4
4
|
let datasetPromise = Dataset.open();
|
|
5
5
|
return new PlaywrightCrawler({
|
|
@@ -21,7 +21,7 @@ export function createPlaywrightCrawler(config) {
|
|
|
21
21
|
const dataset = await datasetPromise;
|
|
22
22
|
await dataset.pushData(pageSnapshot);
|
|
23
23
|
const depth = request.userData.depth || 0;
|
|
24
|
-
if (depth < (maxDepth || Infinity)) {
|
|
24
|
+
if (followLinks && depth < (maxDepth || Infinity)) {
|
|
25
25
|
await enqueueLinks({
|
|
26
26
|
globs: includePatterns || [],
|
|
27
27
|
exclude: excludePatterns || [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB;
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,CAAC,GAAG;gBACxC,OAAO;gBACP,KAAK;gBACL,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;gBACnC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;aACnC,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -4,7 +4,12 @@ export declare class BreakcheckCrawler {
|
|
|
4
4
|
private config;
|
|
5
5
|
private crawler;
|
|
6
6
|
private errors;
|
|
7
|
-
|
|
7
|
+
private readonly startUrls;
|
|
8
|
+
private readonly followLinks;
|
|
9
|
+
constructor(config: CrawlerConfig, options?: {
|
|
10
|
+
startUrls?: string[];
|
|
11
|
+
followLinks?: boolean;
|
|
12
|
+
});
|
|
8
13
|
private createCrawler;
|
|
9
14
|
crawl(): Promise<CrawlResult>;
|
|
10
15
|
}
|
|
@@ -6,16 +6,20 @@ export class BreakcheckCrawler {
|
|
|
6
6
|
config;
|
|
7
7
|
crawler;
|
|
8
8
|
errors = [];
|
|
9
|
-
|
|
9
|
+
startUrls;
|
|
10
|
+
followLinks;
|
|
11
|
+
constructor(config, options = {}) {
|
|
10
12
|
this.config = config;
|
|
13
|
+
this.startUrls = options.startUrls ?? [config.baseUrl];
|
|
14
|
+
this.followLinks = options.followLinks ?? true;
|
|
11
15
|
this.crawler = this.createCrawler();
|
|
12
16
|
}
|
|
13
17
|
createCrawler() {
|
|
14
18
|
switch (this.config.crawlerType) {
|
|
15
19
|
case "cheerio":
|
|
16
|
-
return createCheerioCrawler(this.config);
|
|
20
|
+
return createCheerioCrawler(this.config, this.followLinks);
|
|
17
21
|
case "playwright":
|
|
18
|
-
return createPlaywrightCrawler(this.config);
|
|
22
|
+
return createPlaywrightCrawler(this.config, this.followLinks);
|
|
19
23
|
default:
|
|
20
24
|
const _exhaustiveCheck = this.config.crawlerType;
|
|
21
25
|
throw new Error(`Unsupported crawler type: ${this.config.crawlerType}`);
|
|
@@ -25,7 +29,7 @@ export class BreakcheckCrawler {
|
|
|
25
29
|
try {
|
|
26
30
|
await purgeDefaultStorages();
|
|
27
31
|
this.errors = [];
|
|
28
|
-
await this.crawler.run(
|
|
32
|
+
await this.crawler.run(this.startUrls);
|
|
29
33
|
// Open the dataset and scan for error items only
|
|
30
34
|
const dataset = await Dataset.open();
|
|
31
35
|
await dataset.forEach((item) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAkB;IACzB,MAAM,GAAiB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAkB;IACzB,MAAM,GAAiB,EAAE,CAAC;IACjB,SAAS,CAAW;IACpB,WAAW,CAAU;IAEtC,YACE,MAAqB,EACrB,UAA2D,EAAE;QAE7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC;IAEO,aAAa;QACnB,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChC,KAAK,SAAS;gBACZ,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7D,KAAK,YAAY;gBACf,OAAO,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChE;gBACE,MAAM,gBAAgB,GAAU,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,oBAAoB,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,iDAAiD;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;gBAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;wBACf,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,OAAO,EAAE,IAAI,CAAC,KAAK;wBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC5C,IAAI,iBAAiB,GACnB,WAAW,EAAE,IAAI;gBACjB,aAAa,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,iBAAiB,GAAG,EAAE,CAAC;YAC/C,OAAO;gBACL,WAAW,EAAE,iBAAiB;gBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV;gBACE,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,EACD,iBAAiB,CAClB,CAAC;YACF,MAAM,IAAI,KAAK,CACb,oBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type UrlPathIssue = {
|
|
2
|
+
type: "manifest";
|
|
3
|
+
message: string;
|
|
4
|
+
} | {
|
|
5
|
+
type: "base";
|
|
6
|
+
value: string;
|
|
7
|
+
message: string;
|
|
8
|
+
} | {
|
|
9
|
+
type: "entry";
|
|
10
|
+
index: number;
|
|
11
|
+
value: string;
|
|
12
|
+
message: string;
|
|
13
|
+
};
|
|
14
|
+
export interface UrlPathResolution {
|
|
15
|
+
paths: string[];
|
|
16
|
+
issues: UrlPathIssue[];
|
|
17
|
+
}
|
|
18
|
+
export declare function resolveUrlPaths(baseUrl: string, paths: readonly string[]): UrlPathResolution;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
const ROOT_RELATIVE_PATH_MESSAGE = "Path must be root-relative and begin with exactly one '/'";
|
|
2
|
+
function isHttpUrl(url) {
|
|
3
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
4
|
+
}
|
|
5
|
+
function toPath(url) {
|
|
6
|
+
return `${url.pathname}${url.search}${url.hash}`;
|
|
7
|
+
}
|
|
8
|
+
export function resolveUrlPaths(baseUrl, paths) {
|
|
9
|
+
const issues = [];
|
|
10
|
+
const resolvedPaths = [];
|
|
11
|
+
const seenUrls = new Set();
|
|
12
|
+
let base;
|
|
13
|
+
try {
|
|
14
|
+
base = new URL(baseUrl);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return {
|
|
18
|
+
paths: [],
|
|
19
|
+
issues: [
|
|
20
|
+
{
|
|
21
|
+
type: "base",
|
|
22
|
+
value: baseUrl,
|
|
23
|
+
message: "Base URL must be a valid HTTP(S) URL",
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (!isHttpUrl(base)) {
|
|
29
|
+
return {
|
|
30
|
+
paths: [],
|
|
31
|
+
issues: [
|
|
32
|
+
{
|
|
33
|
+
type: "base",
|
|
34
|
+
value: baseUrl,
|
|
35
|
+
message: "Base URL must be a valid HTTP(S) URL",
|
|
36
|
+
},
|
|
37
|
+
],
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (paths.length === 0) {
|
|
41
|
+
issues.push({
|
|
42
|
+
type: "manifest",
|
|
43
|
+
message: "URL manifest must contain at least one path",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
paths.forEach((path, index) => {
|
|
47
|
+
const value = path.trim();
|
|
48
|
+
if (value === "" || !value.startsWith("/") || value.startsWith("//")) {
|
|
49
|
+
issues.push({
|
|
50
|
+
type: "entry",
|
|
51
|
+
index,
|
|
52
|
+
value: path,
|
|
53
|
+
message: value.startsWith("//")
|
|
54
|
+
? "Cross-origin or scheme-relative paths are not allowed"
|
|
55
|
+
: ROOT_RELATIVE_PATH_MESSAGE,
|
|
56
|
+
});
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
if (/\s/.test(value)) {
|
|
60
|
+
issues.push({
|
|
61
|
+
type: "entry",
|
|
62
|
+
index,
|
|
63
|
+
value: path,
|
|
64
|
+
message: "Path must not contain whitespace",
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (value.includes("\\")) {
|
|
69
|
+
issues.push({
|
|
70
|
+
type: "entry",
|
|
71
|
+
index,
|
|
72
|
+
value: path,
|
|
73
|
+
message: ROOT_RELATIVE_PATH_MESSAGE,
|
|
74
|
+
});
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
if (/%(?![0-9A-Fa-f]{2})/.test(value)) {
|
|
78
|
+
issues.push({
|
|
79
|
+
type: "entry",
|
|
80
|
+
index,
|
|
81
|
+
value: path,
|
|
82
|
+
message: "Path contains an invalid percent escape",
|
|
83
|
+
});
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const url = new URL(value, base);
|
|
88
|
+
if (!isHttpUrl(url) || url.origin !== base.origin) {
|
|
89
|
+
issues.push({
|
|
90
|
+
type: "entry",
|
|
91
|
+
index,
|
|
92
|
+
value: path,
|
|
93
|
+
message: "Cross-origin paths are not allowed",
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (!seenUrls.has(url.href)) {
|
|
98
|
+
seenUrls.add(url.href);
|
|
99
|
+
resolvedPaths.push(toPath(url));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
issues.push({
|
|
104
|
+
type: "entry",
|
|
105
|
+
index,
|
|
106
|
+
value: path,
|
|
107
|
+
message: "Path is not a valid URL path",
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
return { paths: resolvedPaths, issues };
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=url-paths.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-paths.js","sourceRoot":"","sources":["../../../src/core/crawler/url-paths.ts"],"names":[],"mappings":"AAsBA,MAAM,0BAA0B,GAC9B,2DAA2D,CAAC;AAE9D,SAAS,SAAS,CAAC,GAAQ;IACzB,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;AAC/D,CAAC;AAED,SAAS,MAAM,CAAC,GAAQ;IACtB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,KAAwB;IAExB,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,6CAA6C;SACvD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE1B,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;oBAC7B,CAAC,CAAC,uDAAuD;oBACzD,CAAC,CAAC,0BAA0B;aAC/B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,kCAAkC;aAC5C,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,0BAA0B;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,KAAK;oBACL,KAAK,EAAE,IAAI;oBACX,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,8BAA8B;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC"}
|