@cleaver/breakcheck-core 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -1
- package/dist/core/compare/index.js +2 -2
- package/dist/core/compare/index.js.map +1 -1
- package/dist/core/crawler/implementations/cheerio.d.ts +1 -1
- package/dist/core/crawler/implementations/cheerio.js +2 -2
- package/dist/core/crawler/implementations/cheerio.js.map +1 -1
- package/dist/core/crawler/implementations/playwright.d.ts +1 -1
- package/dist/core/crawler/implementations/playwright.js +2 -2
- package/dist/core/crawler/implementations/playwright.js.map +1 -1
- package/dist/core/crawler/index.d.ts +6 -1
- package/dist/core/crawler/index.js +8 -4
- package/dist/core/crawler/index.js.map +1 -1
- package/dist/core/crawler/url-paths.d.ts +18 -0
- package/dist/core/crawler/url-paths.js +113 -0
- package/dist/core/crawler/url-paths.js.map +1 -0
- package/dist/core/rules/RulesDsl.js +273 -56
- package/dist/core/rules/RulesDsl.js.map +1 -1
- package/dist/core/rules/RulesEngine.js +56 -17
- package/dist/core/rules/RulesEngine.js.map +1 -1
- package/dist/core/snapshot/index.d.ts +1 -1
- package/dist/core/snapshot/index.js +36 -1
- package/dist/core/snapshot/index.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/lib/crawlee-logger.d.ts +2 -0
- package/dist/lib/crawlee-logger.js +43 -0
- package/dist/lib/crawlee-logger.js.map +1 -0
- package/dist/lib/logger.d.ts +5 -1
- package/dist/lib/logger.js +11 -1
- package/dist/lib/logger.js.map +1 -1
- package/dist/types/api.d.ts +5 -0
- package/dist/types/rules.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -66,6 +66,45 @@ async function takeSnapshot() {
|
|
|
66
66
|
takeSnapshot();
|
|
67
67
|
```
|
|
68
68
|
|
|
69
|
+
#### Crawling an exact path list
|
|
70
|
+
|
|
71
|
+
Programmatic callers can set `SnapshotConfig.urlPaths` to supply an in-memory
|
|
72
|
+
manifest. It uses the same root-relative path format as `--url-file` and
|
|
73
|
+
switches off link discovery. Only the listed paths are requested; the base
|
|
74
|
+
path `/` must be included explicitly when needed.
|
|
75
|
+
|
|
76
|
+
```javascript
|
|
77
|
+
const snapshotConfig = {
|
|
78
|
+
baseUrl: "https://example.com",
|
|
79
|
+
name: "selected-pages",
|
|
80
|
+
urlPaths: ["/", "/about-me", "/blog/the-nephew-effect"],
|
|
81
|
+
crawlSettings: {
|
|
82
|
+
baseUrl: "https://example.com",
|
|
83
|
+
crawlerType: "cheerio",
|
|
84
|
+
maxConcurrency: 5,
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const result = await createSnapshotFromConfig(snapshotConfig);
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`urlPaths` is validated against `baseUrl`; entries must be root-relative and
|
|
92
|
+
same-origin, and duplicates are removed in first-seen order. An invalid or
|
|
93
|
+
empty manifest fails before crawling. File and stdin reading are CLI concerns;
|
|
94
|
+
the core API receives the already-created array.
|
|
95
|
+
|
|
96
|
+
### Configuring Logs
|
|
97
|
+
|
|
98
|
+
The core package uses pretty Pino logs by default. Call `configureLogger` before
|
|
99
|
+
starting a workflow to select JSON output or a different log level. Crawlee
|
|
100
|
+
messages emitted during snapshotting use the same configured logger.
|
|
101
|
+
|
|
102
|
+
```javascript
|
|
103
|
+
import { configureLogger } from "@cleaver/breakcheck-core";
|
|
104
|
+
|
|
105
|
+
configureLogger({ useJsonLogs: true });
|
|
106
|
+
```
|
|
107
|
+
|
|
69
108
|
### Listing Snapshots
|
|
70
109
|
|
|
71
110
|
You can get a list of all locally saved snapshots using the `listSnapshots` function.
|
|
@@ -84,7 +123,7 @@ async function showSnapshots() {
|
|
|
84
123
|
console.log("Available Snapshots:");
|
|
85
124
|
snapshots.forEach((snapshot) => {
|
|
86
125
|
console.log(
|
|
87
|
-
`- ${snapshot.name} (${snapshot.pageCount} pages, created on ${snapshot.date})
|
|
126
|
+
`- ${snapshot.name} (${snapshot.pageCount} pages, created on ${snapshot.date})`,
|
|
88
127
|
);
|
|
89
128
|
});
|
|
90
129
|
}
|
|
@@ -9,8 +9,8 @@ export async function comparePage(before, after) {
|
|
|
9
9
|
const beautify = jsbeautify.html;
|
|
10
10
|
const beautifyOptions = {
|
|
11
11
|
indent_size: 2,
|
|
12
|
-
preserve_newlines:
|
|
13
|
-
unformatted: ["
|
|
12
|
+
preserve_newlines: false,
|
|
13
|
+
unformatted: ["span", "b", "strong", "i", "em"],
|
|
14
14
|
};
|
|
15
15
|
const beforeContent = beautify(before.content, beautifyOptions);
|
|
16
16
|
const afterContent = beautify(after.content, beautifyOptions);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/compare/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,MAAM,CAAC;AAC9C,OAAO,UAAU,MAAM,aAAa,CAAC;AACrC,OAAO,IAAI,MAAM,MAAM,CAAC;AAQxB;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,MAAoB,EACpB,KAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC;IAEvB,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAC;IAEjC,MAAM,eAAe,GAAG;QACtB,WAAW,EAAE,CAAC;QACd,iBAAiB,EAAE,KAAK;QACxB,WAAW,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC;KAChD,CAAC;IACF,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAE9D,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,YAAY,CAAC,CAAC;IAC5D,MAAM,WAAW,GAAe,SAAS,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACvE,MAAM,cAAc,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9E,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAwB,EACxB,kBAAsC,EACtC,oBAA0C,EAC1C,WAAwB;IAExB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,sBAAsB;IACtB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAC1D,MAAM,CAAC,gBAAgB,CACxB,CAAC;IACF,MAAM,aAAa,GAAG,MAAM,kBAAkB,CAAC,YAAY,CACzD,MAAM,CAAC,eAAe,CACvB,CAAC;IAEF,4BAA4B;IAC5B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,IAAI,UAAU,CAAC;IAChD,MAAM,mBAAmB,GACvB,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC;IAE7E,4BAA4B;IAC5B,MAAM,OAAO,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpE,IAAI,eAAe,GAAG,CAAC,CAAC;IAExB,2BAA2B;IAC3B,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS,CAAC,wCAAwC;QAE3E,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9B,eAAe,EAAE,CAAC;YAClB,SAAS;QACX,CAAC;QACD,MAAM,sBAAsB,GAAG,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACvE,MAAM,qBAAqB,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAErE,IAAI,sBAAsB,IAAI,qBAAqB,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,MAAM,WAAW,CAC5B,EAAE,GAAG,UAAU,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAClD,EAAE,GAAG,SAAS,EAAE,OAAO,EAAE,qBAAqB,EAAE,CACjD,CAAC;YACF,MAAM,oBAAoB,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,eAAe,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,QAAQ,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,kBAAkB,EAAE,CAAC;IAEpE,0BAA0B;IAC1B,OAAO;QACL,MAAM,EAAE,WAAW;QACnB,aAAa,EACX,UAAU,CAAC,QAAQ,CAAC,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAChE,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;QACzC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,mBAAmB;QACnB,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,SAAS;QACxC,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,UAAU;QAClD,oBAAoB,EAAE,UAAU,CAAC,QAAQ,CAAC,oBAAoB;QAC9D,eAAe;QACf,OAAO;QACP,WAAW;QACX,uBAAuB,EAAE,EAAE;QAC3B,YAAY,EAAE,MAAM,CAAC,cAAc;QACnC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,eAAe,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,YAAY,CAAC;QACrD,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CrawlerConfig } from "../../../types/api.js";
|
|
2
2
|
import type { CrawlerInstance } from "../../../types/crawler.js";
|
|
3
|
-
export declare function createCheerioCrawler(config: CrawlerConfig): CrawlerInstance;
|
|
3
|
+
export declare function createCheerioCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CheerioCrawler, Dataset } from "crawlee";
|
|
2
|
-
export function createCheerioCrawler(config) {
|
|
2
|
+
export function createCheerioCrawler(config, followLinks = true) {
|
|
3
3
|
const { maxDepth, includePatterns, excludePatterns } = config;
|
|
4
4
|
let datasetPromise = Dataset.open();
|
|
5
5
|
return new CheerioCrawler({
|
|
@@ -23,7 +23,7 @@ export function createCheerioCrawler(config) {
|
|
|
23
23
|
const dataset = await datasetPromise;
|
|
24
24
|
await dataset.pushData(pageSnapshot);
|
|
25
25
|
const depth = request.userData.depth || 0;
|
|
26
|
-
if (depth < (maxDepth || Infinity)) {
|
|
26
|
+
if (followLinks && depth < (maxDepth || Infinity)) {
|
|
27
27
|
await enqueueLinks({
|
|
28
28
|
globs: includePatterns || [],
|
|
29
29
|
exclude: excludePatterns || [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,
|
|
1
|
+
{"version":3,"file":"cheerio.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/cheerio.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAQlD,MAAM,UAAU,oBAAoB,CAClC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,cAAc,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,EAAE,YAAY,EAAE;YAC/D,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YACrD,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;gBACrC,OAAO,EAAE,IAAI,CAAC,QAAQ,EAAE;gBACxB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,CAAC;gBACpC,OAAO,EAAE,MAAM,CAAC,WAAW,CACzB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC;oBACrD,GAAG;oBACH,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE;iBACtD,CAAC,CACH;gBACD,KAAK;aACN,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
1
|
import type { CrawlerConfig } from "../../../types/api.js";
|
|
2
2
|
import type { CrawlerInstance } from "../../../types/crawler.js";
|
|
3
|
-
export declare function createPlaywrightCrawler(config: CrawlerConfig): CrawlerInstance;
|
|
3
|
+
export declare function createPlaywrightCrawler(config: CrawlerConfig, followLinks?: boolean): CrawlerInstance;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Dataset, PlaywrightCrawler } from "crawlee";
|
|
2
|
-
export function createPlaywrightCrawler(config) {
|
|
2
|
+
export function createPlaywrightCrawler(config, followLinks = true) {
|
|
3
3
|
const { maxDepth, includePatterns, excludePatterns } = config;
|
|
4
4
|
let datasetPromise = Dataset.open();
|
|
5
5
|
return new PlaywrightCrawler({
|
|
@@ -21,7 +21,7 @@ export function createPlaywrightCrawler(config) {
|
|
|
21
21
|
const dataset = await datasetPromise;
|
|
22
22
|
await dataset.pushData(pageSnapshot);
|
|
23
23
|
const depth = request.userData.depth || 0;
|
|
24
|
-
if (depth < (maxDepth || Infinity)) {
|
|
24
|
+
if (followLinks && depth < (maxDepth || Infinity)) {
|
|
25
25
|
await enqueueLinks({
|
|
26
26
|
globs: includePatterns || [],
|
|
27
27
|
exclude: excludePatterns || [],
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB;
|
|
1
|
+
{"version":3,"file":"playwright.js","sourceRoot":"","sources":["../../../../src/core/crawler/implementations/playwright.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAQrD,MAAM,UAAU,uBAAuB,CACrC,MAAqB,EACrB,WAAW,GAAG,IAAI;IAElB,MAAM,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAE9D,IAAI,cAAc,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAEpC,OAAO,IAAI,iBAAiB,CAAC;QAC3B,mBAAmB,EAAE,MAAM,CAAC,WAAW;QACvC,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,iBAAiB,EAAE,CAAC;QACpB,yBAAyB,EAAE,EAAE;QAC7B,KAAK,CAAC,cAAc,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACjC,MAAM,YAAY,GAAiB;gBACjC,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,CAAC,GAAG;gBACxC,OAAO;gBACP,KAAK;gBACL,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC;gBACnC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE;aACnC,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YAErC,MAAM,KAAK,GAAI,OAAO,CAAC,QAAQ,CAAC,KAAgB,IAAI,CAAC,CAAC;YACtD,IAAI,WAAW,IAAI,KAAK,GAAG,CAAC,QAAQ,IAAI,QAAQ,CAAC,EAAE,CAAC;gBAClD,MAAM,YAAY,CAAC;oBACjB,KAAK,EAAE,eAAe,IAAI,EAAE;oBAC5B,OAAO,EAAE,eAAe,IAAI,EAAE;iBAC/B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,oBAAoB,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE;YACjD,MAAM,UAAU,GAAe;gBAC7B,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC/D,UAAU,EACR,KAAK,YAAY,KAAK,IAAI,YAAY,IAAI,KAAK;oBAC7C,CAAC,CAAE,KAAK,CAAC,UAAqB;oBAC9B,CAAC,CAAC,SAAS;aAChB,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;YACrC,MAAM,OAAO,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -4,7 +4,12 @@ export declare class BreakcheckCrawler {
|
|
|
4
4
|
private config;
|
|
5
5
|
private crawler;
|
|
6
6
|
private errors;
|
|
7
|
-
|
|
7
|
+
private readonly startUrls;
|
|
8
|
+
private readonly followLinks;
|
|
9
|
+
constructor(config: CrawlerConfig, options?: {
|
|
10
|
+
startUrls?: string[];
|
|
11
|
+
followLinks?: boolean;
|
|
12
|
+
});
|
|
8
13
|
private createCrawler;
|
|
9
14
|
crawl(): Promise<CrawlResult>;
|
|
10
15
|
}
|
|
@@ -6,16 +6,20 @@ export class BreakcheckCrawler {
|
|
|
6
6
|
config;
|
|
7
7
|
crawler;
|
|
8
8
|
errors = [];
|
|
9
|
-
|
|
9
|
+
startUrls;
|
|
10
|
+
followLinks;
|
|
11
|
+
constructor(config, options = {}) {
|
|
10
12
|
this.config = config;
|
|
13
|
+
this.startUrls = options.startUrls ?? [config.baseUrl];
|
|
14
|
+
this.followLinks = options.followLinks ?? true;
|
|
11
15
|
this.crawler = this.createCrawler();
|
|
12
16
|
}
|
|
13
17
|
createCrawler() {
|
|
14
18
|
switch (this.config.crawlerType) {
|
|
15
19
|
case "cheerio":
|
|
16
|
-
return createCheerioCrawler(this.config);
|
|
20
|
+
return createCheerioCrawler(this.config, this.followLinks);
|
|
17
21
|
case "playwright":
|
|
18
|
-
return createPlaywrightCrawler(this.config);
|
|
22
|
+
return createPlaywrightCrawler(this.config, this.followLinks);
|
|
19
23
|
default:
|
|
20
24
|
const _exhaustiveCheck = this.config.crawlerType;
|
|
21
25
|
throw new Error(`Unsupported crawler type: ${this.config.crawlerType}`);
|
|
@@ -25,7 +29,7 @@ export class BreakcheckCrawler {
|
|
|
25
29
|
try {
|
|
26
30
|
await purgeDefaultStorages();
|
|
27
31
|
this.errors = [];
|
|
28
|
-
await this.crawler.run(
|
|
32
|
+
await this.crawler.run(this.startUrls);
|
|
29
33
|
// Open the dataset and scan for error items only
|
|
30
34
|
const dataset = await Dataset.open();
|
|
31
35
|
await dataset.forEach((item) => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAkB;IACzB,MAAM,GAAiB,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/crawler/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AACvE,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAO7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAE1E,MAAM,OAAO,iBAAiB;IACpB,MAAM,CAAgB;IACtB,OAAO,CAAkB;IACzB,MAAM,GAAiB,EAAE,CAAC;IACjB,SAAS,CAAW;IACpB,WAAW,CAAU;IAEtC,YACE,MAAqB,EACrB,UAA2D,EAAE;QAE7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACvD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;IACtC,CAAC;IAEO,aAAa;QACnB,QAAQ,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YAChC,KAAK,SAAS;gBACZ,OAAO,oBAAoB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAC7D,KAAK,YAAY;gBACf,OAAO,uBAAuB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAChE;gBACE,MAAM,gBAAgB,GAAU,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;gBACxD,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,oBAAoB,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvC,iDAAiD;YACjD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,CAAC;YACrC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,IAAS,EAAE,EAAE;gBAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;wBACf,GAAG,EAAE,IAAI,CAAC,GAAG;wBACb,OAAO,EAAE,IAAI,CAAC,KAAK;wBACnB,UAAU,EAAE,IAAI,CAAC,UAAU;qBAC5B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YACH,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YAC5C,IAAI,iBAAiB,GACnB,WAAW,EAAE,IAAI;gBACjB,aAAa,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;YAC1D,IAAI,CAAC,iBAAiB;gBAAE,iBAAiB,GAAG,EAAE,CAAC;YAC/C,OAAO;gBACL,WAAW,EAAE,iBAAiB;gBAC9B,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV;gBACE,KAAK;gBACL,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;gBAC5B,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,EACD,iBAAiB,CAClB,CAAC;YACF,MAAM,IAAI,KAAK,CACb,oBACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type UrlPathIssue = {
|
|
2
|
+
type: "manifest";
|
|
3
|
+
message: string;
|
|
4
|
+
} | {
|
|
5
|
+
type: "base";
|
|
6
|
+
value: string;
|
|
7
|
+
message: string;
|
|
8
|
+
} | {
|
|
9
|
+
type: "entry";
|
|
10
|
+
index: number;
|
|
11
|
+
value: string;
|
|
12
|
+
message: string;
|
|
13
|
+
};
|
|
14
|
+
export interface UrlPathResolution {
|
|
15
|
+
paths: string[];
|
|
16
|
+
issues: UrlPathIssue[];
|
|
17
|
+
}
|
|
18
|
+
export declare function resolveUrlPaths(baseUrl: string, paths: readonly string[]): UrlPathResolution;
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
const ROOT_RELATIVE_PATH_MESSAGE = "Path must be root-relative and begin with exactly one '/'";
|
|
2
|
+
function isHttpUrl(url) {
|
|
3
|
+
return url.protocol === "http:" || url.protocol === "https:";
|
|
4
|
+
}
|
|
5
|
+
function toPath(url) {
|
|
6
|
+
return `${url.pathname}${url.search}${url.hash}`;
|
|
7
|
+
}
|
|
8
|
+
export function resolveUrlPaths(baseUrl, paths) {
|
|
9
|
+
const issues = [];
|
|
10
|
+
const resolvedPaths = [];
|
|
11
|
+
const seenUrls = new Set();
|
|
12
|
+
let base;
|
|
13
|
+
try {
|
|
14
|
+
base = new URL(baseUrl);
|
|
15
|
+
}
|
|
16
|
+
catch {
|
|
17
|
+
return {
|
|
18
|
+
paths: [],
|
|
19
|
+
issues: [
|
|
20
|
+
{
|
|
21
|
+
type: "base",
|
|
22
|
+
value: baseUrl,
|
|
23
|
+
message: "Base URL must be a valid HTTP(S) URL",
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
if (!isHttpUrl(base)) {
|
|
29
|
+
return {
|
|
30
|
+
paths: [],
|
|
31
|
+
issues: [
|
|
32
|
+
{
|
|
33
|
+
type: "base",
|
|
34
|
+
value: baseUrl,
|
|
35
|
+
message: "Base URL must be a valid HTTP(S) URL",
|
|
36
|
+
},
|
|
37
|
+
],
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (paths.length === 0) {
|
|
41
|
+
issues.push({
|
|
42
|
+
type: "manifest",
|
|
43
|
+
message: "URL manifest must contain at least one path",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
paths.forEach((path, index) => {
|
|
47
|
+
const value = path.trim();
|
|
48
|
+
if (value === "" || !value.startsWith("/") || value.startsWith("//")) {
|
|
49
|
+
issues.push({
|
|
50
|
+
type: "entry",
|
|
51
|
+
index,
|
|
52
|
+
value: path,
|
|
53
|
+
message: value.startsWith("//")
|
|
54
|
+
? "Cross-origin or scheme-relative paths are not allowed"
|
|
55
|
+
: ROOT_RELATIVE_PATH_MESSAGE,
|
|
56
|
+
});
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
if (/\s/.test(value)) {
|
|
60
|
+
issues.push({
|
|
61
|
+
type: "entry",
|
|
62
|
+
index,
|
|
63
|
+
value: path,
|
|
64
|
+
message: "Path must not contain whitespace",
|
|
65
|
+
});
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
if (value.includes("\\")) {
|
|
69
|
+
issues.push({
|
|
70
|
+
type: "entry",
|
|
71
|
+
index,
|
|
72
|
+
value: path,
|
|
73
|
+
message: ROOT_RELATIVE_PATH_MESSAGE,
|
|
74
|
+
});
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
if (/%(?![0-9A-Fa-f]{2})/.test(value)) {
|
|
78
|
+
issues.push({
|
|
79
|
+
type: "entry",
|
|
80
|
+
index,
|
|
81
|
+
value: path,
|
|
82
|
+
message: "Path contains an invalid percent escape",
|
|
83
|
+
});
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
const url = new URL(value, base);
|
|
88
|
+
if (!isHttpUrl(url) || url.origin !== base.origin) {
|
|
89
|
+
issues.push({
|
|
90
|
+
type: "entry",
|
|
91
|
+
index,
|
|
92
|
+
value: path,
|
|
93
|
+
message: "Cross-origin paths are not allowed",
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (!seenUrls.has(url.href)) {
|
|
98
|
+
seenUrls.add(url.href);
|
|
99
|
+
resolvedPaths.push(toPath(url));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
issues.push({
|
|
104
|
+
type: "entry",
|
|
105
|
+
index,
|
|
106
|
+
value: path,
|
|
107
|
+
message: "Path is not a valid URL path",
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
});
|
|
111
|
+
return { paths: resolvedPaths, issues };
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=url-paths.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-paths.js","sourceRoot":"","sources":["../../../src/core/crawler/url-paths.ts"],"names":[],"mappings":"AAsBA,MAAM,0BAA0B,GAC9B,2DAA2D,CAAC;AAE9D,SAAS,SAAS,CAAC,GAAQ;IACzB,OAAO,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC;AAC/D,CAAC;AAED,SAAS,MAAM,CAAC,GAAQ;IACtB,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,KAAwB;IAExB,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,MAAM,aAAa,GAAa,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,IAAI,IAAS,CAAC;IACd,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACrB,OAAO;YACL,KAAK,EAAE,EAAE;YACT,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,OAAO;oBACd,OAAO,EAAE,sCAAsC;iBAChD;aACF;SACF,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,6CAA6C;SACvD,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAE1B,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC;oBAC7B,CAAC,CAAC,uDAAuD;oBACzD,CAAC,CAAC,0BAA0B;aAC/B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,kCAAkC;aAC5C,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,0BAA0B;aACpC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,yCAAyC;aACnD,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;gBAClD,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,OAAO;oBACb,KAAK;oBACL,KAAK,EAAE,IAAI;oBACX,OAAO,EAAE,oCAAoC;iBAC9C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACvB,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,KAAK;gBACL,KAAK,EAAE,IAAI;gBACX,OAAO,EAAE,8BAA8B;aACxC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC;AAC1C,CAAC"}
|