@xberg-io/crawlberg 1.12.2 → 1.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +33 -33
- package/package.json +7 -7
package/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:0d4aec5266b8d358067295a8183a3dd48d59d5a2aedf733d0af70004d5e79b88
|
|
3
3
|
// To regenerate: alef generate
|
|
4
4
|
// To verify freshness: alef verify
|
|
5
5
|
/* eslint-disable */
|
|
@@ -877,35 +877,35 @@ export type CrawlEvent =
|
|
|
877
877
|
/** The result of crawling a single page during a crawl operation. */
|
|
878
878
|
export interface CrawlPageResult {
|
|
879
879
|
/** The original URL of the page. */
|
|
880
|
-
readonly url
|
|
880
|
+
readonly url: string;
|
|
881
881
|
/** The normalized URL of the page. */
|
|
882
|
-
readonly normalizedUrl
|
|
882
|
+
readonly normalizedUrl: string;
|
|
883
883
|
/** The HTTP status code of the response. */
|
|
884
|
-
readonly statusCode
|
|
884
|
+
readonly statusCode: number;
|
|
885
885
|
/** The Content-Type header value. */
|
|
886
|
-
readonly contentType
|
|
886
|
+
readonly contentType: string;
|
|
887
887
|
/** The HTML body of the response. */
|
|
888
|
-
readonly html
|
|
888
|
+
readonly html: string;
|
|
889
889
|
/** The size of the response body in bytes. */
|
|
890
|
-
readonly bodySize
|
|
890
|
+
readonly bodySize: number;
|
|
891
891
|
/** Extracted metadata from the page. */
|
|
892
|
-
readonly metadata
|
|
892
|
+
readonly metadata: PageMetadata;
|
|
893
893
|
/** Links found on the page. */
|
|
894
|
-
readonly links
|
|
894
|
+
readonly links: Array<LinkInfo>;
|
|
895
895
|
/** Images found on the page. */
|
|
896
|
-
readonly images
|
|
896
|
+
readonly images: Array<ImageInfo>;
|
|
897
897
|
/** Feed links found on the page. */
|
|
898
|
-
readonly feeds
|
|
898
|
+
readonly feeds: Array<FeedInfo>;
|
|
899
899
|
/** JSON-LD entries found on the page. */
|
|
900
|
-
readonly jsonLd
|
|
900
|
+
readonly jsonLd: Array<JsonLdEntry>;
|
|
901
901
|
/** The depth of this page from the start URL. */
|
|
902
|
-
readonly depth
|
|
902
|
+
readonly depth: number;
|
|
903
903
|
/** Whether this page is on the same domain as the start URL. */
|
|
904
|
-
readonly stayedOnDomain
|
|
904
|
+
readonly stayedOnDomain: boolean;
|
|
905
905
|
/** Whether this page was skipped (binary or PDF content). */
|
|
906
|
-
readonly wasSkipped
|
|
906
|
+
readonly wasSkipped: boolean;
|
|
907
907
|
/** Whether the content is a PDF. */
|
|
908
|
-
readonly isPdf
|
|
908
|
+
readonly isPdf: boolean;
|
|
909
909
|
/** The detected character set encoding. */
|
|
910
910
|
readonly detectedCharset?: string;
|
|
911
911
|
/** Markdown conversion of the page content. */
|
|
@@ -917,30 +917,30 @@ export interface CrawlPageResult {
|
|
|
917
917
|
/** Downloaded non-HTML document (PDF, DOCX, image, code, etc.). */
|
|
918
918
|
readonly downloadedDocument?: DownloadedDocument;
|
|
919
919
|
/** Whether the browser fallback was used to fetch this page. */
|
|
920
|
-
readonly browserUsed
|
|
920
|
+
readonly browserUsed: boolean;
|
|
921
921
|
/**
|
|
922
922
|
* The URL this page's content was actually fetched from, after following any HTTP,
|
|
923
923
|
* `Refresh` header, or `<meta http-equiv="refresh">` redirect `url` pointed at.
|
|
924
924
|
* Equal to `url` when the fetch did not redirect.
|
|
925
925
|
*/
|
|
926
|
-
readonly finalUrl
|
|
926
|
+
readonly finalUrl: string;
|
|
927
927
|
/** Redirect hops taken to reach `final_url` from `url`. */
|
|
928
|
-
readonly redirectCount
|
|
928
|
+
readonly redirectCount: number;
|
|
929
929
|
/**
|
|
930
930
|
* Whether the page asked not to be indexed, by its robots meta tag or `X-Robots-Tag`
|
|
931
931
|
* header. The page is still crawled and its links still followed.
|
|
932
932
|
*/
|
|
933
|
-
readonly noindexDetected
|
|
933
|
+
readonly noindexDetected: boolean;
|
|
934
934
|
/**
|
|
935
935
|
* Whether the page asked that its links not be followed, by its robots meta tag or
|
|
936
936
|
* `X-Robots-Tag` header. When the crawl respects robots, its links are not followed.
|
|
937
937
|
*/
|
|
938
|
-
readonly nofollowDetected
|
|
938
|
+
readonly nofollowDetected: boolean;
|
|
939
939
|
/**
|
|
940
940
|
* URLs of the requests the page sent in browser mode that the SSRF policy refused, without
|
|
941
941
|
* their credentials, each listed once. The page is kept; only the refused requests failed.
|
|
942
942
|
*/
|
|
943
|
-
readonly ssrfRefusedUrls
|
|
943
|
+
readonly ssrfRefusedUrls: Array<string>;
|
|
944
944
|
/**
|
|
945
945
|
* Seed address as supplied, or the discovered link address resolved against its base.
|
|
946
946
|
* Preserves source spelling through redirects; URL credentials are removed. <!-- -->
|
|
@@ -951,21 +951,21 @@ export interface CrawlPageResult {
|
|
|
951
951
|
/** The result of a multi-page crawl operation. */
|
|
952
952
|
export interface CrawlResult {
|
|
953
953
|
/** The list of crawled pages. */
|
|
954
|
-
readonly pages
|
|
954
|
+
readonly pages: Array<CrawlPageResult>;
|
|
955
955
|
/** The final URL after following redirects. */
|
|
956
|
-
readonly finalUrl
|
|
956
|
+
readonly finalUrl: string;
|
|
957
957
|
/** The number of redirects followed. */
|
|
958
|
-
readonly redirectCount
|
|
958
|
+
readonly redirectCount: number;
|
|
959
959
|
/** Whether any page was skipped during crawling. */
|
|
960
|
-
readonly wasSkipped
|
|
960
|
+
readonly wasSkipped: boolean;
|
|
961
961
|
/** An error message, if the crawl encountered an issue. */
|
|
962
962
|
readonly error?: string;
|
|
963
963
|
/** Cookies collected during the crawl. */
|
|
964
|
-
readonly cookies
|
|
964
|
+
readonly cookies: Array<CookieInfo>;
|
|
965
965
|
/** Whether all crawled pages stayed on the same domain as the start URL. */
|
|
966
|
-
readonly stayedOnDomain
|
|
966
|
+
readonly stayedOnDomain: boolean;
|
|
967
967
|
/** Whether the browser fallback was used for any page in this crawl. */
|
|
968
|
-
readonly browserUsed
|
|
968
|
+
readonly browserUsed: boolean;
|
|
969
969
|
/** Details of the first WAF block encountered during the crawl. <!-- --> */
|
|
970
970
|
readonly wafBlock?: WafBlockDetails;
|
|
971
971
|
}
|
|
@@ -1211,15 +1211,15 @@ export interface JsonLdEntry {
|
|
|
1211
1211
|
/** Information about a link found on a page. */
|
|
1212
1212
|
export interface LinkInfo {
|
|
1213
1213
|
/** The resolved URL of the link. */
|
|
1214
|
-
readonly url
|
|
1214
|
+
readonly url: string;
|
|
1215
1215
|
/** The visible text of the link. */
|
|
1216
|
-
readonly text
|
|
1216
|
+
readonly text: string;
|
|
1217
1217
|
/** The classification of the link. */
|
|
1218
|
-
readonly linkType
|
|
1218
|
+
readonly linkType: LinkType;
|
|
1219
1219
|
/** The `rel` attribute value, if present. */
|
|
1220
1220
|
readonly rel?: string;
|
|
1221
1221
|
/** Whether the link has `rel="nofollow"`. */
|
|
1222
|
-
readonly nofollow
|
|
1222
|
+
readonly nofollow: boolean;
|
|
1223
1223
|
/**
|
|
1224
1224
|
* Link address resolved against the page's base without changing path spelling.
|
|
1225
1225
|
* Percent escapes, Unicode, spaces, backslashes and dot segments are preserved.
|
package/package.json
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
}
|
|
11
11
|
],
|
|
12
12
|
"name": "@xberg-io/crawlberg",
|
|
13
|
-
"version": "1.12.
|
|
13
|
+
"version": "1.12.3",
|
|
14
14
|
"description": "High-performance web crawling engine",
|
|
15
15
|
"license": "MIT",
|
|
16
16
|
"repository": {
|
|
@@ -35,12 +35,12 @@
|
|
|
35
35
|
"*.node"
|
|
36
36
|
],
|
|
37
37
|
"optionalDependencies": {
|
|
38
|
-
"@xberg-io/crawlberg-darwin-arm64": "1.12.
|
|
39
|
-
"@xberg-io/crawlberg-darwin-x64": "1.12.
|
|
40
|
-
"@xberg-io/crawlberg-linux-arm64-gnu": "1.12.
|
|
41
|
-
"@xberg-io/crawlberg-linux-x64-gnu": "1.12.
|
|
42
|
-
"@xberg-io/crawlberg-win32-arm64-msvc": "1.12.
|
|
43
|
-
"@xberg-io/crawlberg-win32-x64-msvc": "1.12.
|
|
38
|
+
"@xberg-io/crawlberg-darwin-arm64": "1.12.3",
|
|
39
|
+
"@xberg-io/crawlberg-darwin-x64": "1.12.3",
|
|
40
|
+
"@xberg-io/crawlberg-linux-arm64-gnu": "1.12.3",
|
|
41
|
+
"@xberg-io/crawlberg-linux-x64-gnu": "1.12.3",
|
|
42
|
+
"@xberg-io/crawlberg-win32-arm64-msvc": "1.12.3",
|
|
43
|
+
"@xberg-io/crawlberg-win32-x64-msvc": "1.12.3"
|
|
44
44
|
},
|
|
45
45
|
"napi": {
|
|
46
46
|
"packageName": "@xberg-io/crawlberg",
|