@x12i/api-hub-crawl 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/index.d.ts +44 -0
- package/dist/index.js +85 -0
- package/dist/index.js.map +1 -0
- package/package.json +36 -4
package/README.md
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @x12i/api-hub-crawl
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Robots-aware site crawl helpers for the x12i API hub
|
|
4
|
+
|
|
5
|
+
ESM package for Node.js 24 or newer. Import the public name:
|
|
6
|
+
|
|
7
|
+
```js
|
|
8
|
+
import * as crawl from '@x12i/api-hub-crawl';
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
The whole set is also available from [`@x12i/api-hub`](https://www.npmjs.com/package/@x12i/api-hub).
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { type SafeFetch } from '@x12i/api-hub-core';
|
|
2
|
+
import { parseSitemapUrls, type RetrieveFn } from '@x12i/crawl';
|
|
3
|
+
export { parseSitemapUrls };
|
|
4
|
+
export interface CrawlSiteRequest {
|
|
5
|
+
url: string;
|
|
6
|
+
maxPages?: number;
|
|
7
|
+
maxDepth?: number;
|
|
8
|
+
discovery?: 'links' | 'sitemap' | 'both';
|
|
9
|
+
allowSubdomains?: boolean;
|
|
10
|
+
/** Path prefixes or globs, as `@x12i/crawl` scopes them. */
|
|
11
|
+
include?: string[];
|
|
12
|
+
exclude?: string[];
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
signal?: AbortSignal;
|
|
15
|
+
}
|
|
16
|
+
export interface CrawledPage {
|
|
17
|
+
url: string;
|
|
18
|
+
status: 'fetched' | 'skipped' | 'failed' | 'duplicate' | 'queued';
|
|
19
|
+
depth: number;
|
|
20
|
+
httpStatus?: number;
|
|
21
|
+
title?: string;
|
|
22
|
+
reason?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface CrawlSiteResult {
|
|
25
|
+
pages: CrawledPage[];
|
|
26
|
+
counts: {
|
|
27
|
+
fetched: number;
|
|
28
|
+
skipped: number;
|
|
29
|
+
failed: number;
|
|
30
|
+
duplicate: number;
|
|
31
|
+
};
|
|
32
|
+
stoppedBecause: string;
|
|
33
|
+
}
|
|
34
|
+
export declare const CRAWL_LIMITS: {
|
|
35
|
+
maxPages: number;
|
|
36
|
+
maxDepth: number;
|
|
37
|
+
pageBytes: number;
|
|
38
|
+
pageTimeoutMs: number;
|
|
39
|
+
textChars: number;
|
|
40
|
+
};
|
|
41
|
+
/** Retrieval for `@x12i/crawl` built only on `@x12i/api-hub-safe-fetch`: public hosts, robots.txt, size and time caps. */
|
|
42
|
+
export declare function safeRetrieve(fetch: SafeFetch, signal?: AbortSignal): RetrieveFn;
|
|
43
|
+
/** Bounded breadth-first crawl of one site (`@x12i/crawl`), always honouring robots.txt. */
|
|
44
|
+
export declare function crawlSite(fetch: SafeFetch, req: CrawlSiteRequest): Promise<CrawlSiteResult>;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { HubError } from '@x12i/api-hub-core';
|
|
2
|
+
import { crawl, parseSitemapUrls } from '@x12i/crawl';
|
|
3
|
+
import { parseHTML } from 'linkedom';
|
|
4
|
+
export { parseSitemapUrls };
|
|
5
|
+
export const CRAWL_LIMITS = { maxPages: 100, maxDepth: 5, pageBytes: 2_000_000, pageTimeoutMs: 10_000, textChars: 20_000 };
|
|
6
|
+
const HTML = /html|xml/i;
|
|
7
|
+
/** Retrieval for `@x12i/crawl` built only on `@x12i/api-hub-safe-fetch`: public hosts, robots.txt, size and time caps. */
|
|
8
|
+
export function safeRetrieve(fetch, signal) {
|
|
9
|
+
return async (url) => {
|
|
10
|
+
const isRobots = new URL(url).pathname === '/robots.txt';
|
|
11
|
+
const res = await fetch(url, {
|
|
12
|
+
maxBytes: CRAWL_LIMITS.pageBytes,
|
|
13
|
+
truncate: true,
|
|
14
|
+
timeoutMs: CRAWL_LIMITS.pageTimeoutMs,
|
|
15
|
+
maxRedirects: 5,
|
|
16
|
+
respectRobots: !isRobots,
|
|
17
|
+
signal,
|
|
18
|
+
});
|
|
19
|
+
const contentType = res.headers['content-type'] ?? '';
|
|
20
|
+
const fetchedAt = new Date().toISOString();
|
|
21
|
+
const base = { url: res.url, requestedUrl: url, finalUrl: res.url, status: res.status, fetchedAt, html: '' };
|
|
22
|
+
if (res.status >= 400)
|
|
23
|
+
throw new HubError('upstream_unavailable', `HTTP ${res.status}`, { status: res.status });
|
|
24
|
+
const text = res.text();
|
|
25
|
+
if (isRobots || !/text\/html|application\/xhtml/i.test(contentType)) {
|
|
26
|
+
return { ...base, markdown: HTML.test(contentType) || isRobots || /text\/plain/i.test(contentType) ? text : '' };
|
|
27
|
+
}
|
|
28
|
+
const { document } = parseHTML(text);
|
|
29
|
+
const origin = new URL(res.url);
|
|
30
|
+
const links = [];
|
|
31
|
+
for (const a of document.querySelectorAll('a[href]')) {
|
|
32
|
+
let abs;
|
|
33
|
+
try {
|
|
34
|
+
abs = new URL(a.getAttribute('href') ?? '', res.url);
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (abs.protocol !== 'http:' && abs.protocol !== 'https:')
|
|
40
|
+
continue;
|
|
41
|
+
abs.hash = '';
|
|
42
|
+
const sameDomain = abs.hostname === origin.hostname;
|
|
43
|
+
links.push({
|
|
44
|
+
url: abs.href,
|
|
45
|
+
href: a.getAttribute('href') ?? '',
|
|
46
|
+
text: (a.textContent ?? '').trim().slice(0, 200),
|
|
47
|
+
sameDomain,
|
|
48
|
+
classification: sameDomain ? 'navigation' : 'external',
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
const title = document.querySelector('title')?.textContent?.trim() || undefined;
|
|
52
|
+
const body = (document.body?.textContent ?? '').replace(/\s+/g, ' ').trim().slice(0, CRAWL_LIMITS.textChars);
|
|
53
|
+
return { ...base, title, markdown: body, links };
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** Bounded breadth-first crawl of one site (`@x12i/crawl`), always honouring robots.txt. */
|
|
57
|
+
export async function crawlSite(fetch, req) {
|
|
58
|
+
const result = await crawl({
|
|
59
|
+
seeds: [req.url],
|
|
60
|
+
maxPages: Math.min(req.maxPages ?? 20, CRAWL_LIMITS.maxPages),
|
|
61
|
+
maxDepth: Math.min(req.maxDepth ?? 2, CRAWL_LIMITS.maxDepth),
|
|
62
|
+
maxDurationMs: req.timeoutMs ?? 60_000,
|
|
63
|
+
maxTotalBytes: CRAWL_LIMITS.maxPages * CRAWL_LIMITS.pageBytes,
|
|
64
|
+
concurrency: 4,
|
|
65
|
+
perHostDelayMs: 250,
|
|
66
|
+
respectRobots: true,
|
|
67
|
+
discovery: req.discovery ?? 'links',
|
|
68
|
+
scope: { allowSubdomains: req.allowSubdomains ?? false, include: req.include, exclude: req.exclude },
|
|
69
|
+
}, { retrieve: safeRetrieve(fetch, req.signal), signal: req.signal });
|
|
70
|
+
if (result.terminationReason === 'cancelled')
|
|
71
|
+
throw new HubError('timeout', 'The crawl was cancelled');
|
|
72
|
+
return {
|
|
73
|
+
pages: result.pages.map((p) => ({
|
|
74
|
+
url: p.url,
|
|
75
|
+
status: p.status,
|
|
76
|
+
depth: p.depth,
|
|
77
|
+
...(p.result?.status ? { httpStatus: p.result.status } : {}),
|
|
78
|
+
...(p.result?.title ? { title: p.result.title } : {}),
|
|
79
|
+
...(p.reason ? { reason: p.reason } : {}),
|
|
80
|
+
})),
|
|
81
|
+
counts: { fetched: result.counts.fetched, skipped: result.counts.skipped, failed: result.counts.failed, duplicate: result.counts.duplicate },
|
|
82
|
+
stoppedBecause: result.terminationReason,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../packages/crawl/src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAkB,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,KAAK,EAAE,gBAAgB,EAAmB,MAAM,aAAa,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,OAAO,EAAE,gBAAgB,EAAE,CAAC;AA8B5B,MAAM,CAAC,MAAM,YAAY,GAAG,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAE3H,MAAM,IAAI,GAAG,WAAW,CAAC;AAEzB,iHAAiH;AACjH,MAAM,UAAU,YAAY,CAAC,KAAgB,EAAE,MAAoB;IACjE,OAAO,KAAK,EAAE,GAAG,EAAE,EAAE;QACnB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,aAAa,CAAC;QACzD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC3B,QAAQ,EAAE,YAAY,CAAC,SAAS;YAChC,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,YAAY,CAAC,aAAa;YACrC,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC,QAAQ;YACxB,MAAM;SACP,CAAC,CAAC;QACH,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;QAC7G,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM,IAAI,QAAQ,CAAC,sBAAsB,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAChH,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,QAAQ,IAAI,CAAC,gCAAgC,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YACpE,OAAO,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,QAAQ,IAAI,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACnH,CAAC;QACD,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,EAAE,CAAC;QACjB,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;YACrD,IAAI,GAAQ,CAAC;YACb,IAAI,CAAC;gBACH,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC;YACvD,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ;gBAAE,SAAS;YACpE,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,MAAM,UAAU,GAAG,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,QAAQ,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC;gBACT,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClC,IAAI,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBAChD,UAAU;gBACV,cAAc,EAAE,UAAU,CAAC,CAAC,CAAE,YAAsB,CAAC,CAAC,CAAE,UAAoB;aAC7E,CAAC,CAAC;QACL,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QAChF,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,CAAC;QAC7G,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACnD,CAAC,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAgB,EAAE,GAAqB;IACrE,MAAM,MAAM,GAAG,MAAM,KAAK,CACxB;QACE,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC;QAChB,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,EAAE,YAAY,CAAC,QAAQ,CAAC;QAC7D,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,EAAE,YAAY,CAAC,QAAQ,CAAC;QAC5D,aAAa,EAAE,GAAG,CAAC,SAAS,IAAI,MAAM;QACtC,aAAa,EAAE,YAAY,CAAC,QAAQ,GAAG,YAAY,CAAC,SAAS;QAC7D,WAAW,EAAE,CAAC;QACd,cAAc,EAAE,GAAG;QACnB,aAAa,EAAE,IAAI;QACnB,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,OAAO;QACnC,KAAK,EAAE,EAAE,eAAe,EAAE,GAAG,CAAC,eAAe,IAAI,KAAK,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,EAAE;KACrG,EACD,EAAE,QAAQ,EAAE,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAClE,CAAC;IACF,IAAI,MAAM,CAAC,iBAAiB,KAAK,WAAW;QAAE,MAAM,IAAI,QAAQ,CAAC,SAAS,EAAE,yBAAyB,CAAC,CAAC;IACvG,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9B,GAAG,EAAE,CAAC,CAAC,GAAG;YACV,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1C,CAAC,CAAC;QACH,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE;QAC5I,cAAc,EAAE,MAAM,CAAC,iBAAiB;KACzC,CAAC;AACJ,CAAC","sourcesContent":["import { HubError, type SafeFetch } from '@x12i/api-hub-core';\nimport { crawl, parseSitemapUrls, type RetrieveFn } from '@x12i/crawl';\nimport { parseHTML } from 'linkedom';\n\nexport { parseSitemapUrls };\n\nexport interface CrawlSiteRequest {\n url: string;\n maxPages?: number;\n maxDepth?: number;\n discovery?: 'links' | 'sitemap' | 'both';\n allowSubdomains?: boolean;\n /** Path prefixes or globs, as `@x12i/crawl` scopes them. */\n include?: string[];\n exclude?: string[];\n timeoutMs?: number;\n signal?: AbortSignal;\n}\n\nexport interface CrawledPage {\n url: string;\n status: 'fetched' | 'skipped' | 'failed' | 'duplicate' | 'queued';\n depth: number;\n httpStatus?: number;\n title?: string;\n reason?: string;\n}\n\nexport interface CrawlSiteResult {\n pages: CrawledPage[];\n counts: { fetched: number; skipped: number; failed: number; duplicate: number };\n stoppedBecause: string;\n}\n\nexport const CRAWL_LIMITS = { maxPages: 100, maxDepth: 5, pageBytes: 2_000_000, pageTimeoutMs: 10_000, textChars: 20_000 };\n\nconst HTML = /html|xml/i;\n\n/** Retrieval for `@x12i/crawl` built only on `@x12i/api-hub-safe-fetch`: public hosts, robots.txt, size and time caps. */\nexport function safeRetrieve(fetch: SafeFetch, signal?: AbortSignal): RetrieveFn {\n return async (url) => {\n const isRobots = new URL(url).pathname === '/robots.txt';\n const res = await fetch(url, {\n maxBytes: CRAWL_LIMITS.pageBytes,\n truncate: true,\n timeoutMs: CRAWL_LIMITS.pageTimeoutMs,\n maxRedirects: 5,\n respectRobots: !isRobots,\n signal,\n });\n const contentType = res.headers['content-type'] ?? '';\n const fetchedAt = new Date().toISOString();\n const base = { url: res.url, requestedUrl: url, finalUrl: res.url, status: res.status, fetchedAt, html: '' };\n if (res.status >= 400) throw new HubError('upstream_unavailable', `HTTP ${res.status}`, { status: res.status });\n const text = res.text();\n if (isRobots || !/text\\/html|application\\/xhtml/i.test(contentType)) {\n return { ...base, markdown: HTML.test(contentType) || isRobots || /text\\/plain/i.test(contentType) ? text : '' };\n }\n const { document } = parseHTML(text);\n const origin = new URL(res.url);\n const links = [];\n for (const a of document.querySelectorAll('a[href]')) {\n let abs: URL;\n try {\n abs = new URL(a.getAttribute('href') ?? '', res.url);\n } catch {\n continue;\n }\n if (abs.protocol !== 'http:' && abs.protocol !== 'https:') continue;\n abs.hash = '';\n const sameDomain = abs.hostname === origin.hostname;\n links.push({\n url: abs.href,\n href: a.getAttribute('href') ?? '',\n text: (a.textContent ?? '').trim().slice(0, 200),\n sameDomain,\n classification: sameDomain ? ('navigation' as const) : ('external' as const),\n });\n }\n const title = document.querySelector('title')?.textContent?.trim() || undefined;\n const body = (document.body?.textContent ?? '').replace(/\\s+/g, ' ').trim().slice(0, CRAWL_LIMITS.textChars);\n return { ...base, title, markdown: body, links };\n };\n}\n\n/** Bounded breadth-first crawl of one site (`@x12i/crawl`), always honouring robots.txt. */\nexport async function crawlSite(fetch: SafeFetch, req: CrawlSiteRequest): Promise<CrawlSiteResult> {\n const result = await crawl(\n {\n seeds: [req.url],\n maxPages: Math.min(req.maxPages ?? 20, CRAWL_LIMITS.maxPages),\n maxDepth: Math.min(req.maxDepth ?? 2, CRAWL_LIMITS.maxDepth),\n maxDurationMs: req.timeoutMs ?? 60_000,\n maxTotalBytes: CRAWL_LIMITS.maxPages * CRAWL_LIMITS.pageBytes,\n concurrency: 4,\n perHostDelayMs: 250,\n respectRobots: true,\n discovery: req.discovery ?? 'links',\n scope: { allowSubdomains: req.allowSubdomains ?? false, include: req.include, exclude: req.exclude },\n },\n { retrieve: safeRetrieve(fetch, req.signal), signal: req.signal },\n );\n if (result.terminationReason === 'cancelled') throw new HubError('timeout', 'The crawl was cancelled');\n return {\n pages: result.pages.map((p) => ({\n url: p.url,\n status: p.status,\n depth: p.depth,\n ...(p.result?.status ? { httpStatus: p.result.status } : {}),\n ...(p.result?.title ? { title: p.result.title } : {}),\n ...(p.reason ? { reason: p.reason } : {}),\n })),\n counts: { fetched: result.counts.fetched, skipped: result.counts.skipped, failed: result.counts.failed, duplicate: result.counts.duplicate },\n stoppedBecause: result.terminationReason,\n };\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,38 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@x12i/api-hub-crawl",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Robots-aware site crawl helpers for the x12i API hub",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"engines": {
|
|
7
|
+
"node": ">=24"
|
|
8
|
+
},
|
|
9
|
+
"main": "./dist/index.js",
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"exports": {
|
|
12
|
+
".": {
|
|
13
|
+
"types": "./dist/index.d.ts",
|
|
14
|
+
"default": "./dist/index.js"
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
"files": [
|
|
18
|
+
"dist",
|
|
19
|
+
"README.md"
|
|
20
|
+
],
|
|
21
|
+
"repository": {
|
|
22
|
+
"type": "git",
|
|
23
|
+
"url": "git+https://github.com/x12i/apis-hub.git",
|
|
24
|
+
"directory": "packages/crawl"
|
|
25
|
+
},
|
|
26
|
+
"bugs": {
|
|
27
|
+
"url": "https://github.com/x12i/apis-hub/issues"
|
|
28
|
+
},
|
|
29
|
+
"homepage": "https://github.com/x12i/apis-hub",
|
|
30
|
+
"publishConfig": {
|
|
31
|
+
"access": "public"
|
|
32
|
+
},
|
|
33
|
+
"dependencies": {
|
|
34
|
+
"@x12i/api-hub-core": "0.1.1",
|
|
35
|
+
"@x12i/crawl": "0.4.0",
|
|
36
|
+
"linkedom": "0.18.15"
|
|
37
|
+
}
|
|
38
|
+
}
|