@xapp/arachne-web-service 1.19.0 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +32 -3
- package/lib/ArachneWebService.js +162 -61
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/GenerativeAIService.d.ts +17 -0
- package/lib/GenerativeAIService.js +70 -26
- package/lib/GenerativeAIService.js.map +1 -1
- package/lib/contants.d.ts +13 -1
- package/lib/contants.js +57 -1
- package/lib/contants.js.map +1 -1
- package/lib/models.d.ts +72 -1
- package/lib/models.js +10 -0
- package/lib/models.js.map +1 -1
- package/lib/utils/aboutUs.d.ts +34 -0
- package/lib/utils/aboutUs.js +144 -0
- package/lib/utils/aboutUs.js.map +1 -0
- package/lib/utils/getPrompt.d.ts +8 -0
- package/lib/utils/getPrompt.js +27 -0
- package/lib/utils/getPrompt.js.map +1 -1
- package/lib/utils/getSiteMap.d.ts +34 -1
- package/lib/utils/getSiteMap.js +127 -3
- package/lib/utils/getSiteMap.js.map +1 -1
- package/lib/utils/pagePatterns.d.ts +16 -1
- package/lib/utils/pagePatterns.js +21 -1
- package/lib/utils/pagePatterns.js.map +1 -1
- package/lib/utils/subpageDetection.d.ts +140 -0
- package/lib/utils/subpageDetection.js +259 -0
- package/lib/utils/subpageDetection.js.map +1 -0
- package/package.json +1 -1
|
@@ -42,11 +42,44 @@ export interface SiteMapData {
|
|
|
42
42
|
* A terms of service page, typically found at /terms, /terms-of-service, /tos, /legal/terms, /terms-and-conditions
|
|
43
43
|
*/
|
|
44
44
|
termsOfService?: string;
|
|
45
|
+
/**
|
|
46
|
+
* A page where customers book or request an appointment, typically found at /book-online, /schedule, /appointments,
|
|
47
|
+
* /make-appointment, /request-service, /estimate
|
|
48
|
+
*/
|
|
49
|
+
bookingPage?: string;
|
|
50
|
+
/**
|
|
51
|
+
* Service slugs read from the sitemap URLs, e.g. "water-heater-repair" from /services/water-heater-repair
|
|
52
|
+
* or "drain-cleaning" from /drain-cleaning. No page is loaded to find them.
|
|
53
|
+
*
|
|
54
|
+
* Only present when requested, see `dataOptions.serviceSlugsFromSitemap`.
|
|
55
|
+
*/
|
|
56
|
+
serviceSlugs?: string[];
|
|
57
|
+
}
|
|
58
|
+
export interface SiteMapDataOptions {
|
|
59
|
+
/**
|
|
60
|
+
* Add `serviceSlugs` read from the sitemap URLs.
|
|
61
|
+
*/
|
|
62
|
+
serviceSlugs?: boolean;
|
|
45
63
|
}
|
|
64
|
+
/** Most service slugs returned, so a sitemap full of location pages cannot bloat the result. */
|
|
65
|
+
export declare const MAX_SERVICE_SLUGS = 100;
|
|
66
|
+
/**
|
|
67
|
+
* Reads the services a business names in its URLs, without loading any page.
|
|
68
|
+
*
|
|
69
|
+
* Two shapes are recognised:
|
|
70
|
+
* - every segment under a services section: /services/water-heater-repair, /our-services/plumbing/leak-detection
|
|
71
|
+
* - a single top-level segment containing a service word: /drain-cleaning, /ac-repair.html, /hvac-services
|
|
72
|
+
*
|
|
73
|
+
* A bare top-level slug such as /plumbing is not taken, since it cannot be told apart from /gallery.
|
|
74
|
+
*
|
|
75
|
+
* @param urls URLs, typically every URL in the sitemap
|
|
76
|
+
* @returns Unique lowercase slugs in first-seen order, at most MAX_SERVICE_SLUGS
|
|
77
|
+
*/
|
|
78
|
+
export declare function extractServiceSlugs(urls: string[]): string[];
|
|
46
79
|
/**
|
|
47
80
|
* Returns descriptive data about a sitemap.
|
|
48
81
|
*
|
|
49
82
|
* @param url
|
|
50
83
|
* @returns
|
|
51
84
|
*/
|
|
52
|
-
export declare function getSiteMapData(url: string): Promise<SiteMapData | undefined>;
|
|
85
|
+
export declare function getSiteMapData(url: string, options?: SiteMapDataOptions): Promise<SiteMapData | undefined>;
|
package/lib/utils/getSiteMap.js
CHANGED
|
@@ -42,7 +42,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
42
42
|
});
|
|
43
43
|
};
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.MAX_SERVICE_SLUGS = void 0;
|
|
45
46
|
exports.getSitemap = getSitemap;
|
|
47
|
+
exports.extractServiceSlugs = extractServiceSlugs;
|
|
46
48
|
exports.getSiteMapData = getSiteMapData;
|
|
47
49
|
/*! Copyright (c) 2023, XAPP AI */
|
|
48
50
|
const cheerio = __importStar(require("cheerio"));
|
|
@@ -139,14 +141,123 @@ function crawlSitemap(sitemapUrl, visited) {
|
|
|
139
141
|
return allUrls;
|
|
140
142
|
});
|
|
141
143
|
}
|
|
144
|
+
/** Most service slugs returned, so a sitemap full of location pages cannot bloat the result. */
|
|
145
|
+
exports.MAX_SERVICE_SLUGS = 100;
|
|
146
|
+
/** A path segment under which every following segment names a service. */
|
|
147
|
+
const SERVICE_PARENT_SEGMENT = /^((our|residential|commercial)-)?services?$|^what-we-do$/;
|
|
148
|
+
/** Words that mark a top-level slug as naming a service, e.g. /drain-cleaning, /ac-repair, /hvac-services. */
|
|
149
|
+
const SERVICE_WORDS = new Set([
|
|
150
|
+
"repair",
|
|
151
|
+
"repairs",
|
|
152
|
+
"installation",
|
|
153
|
+
"installations",
|
|
154
|
+
"install",
|
|
155
|
+
"replacement",
|
|
156
|
+
"replacements",
|
|
157
|
+
"cleaning",
|
|
158
|
+
"maintenance",
|
|
159
|
+
"inspection",
|
|
160
|
+
"inspections",
|
|
161
|
+
"removal",
|
|
162
|
+
"restoration",
|
|
163
|
+
"remodeling",
|
|
164
|
+
"renovation",
|
|
165
|
+
"treatment",
|
|
166
|
+
"service",
|
|
167
|
+
"services",
|
|
168
|
+
]);
|
|
169
|
+
/** Words that mark a slug as something other than a service, e.g. /customer-service, /service-areas. */
|
|
170
|
+
const NON_SERVICE_WORDS = new Set(["area", "areas", "customer", "terms", "privacy", "policy", "careers", "jobs"]);
|
|
171
|
+
/** Slugs that never name a service on their own. */
|
|
172
|
+
const NON_SERVICE_SLUGS = new Set(["page", "all", "residential", "commercial", "category", "tag"]);
|
|
173
|
+
/** A path containing any of these segments is content or taxonomy, not a service page. */
|
|
174
|
+
const NON_SERVICE_SECTIONS = new Set([
|
|
175
|
+
"blog",
|
|
176
|
+
"news",
|
|
177
|
+
"article",
|
|
178
|
+
"articles",
|
|
179
|
+
"post",
|
|
180
|
+
"posts",
|
|
181
|
+
"category",
|
|
182
|
+
"categories",
|
|
183
|
+
"tag",
|
|
184
|
+
"tags",
|
|
185
|
+
"author",
|
|
186
|
+
"wp-content",
|
|
187
|
+
"feed",
|
|
188
|
+
"careers",
|
|
189
|
+
"jobs",
|
|
190
|
+
"events",
|
|
191
|
+
"press",
|
|
192
|
+
]);
|
|
193
|
+
const SLUG = /^[a-z0-9]+(-[a-z0-9]+)*$/;
|
|
194
|
+
function isServiceSlug(slug) {
|
|
195
|
+
return SLUG.test(slug) && /[a-z]/.test(slug) && slug.length <= 80 && !NON_SERVICE_SLUGS.has(slug);
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* Reads the services a business names in its URLs, without loading any page.
|
|
199
|
+
*
|
|
200
|
+
* Two shapes are recognised:
|
|
201
|
+
* - every segment under a services section: /services/water-heater-repair, /our-services/plumbing/leak-detection
|
|
202
|
+
* - a single top-level segment containing a service word: /drain-cleaning, /ac-repair.html, /hvac-services
|
|
203
|
+
*
|
|
204
|
+
* A bare top-level slug such as /plumbing is not taken, since it cannot be told apart from /gallery.
|
|
205
|
+
*
|
|
206
|
+
* @param urls URLs, typically every URL in the sitemap
|
|
207
|
+
* @returns Unique lowercase slugs in first-seen order, at most MAX_SERVICE_SLUGS
|
|
208
|
+
*/
|
|
209
|
+
function extractServiceSlugs(urls) {
|
|
210
|
+
const slugs = new Set();
|
|
211
|
+
for (const value of urls) {
|
|
212
|
+
if (slugs.size >= exports.MAX_SERVICE_SLUGS)
|
|
213
|
+
break;
|
|
214
|
+
let pathname;
|
|
215
|
+
try {
|
|
216
|
+
pathname = new URL(value).pathname;
|
|
217
|
+
}
|
|
218
|
+
catch (_a) {
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
const segments = pathname
|
|
222
|
+
.toLowerCase()
|
|
223
|
+
.split("/")
|
|
224
|
+
.filter((segment) => segment.length > 0)
|
|
225
|
+
.map((segment) => segment.replace(/\.(html?|php|aspx?)$/, ""));
|
|
226
|
+
if (segments.some((segment) => NON_SERVICE_SECTIONS.has(segment)))
|
|
227
|
+
continue;
|
|
228
|
+
const candidates = [];
|
|
229
|
+
const parent = segments.findIndex((segment) => SERVICE_PARENT_SEGMENT.test(segment));
|
|
230
|
+
if (parent >= 0) {
|
|
231
|
+
const rest = segments.slice(parent + 1);
|
|
232
|
+
// pagination of the services index, e.g. /services/page/2
|
|
233
|
+
if (rest[0] === "page")
|
|
234
|
+
continue;
|
|
235
|
+
candidates.push(...rest);
|
|
236
|
+
}
|
|
237
|
+
else if (segments.length === 1) {
|
|
238
|
+
const words = segments[0].split("-");
|
|
239
|
+
if (words.some((word) => SERVICE_WORDS.has(word)) && !words.some((word) => NON_SERVICE_WORDS.has(word))) {
|
|
240
|
+
candidates.push(segments[0]);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
for (const candidate of candidates) {
|
|
244
|
+
if (isServiceSlug(candidate) && !SERVICE_PARENT_SEGMENT.test(candidate)) {
|
|
245
|
+
slugs.add(candidate);
|
|
246
|
+
if (slugs.size >= exports.MAX_SERVICE_SLUGS)
|
|
247
|
+
break;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return [...slugs];
|
|
252
|
+
}
|
|
142
253
|
/**
|
|
143
254
|
* Returns descriptive data about a sitemap.
|
|
144
255
|
*
|
|
145
256
|
* @param url
|
|
146
257
|
* @returns
|
|
147
258
|
*/
|
|
148
|
-
function getSiteMapData(
|
|
149
|
-
return __awaiter(this,
|
|
259
|
+
function getSiteMapData(url_1) {
|
|
260
|
+
return __awaiter(this, arguments, void 0, function* (url, options = {}) {
|
|
150
261
|
let siteUrl;
|
|
151
262
|
let originalPath;
|
|
152
263
|
let sitemapUrl;
|
|
@@ -233,7 +344,14 @@ function getSiteMapData(url) {
|
|
|
233
344
|
if (termsOfService) {
|
|
234
345
|
(0, stentor_logger_1.log)().debug(`Found terms of service page at ${termsOfService}`);
|
|
235
346
|
}
|
|
236
|
-
|
|
347
|
+
const bookingPage = filtered.find((value) => {
|
|
348
|
+
const parsed = new URL(value);
|
|
349
|
+
return pagePatterns_1.BOOKING_PAGE_PATH_REGEX.test(parsed.pathname);
|
|
350
|
+
});
|
|
351
|
+
if (bookingPage) {
|
|
352
|
+
(0, stentor_logger_1.log)().debug(`Found booking page at ${bookingPage}`);
|
|
353
|
+
}
|
|
354
|
+
const data = {
|
|
237
355
|
siteMapUrl: sitemapUrl.toString(),
|
|
238
356
|
totalPages: filtered.length,
|
|
239
357
|
aboutUs,
|
|
@@ -242,7 +360,13 @@ function getSiteMapData(url) {
|
|
|
242
360
|
paymentPage,
|
|
243
361
|
privacyPolicy,
|
|
244
362
|
termsOfService,
|
|
363
|
+
bookingPage,
|
|
245
364
|
};
|
|
365
|
+
if (options.serviceSlugs) {
|
|
366
|
+
data.serviceSlugs = extractServiceSlugs(filtered);
|
|
367
|
+
(0, stentor_logger_1.log)().debug(`Found ${data.serviceSlugs.length} service slugs in the sitemap`);
|
|
368
|
+
}
|
|
369
|
+
return data;
|
|
246
370
|
});
|
|
247
371
|
}
|
|
248
372
|
//# sourceMappingURL=getSiteMap.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getSiteMap.js","sourceRoot":"","sources":["../../src/utils/getSiteMap.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"getSiteMap.js","sourceRoot":"","sources":["../../src/utils/getSiteMap.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA2DA,gCAEC;AA2LD,kDA6CC;AAQD,wCAsIC;AAnbD,kCAAkC;AAClC,iDAAmC;AAEnC,2CAA6C;AAC7C,mDAAqC;AAErC,iDAQwB;AAExB,MAAM,YAAY,GAAG,CAAO,GAAW,EAAmB,EAAE;IACxD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;IAE9B,MAAM,OAAO,GAAG,IAAI,CAAC;IAErB,wFAAwF;IACxF,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;QAC9B,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,WAAW,OAAO,+BAA+B,GAAG,EAAE,CAAC,CAAC;QACpE,UAAU,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC,EAAE,OAAO,CAAC,CAAC;IAEZ,aAAa;IACb,IAAI,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAAC;aACpC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;YACf,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAChF,CAAC;YAED,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACV,YAAY,CAAC,SAAS,CAAC,CAAC;QAC5B,CAAC,CAAC,CAAC;QAEP,OAAO,IAAI,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,CAAC;IACd,CAAC;AACL,CAAC,CAAA,CAAC;AAEF;;;;;;;;;GASG;AACH,SAAsB,UAAU,CAAC,UAAkB;;QAC/C,OAAO,YAAY,CAAC,UAAU,EAAE,IAAI,GAAG,EAAU,CAAC,CAAC;IACvD,CAAC;CAAA;AAED;;;;;;;;;;;GAWG;AACH,SAAe,YAAY,CAAC,UAAkB,EAAE,OAAoB;;QAChE,MAAM,oBAAoB,GAAG,IAAA,sBAAY,EAAC,UAAU,CAAC,CAAC;QAEtD,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,CAAC;YACpC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,2BAA2B,oBAAoB,aAAa,CAAC,CAAC;YAC1E,OAAO,EAAE,CAAC;QACd,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAC;QAElC,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,oBAAoB,CAAC,CAAC;QAEzD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YAC5B,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,OAAO,GAAa,EAAE,CAAC;QAE3B,mBAAmB;QACnB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACxC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAE5C,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAE9C,IAAI,CAAC,GAAG,EAAE,CAAC;gBACP,SAAS;YACb,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CAAA;AAwDD,gGAAgG;AACnF,QAAA,iBAAiB,GAAG,GAAG,CAAC;AAErC,0EAA0E;AAC1E,MAAM,sBAAsB,GAAG,0DAA0D,CAAC;AAE1F,8GAA8G;AAC9G,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC;IAC1B,QAAQ;IACR,SAAS;IACT,cAAc;IACd,eAAe;IACf,SAAS;IACT,aAAa;IACb,cAAc;IACd,UAAU;IACV,aAAa;IACb,YAAY;IACZ,aAAa;IACb,SAAS;IACT,aAAa;IACb,YAAY;IACZ,YAAY;IACZ,WAAW;IACX,SAAS;IACT,UAAU;CACb,CAAC,CAAC;AAEH,wGAAwG;AACxG,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC;AAElH,oDAAoD;AACpD,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;AAEnG,0FAA0F;AAC1F,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACjC,MAAM;IACN,MAAM;IACN,SAAS;IACT,UAAU;IACV,MAAM;IACN,OAAO;IACP,UAAU;IACV,YAAY;IACZ,KAAK;IACL,MAAM;IACN,QAAQ;IACR,YAAY;IACZ,MAAM;IACN,SAAS;IACT,MAAM;IACN,QAAQ;IACR,OAAO;CACV,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,0BAA0B,CAAC;AAExC,SAAS,aAAa,CAAC,IAAY;IAC/B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,EAAE,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACtG,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAgB,mBAAmB,CAAC,IAAc;IAC9C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,KAAK,CAAC,IAAI,IAAI,yBAAiB;YAAE,MAAM;QAE3C,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACD,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC;QACvC,CAAC;QAAC,WAAM,CAAC;YACL,SAAS;QACb,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ;aACpB,WAAW,EAAE;aACb,KAAK,CAAC,GAAG,CAAC;aACV,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;aACvC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,CAAC;QAEnE,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAAE,SAAS;QAE5E,MAAM,UAAU,GAAa,EAAE,CAAC;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,sBAAsB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAErF,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACxC,0DAA0D;YAC1D,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,MAAM;gBAAE,SAAS;YACjC,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACtG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACjC,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACtE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBACrB,IAAI,KAAK,CAAC,IAAI,IAAI,yBAAiB;oBAAE,MAAM;YAC/C,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,SAAsB,cAAc;yDAAC,GAAW,EAAE,UAA8B,EAAE;QAC9E,IAAI,OAAY,CAAC;QACjB,IAAI,YAAoB,CAAC;QACzB,IAAI,UAAe,CAAC;QAEpB,IAAI,CAAC;YACD,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEhC,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1B,UAAU,CAAC,QAAQ,GAAG,GAAG,YAAY,aAAa,CAAC;QACvD,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,GAAG,GAAG,CAAC,CAAC;YACxD,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,GAAG,GAAG,CAAC,CAAC;YACxD,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,sBAAsB,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;QAExD,QAAQ;QACR,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAClC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChC,UAAU,CAAC,QAAQ,GAAG,KAAK,CAAC;gBAC5B,OAAO,UAAU,CAAC,QAAQ,EAAE,CAAC;YACjC,CAAC;iBAAM,CAAC;gBACJ,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,wFAAwF;QACxF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;YACtC,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;gBACjC,OAAO,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;YACnD,CAAC;YACD,OAAO,KAAK,CAAC;QACjB,CAAC,CAAC,CAAC;QAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,eAAe,OAAO,CAAC,MAAM,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAEnF,qBAAqB;QACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,kCAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,CAAC;YACV,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,0BAA0B,OAAO,EAAE,CAAC,CAAC;QACrD,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,oCAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QAEH,IAAI,SAAS,EAAE,CAAC;YACZ,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,4BAA4B,SAAS,EAAE,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,kCAAmB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,IAAI,OAAO,EAAE,CAAC;YACV,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,sCAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,EAAE,CAAC;YACd,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YAC1C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,wCAAyB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAChB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,gCAAgC,aAAa,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,cAAc,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,0CAA2B,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC;QAEH,IAAI,cAAc,EAAE,CAAC;YACjB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,cAAc,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACxC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;YAC9B,OAAO,sCAAuB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QAEH,IAAI,WAAW,EAAE,CAAC;YACd,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,yBAAyB,WAAW,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,IAAI,GAAgB;YACtB,UAAU,EAAE,UAAU,CAAC,QAAQ,EAAE;YAEjC,UAAU,EAAE,QAAQ,CAAC,MAAM;YAE3B,OAAO;YACP,SAAS;YACT,OAAO;YACP,WAAW;YACX,aAAa;YACb,cAAc;YACd,WAAW;SACd,CAAC;QAEF,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACvB,IAAI,CAAC,YAAY,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;YAClD,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,SAAS,IAAI,CAAC,YAAY,CAAC,MAAM,+BAA+B,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;CAAA"}
|
|
@@ -10,7 +10,7 @@ export interface PagePattern {
|
|
|
10
10
|
/** Regex to match link text */
|
|
11
11
|
textRegex: RegExp;
|
|
12
12
|
}
|
|
13
|
-
export type PageType = "aboutUs" | "contactUs" | "faqPage" | "paymentPage" | "privacyPolicy" | "termsOfService";
|
|
13
|
+
export type PageType = "aboutUs" | "contactUs" | "faqPage" | "paymentPage" | "privacyPolicy" | "termsOfService" | "bookingPage";
|
|
14
14
|
/**
|
|
15
15
|
* About Us page pattern
|
|
16
16
|
* Matches: /about, /about-us, /company/about, /who-we-are, /our-story, /info/about, /en/about, /about/our-company
|
|
@@ -47,6 +47,21 @@ export declare const PRIVACY_POLICY_TEXT_REGEX: RegExp;
|
|
|
47
47
|
*/
|
|
48
48
|
export declare const TERMS_OF_SERVICE_PATH_REGEX: RegExp;
|
|
49
49
|
export declare const TERMS_OF_SERVICE_TEXT_REGEX: RegExp;
|
|
50
|
+
/**
|
|
51
|
+
* Booking page pattern, where a business embeds its scheduler (Demandforce, Housecall Pro, ServiceTitan, Jobber...)
|
|
52
|
+
* Matches: /book, /book-online, /book-now, /booking, /schedule, /schedule-service, /schedule-online,
|
|
53
|
+
* /schedule-appointment, /appointment(s), /make-appointment(s), /make-an-appointment, /request-appointment,
|
|
54
|
+
* /request-an-appointment, /request-service, /estimate, /free-estimate
|
|
55
|
+
*
|
|
56
|
+
* Only the last path segment is matched, so a blog post such as /blog/how-to-book-a-plumber is not a booking page.
|
|
57
|
+
*/
|
|
58
|
+
export declare const BOOKING_PAGE_PATH_REGEX: RegExp;
|
|
59
|
+
/**
|
|
60
|
+
* Link text for a booking page: "Book Online", "Schedule Service", "Request an Appointment", "Make an Appointment"...
|
|
61
|
+
*
|
|
62
|
+
* Anchored to the whole link text, so a headline such as "How to book a plumber" does not match.
|
|
63
|
+
*/
|
|
64
|
+
export declare const BOOKING_PAGE_TEXT_REGEX: RegExp;
|
|
50
65
|
/**
|
|
51
66
|
* All page patterns in a single array for iteration
|
|
52
67
|
*/
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/*! Copyright (c) 2026, XAPP AI */
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.PAGE_KEYS = exports.PAGE_PATTERNS = exports.TERMS_OF_SERVICE_TEXT_REGEX = exports.TERMS_OF_SERVICE_PATH_REGEX = exports.PRIVACY_POLICY_TEXT_REGEX = exports.PRIVACY_POLICY_PATH_REGEX = exports.PAYMENT_PAGE_TEXT_REGEX = exports.PAYMENT_PAGE_PATH_REGEX = exports.FAQ_PAGE_TEXT_REGEX = exports.FAQ_PAGE_PATH_REGEX = exports.CONTACT_US_TEXT_REGEX = exports.CONTACT_US_PATH_REGEX = exports.ABOUT_US_TEXT_REGEX = exports.ABOUT_US_PATH_REGEX = void 0;
|
|
4
|
+
exports.PAGE_KEYS = exports.PAGE_PATTERNS = exports.BOOKING_PAGE_TEXT_REGEX = exports.BOOKING_PAGE_PATH_REGEX = exports.TERMS_OF_SERVICE_TEXT_REGEX = exports.TERMS_OF_SERVICE_PATH_REGEX = exports.PRIVACY_POLICY_TEXT_REGEX = exports.PRIVACY_POLICY_PATH_REGEX = exports.PAYMENT_PAGE_TEXT_REGEX = exports.PAYMENT_PAGE_PATH_REGEX = exports.FAQ_PAGE_TEXT_REGEX = exports.FAQ_PAGE_PATH_REGEX = exports.CONTACT_US_TEXT_REGEX = exports.CONTACT_US_PATH_REGEX = exports.ABOUT_US_TEXT_REGEX = exports.ABOUT_US_PATH_REGEX = void 0;
|
|
5
5
|
/**
|
|
6
6
|
* About Us page pattern
|
|
7
7
|
* Matches: /about, /about-us, /company/about, /who-we-are, /our-story, /info/about, /en/about, /about/our-company
|
|
@@ -38,6 +38,21 @@ exports.PRIVACY_POLICY_TEXT_REGEX = /^privacy(\s+policy)?$/i;
|
|
|
38
38
|
*/
|
|
39
39
|
exports.TERMS_OF_SERVICE_PATH_REGEX = /\/(terms(-of-(service|use))?|tos|legal\/terms|terms-and-conditions)(\/)?$/i;
|
|
40
40
|
exports.TERMS_OF_SERVICE_TEXT_REGEX = /^(terms(\s+(of\s+(service|use)|and\s+conditions)?)?|tos)$/i;
|
|
41
|
+
/**
|
|
42
|
+
* Booking page pattern, where a business embeds its scheduler (Demandforce, Housecall Pro, ServiceTitan, Jobber...)
|
|
43
|
+
* Matches: /book, /book-online, /book-now, /booking, /schedule, /schedule-service, /schedule-online,
|
|
44
|
+
* /schedule-appointment, /appointment(s), /make-appointment(s), /make-an-appointment, /request-appointment,
|
|
45
|
+
* /request-an-appointment, /request-service, /estimate, /free-estimate
|
|
46
|
+
*
|
|
47
|
+
* Only the last path segment is matched, so a blog post such as /blog/how-to-book-a-plumber is not a booking page.
|
|
48
|
+
*/
|
|
49
|
+
exports.BOOKING_PAGE_PATH_REGEX = /\/(book(-online|-now)?|booking|schedule(-service|-online|-appointment|-an-appointment)?|appointments?|make-(an-)?appointments?|request-(an-)?appointment|request-service|(free-)?estimate)(\/)?$/i;
|
|
50
|
+
/**
|
|
51
|
+
* Link text for a booking page: "Book Online", "Schedule Service", "Request an Appointment", "Make an Appointment"...
|
|
52
|
+
*
|
|
53
|
+
* Anchored to the whole link text, so a headline such as "How to book a plumber" does not match.
|
|
54
|
+
*/
|
|
55
|
+
exports.BOOKING_PAGE_TEXT_REGEX = /^((book|schedule)(\s+(online|now|(an?\s+)?(appointment|service|visit|consultation)))?|request\s+(an?\s+)?(appointment|service|visit|consultation|estimate)|make\s+an?\s+appointment|free\s+estimate)(\s+(online|now|today))?$/i;
|
|
41
56
|
/**
|
|
42
57
|
* All page patterns in a single array for iteration
|
|
43
58
|
*/
|
|
@@ -72,6 +87,11 @@ exports.PAGE_PATTERNS = [
|
|
|
72
87
|
pathRegex: exports.TERMS_OF_SERVICE_PATH_REGEX,
|
|
73
88
|
textRegex: exports.TERMS_OF_SERVICE_TEXT_REGEX,
|
|
74
89
|
},
|
|
90
|
+
{
|
|
91
|
+
key: "bookingPage",
|
|
92
|
+
pathRegex: exports.BOOKING_PAGE_PATH_REGEX,
|
|
93
|
+
textRegex: exports.BOOKING_PAGE_TEXT_REGEX,
|
|
94
|
+
},
|
|
75
95
|
];
|
|
76
96
|
/**
|
|
77
97
|
* All page type keys, derived from PAGE_PATTERNS to maintain single source of truth
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pagePatterns.js","sourceRoot":"","sources":["../../src/utils/pagePatterns.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;
|
|
1
|
+
{"version":3,"file":"pagePatterns.js","sourceRoot":"","sources":["../../src/utils/pagePatterns.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;AAwBlC;;;GAGG;AACU,QAAA,mBAAmB,GAC5B,qGAAqG,CAAC;AAC7F,QAAA,mBAAmB,GAAG,kBAAkB,CAAC;AAEtD;;;GAGG;AACU,QAAA,qBAAqB,GAC9B,oEAAoE,CAAC;AAC5D,QAAA,qBAAqB,GAAG,oBAAoB,CAAC;AAE1D;;;GAGG;AACU,QAAA,mBAAmB,GAAG,0DAA0D,CAAC;AACjF,QAAA,mBAAmB,GAAG,+CAA+C,CAAC;AAEnF;;;GAGG;AACU,QAAA,uBAAuB,GAAG,+DAA+D,CAAC;AAC1F,QAAA,uBAAuB,GAAG,uDAAuD,CAAC;AAE/F;;;GAGG;AACU,QAAA,yBAAyB,GAAG,+DAA+D,CAAC;AAC5F,QAAA,yBAAyB,GAAG,wBAAwB,CAAC;AAElE;;;GAGG;AACU,QAAA,2BAA2B,GACpC,4EAA4E,CAAC;AACpE,QAAA,2BAA2B,GAAG,4DAA4D,CAAC;AAExG;;;;;;;GAOG;AACU,QAAA,uBAAuB,GAChC,mMAAmM,CAAC;AACxM;;;;GAIG;AACU,QAAA,uBAAuB,GAChC,gOAAgO,CAAC;AAErO;;GAEG;AACU,QAAA,aAAa,GAAkB;IACxC;QACI,GAAG,EAAE,SAAS;QACd,SAAS,EAAE,2BAAmB;QAC9B,SAAS,EAAE,2BAAmB;KACjC;IACD;QACI,GAAG,EAAE,WAAW;QAChB,SAAS,EAAE,6BAAqB;QAChC,SAAS,EAAE,6BAAqB;KACnC;IACD;QACI,GAAG,EAAE,SAAS;QACd,SAAS,EAAE,2BAAmB;QAC9B,SAAS,EAAE,2BAAmB;KACjC;IACD;QACI,GAAG,EAAE,aAAa;QAClB,SAAS,EAAE,+BAAuB;QAClC,SAAS,EAAE,+BAAuB;KACrC;IACD;QACI,GAAG,EAAE,eAAe;QACpB,SAAS,EAAE,iCAAyB;QACpC,SAAS,EAAE,iCAAyB;KACvC;IACD;QACI,GAAG,EAAE,gBAAgB;QACrB,SAAS,EAAE,mCAA2B;QACtC,SAAS,EAAE,mCAA2B;KACzC;IACD;QACI,GAAG,EAAE,aAAa;QAClB,SAAS,EAAE,+BAAuB;QAClC,SAAS,EAAE,+BAAuB;KACrC;CACJ,CAAC;AAEF;;GAEG;AACU,QAAA,SAAS,GAAwB,qBAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC"}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { DetectorResult, Evaluator } from "@xapp/arachne-parsers";
|
|
2
|
+
import { DetectorConstructor } from "../contants";
|
|
3
|
+
import { DetectionSource, SourcedDetectorResult, SubpageDetectionOptions, SubpageType } from "../models";
|
|
4
|
+
/** Default budget for one subpage: navigation plus detectors. */
|
|
5
|
+
export declare const DEFAULT_SUBPAGE_TIMEOUT_MS = 10000;
|
|
6
|
+
/** Default cap on subpages visited in one crawl. */
|
|
7
|
+
export declare const DEFAULT_MAX_SUBPAGES = 3;
|
|
8
|
+
/** A subpage is not started with less than this left of the overall budget. */
|
|
9
|
+
export declare const MIN_SUBPAGE_BUDGET_MS = 1000;
|
|
10
|
+
/** Longest wait for a subpage tab to close before moving on. */
|
|
11
|
+
export declare const TAB_CLOSE_TIMEOUT_MS = 2000;
|
|
12
|
+
export interface SubpageTarget {
|
|
13
|
+
pageType: SubpageType;
|
|
14
|
+
url: string;
|
|
15
|
+
}
|
|
16
|
+
export interface SubpageVisit extends SubpageTarget {
|
|
17
|
+
status: "detected" | "skipped";
|
|
18
|
+
/**
|
|
19
|
+
* Why the page was skipped.
|
|
20
|
+
*/
|
|
21
|
+
reason?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* True when `candidate` is an http(s) URL on the same site as `base`. www and the apex count as the same site;
|
|
25
|
+
* any other subdomain does not.
|
|
26
|
+
*/
|
|
27
|
+
export declare function isSameSite(candidate: string, base: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* True when the options ask for at least one page.
|
|
30
|
+
*/
|
|
31
|
+
export declare function isSubpageDetectionEnabled(options: SubpageDetectionOptions | undefined): boolean;
|
|
32
|
+
/**
|
|
33
|
+
* Picks the subpages to visit: the configured page types that were discovered and are on the same site as the
|
|
34
|
+
* homepage, one visit per page type and per URL, never the homepage itself, at most `maxPages`.
|
|
35
|
+
*
|
|
36
|
+
* @param pages Discovered pages, e.g. `site.pages`
|
|
37
|
+
* @param homepageUrl The URL the crawl started from
|
|
38
|
+
* @param options Subpage detection options
|
|
39
|
+
*/
|
|
40
|
+
export declare function resolveSubpageTargets(pages: Partial<Record<SubpageType, string>> | undefined, homepageUrl: string, options: SubpageDetectionOptions | undefined): SubpageTarget[];
|
|
41
|
+
/**
|
|
42
|
+
* Merges one page's detector results into the results gathered so far.
|
|
43
|
+
*
|
|
44
|
+
* Each technology appears once. A technology not seen yet is added with `source`. A technology already detected
|
|
45
|
+
* keeps its first source; one only reported as not installed so far is replaced, in place, by a detection.
|
|
46
|
+
*
|
|
47
|
+
* @param target Results gathered so far, mutated
|
|
48
|
+
* @param incoming Results from the page
|
|
49
|
+
* @param source The page they came from
|
|
50
|
+
*/
|
|
51
|
+
export declare function mergeDetectorResults(target: SourcedDetectorResult[], incoming: DetectorResult[], source: DetectionSource): void;
|
|
52
|
+
/**
|
|
53
|
+
* Navigates to a page on the same site and returns its rendered HTML.
|
|
54
|
+
*
|
|
55
|
+
* Rejects, without navigating, for an off-site URL; and rejects when the page does not load within `timeoutMs`,
|
|
56
|
+
* answers with an error status, or redirects off-site.
|
|
57
|
+
*
|
|
58
|
+
* @param page The live page to navigate
|
|
59
|
+
* @param url The page to load
|
|
60
|
+
* @param homepageUrl The URL the crawl started from
|
|
61
|
+
* @param timeoutMs Budget for the navigation and reading the content
|
|
62
|
+
*/
|
|
63
|
+
export declare function loadSameSitePage(page: Evaluator, url: string, homepageUrl: string, timeoutMs: number): Promise<string>;
|
|
64
|
+
/**
|
|
65
|
+
* A browser tab used for subpage visits, i.e. a puppeteer Page.
|
|
66
|
+
*/
|
|
67
|
+
export interface SubpageTab extends Evaluator {
|
|
68
|
+
isClosed(): boolean;
|
|
69
|
+
close(options?: {
|
|
70
|
+
runBeforeUnload?: boolean;
|
|
71
|
+
}): Promise<void>;
|
|
72
|
+
}
|
|
73
|
+
export interface LazyTab {
|
|
74
|
+
/**
|
|
75
|
+
* The tab, opened on first use. Rejects once the tab has been closed.
|
|
76
|
+
*/
|
|
77
|
+
get(): Promise<SubpageTab>;
|
|
78
|
+
/**
|
|
79
|
+
* Closes the tab if one was opened. Never throws, and waits at most the close timeout; a tab still opening is
|
|
80
|
+
* closed as soon as it opens.
|
|
81
|
+
*/
|
|
82
|
+
close(): Promise<void>;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* A tab that is only opened when first needed, so subpage visits never navigate the page the homepage detectors
|
|
86
|
+
* ran on: a detector that outlived its budget keeps reading the homepage's DOM, not a subpage's.
|
|
87
|
+
*
|
|
88
|
+
* Use one per page visit and `close()` it in a `finally`. Closing is what cancels a navigation that outlived its
|
|
89
|
+
* budget; a timeout race alone leaves it pending, and a second `goto()` on that tab would fail.
|
|
90
|
+
*
|
|
91
|
+
* @param open Opens a new tab and applies the same setup as the crawler's own pages
|
|
92
|
+
* @param closeTimeoutMs Longest wait for the tab to close
|
|
93
|
+
*/
|
|
94
|
+
export declare function createLazyTab(open: () => Promise<SubpageTab>, closeTimeoutMs?: number): LazyTab;
|
|
95
|
+
export interface DetectOnSubpagesProps {
|
|
96
|
+
/**
|
|
97
|
+
* Opens a new tab, with the crawler's page setup. Called once per target; never returns the page the homepage
|
|
98
|
+
* detectors ran on, nor a tab used before.
|
|
99
|
+
*/
|
|
100
|
+
openTab: () => Promise<SubpageTab>;
|
|
101
|
+
/**
|
|
102
|
+
* The URL the crawl started from.
|
|
103
|
+
*/
|
|
104
|
+
homepageUrl: string;
|
|
105
|
+
/**
|
|
106
|
+
* Pages to visit, from `resolveSubpageTargets()`.
|
|
107
|
+
*/
|
|
108
|
+
targets: SubpageTarget[];
|
|
109
|
+
detectors: DetectorConstructor[];
|
|
110
|
+
/**
|
|
111
|
+
* Results gathered so far; each page's results are merged in once that page finishes.
|
|
112
|
+
*/
|
|
113
|
+
detectedTechnologies: SourcedDetectorResult[];
|
|
114
|
+
/**
|
|
115
|
+
* Budget for each page.
|
|
116
|
+
*/
|
|
117
|
+
pageTimeoutMs: number;
|
|
118
|
+
/**
|
|
119
|
+
* Epoch milliseconds by which all subpages must be done.
|
|
120
|
+
*/
|
|
121
|
+
deadline: number;
|
|
122
|
+
/**
|
|
123
|
+
* Receives the HTML of each page that loaded.
|
|
124
|
+
*/
|
|
125
|
+
onPageContent?: (target: SubpageTarget, html: string) => void;
|
|
126
|
+
/**
|
|
127
|
+
* Longest wait for each tab to close. Defaults to TAB_CLOSE_TIMEOUT_MS.
|
|
128
|
+
*/
|
|
129
|
+
tabCloseTimeoutMs?: number;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Visits each target in turn, each on a fresh tab closed once that target is done, and runs the detector chain on it.
|
|
133
|
+
*
|
|
134
|
+
* A page that fails to load, errors, or does not finish its detectors within its budget is skipped and its partial
|
|
135
|
+
* results are discarded; results already gathered are never lost. Nothing is started once less than
|
|
136
|
+
* MIN_SUBPAGE_BUDGET_MS is left before `deadline`.
|
|
137
|
+
*
|
|
138
|
+
* @returns What happened on each target
|
|
139
|
+
*/
|
|
140
|
+
export declare function detectOnSubpages(props: DetectOnSubpagesProps): Promise<SubpageVisit[]>;
|