@xapp/arachne-web-service 1.8.13 → 1.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +13 -1
- package/lib/ArachneWebService.js +227 -129
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/GenerativeAIService.d.ts +2 -2
- package/lib/GenerativeAIService.js +2 -3
- package/lib/GenerativeAIService.js.map +1 -1
- package/lib/contants.js +1 -0
- package/lib/contants.js.map +1 -1
- package/lib/models.d.ts +27 -3
- package/lib/utils/extractContacts.d.ts +16 -1
- package/lib/utils/extractContacts.js +30 -4
- package/lib/utils/extractContacts.js.map +1 -1
- package/lib/utils/findBestFaviconURL.js +47 -15
- package/lib/utils/findBestFaviconURL.js.map +1 -1
- package/lib/utils/getBusinessCategoryFromSchema.d.ts +2 -2
- package/lib/utils/getBusinessCategoryFromSchema.js +16 -16
- package/lib/utils/getBusinessCategoryFromSchema.js.map +1 -1
- package/lib/utils/getBusinessData.d.ts +10 -4
- package/lib/utils/getBusinessData.js +69 -23
- package/lib/utils/getBusinessData.js.map +1 -1
- package/lib/utils/getPrompt.d.ts +4 -1
- package/lib/utils/getPrompt.js +13 -5
- package/lib/utils/getPrompt.js.map +1 -1
- package/lib/utils/getSiteMap.js +17 -7
- package/lib/utils/getSiteMap.js.map +1 -1
- package/package.json +18 -18
|
@@ -118,6 +118,18 @@ export interface WebsiteData {
|
|
|
118
118
|
*/
|
|
119
119
|
failedRequests?: ArachneRequest[];
|
|
120
120
|
}
|
|
121
|
+
export interface Callbacks {
|
|
122
|
+
/**
|
|
123
|
+
* A callback that will be called with partial data as it is collected.
|
|
124
|
+
*
|
|
125
|
+
* The point field is a reference to where the data was collected.
|
|
126
|
+
*
|
|
127
|
+
* @param data
|
|
128
|
+
* @param point
|
|
129
|
+
* @returns
|
|
130
|
+
*/
|
|
131
|
+
onPartialData?: (data: Partial<WebsiteData>, point?: string) => void;
|
|
132
|
+
}
|
|
121
133
|
export declare class ArachneWebService {
|
|
122
134
|
/**
|
|
123
135
|
* Get data on the provided website.
|
|
@@ -126,5 +138,5 @@ export declare class ArachneWebService {
|
|
|
126
138
|
* @param options
|
|
127
139
|
* @returns
|
|
128
140
|
*/
|
|
129
|
-
getWebsiteData(url: string, options?: ArachneAppGetInfoOptions): Promise<WebsiteData>;
|
|
141
|
+
getWebsiteData(url: string, options?: ArachneAppGetInfoOptions, callbacks?: Callbacks): Promise<WebsiteData>;
|
|
130
142
|
}
|
package/lib/ArachneWebService.js
CHANGED
|
@@ -15,13 +15,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
|
|
|
15
15
|
}) : function(o, v) {
|
|
16
16
|
o["default"] = v;
|
|
17
17
|
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
};
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
25
35
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
36
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
37
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -79,12 +89,18 @@ class ArachneWebService {
|
|
|
79
89
|
* @returns
|
|
80
90
|
*/
|
|
81
91
|
getWebsiteData(url_1) {
|
|
82
|
-
return __awaiter(this, arguments, void 0, function* (url, options = {}) {
|
|
92
|
+
return __awaiter(this, arguments, void 0, function* (url, options = {}, callbacks) {
|
|
93
|
+
// Setup initial data structures, these will be populated as we gather more information
|
|
83
94
|
const site = {};
|
|
84
95
|
const detectedTechnologies = [];
|
|
85
96
|
const schema = {};
|
|
86
97
|
const screenshots = [];
|
|
87
98
|
let business = {};
|
|
99
|
+
// if we have a callback, we will call it with partial data
|
|
100
|
+
let onPartialData = () => { };
|
|
101
|
+
if (typeof (callbacks === null || callbacks === void 0 ? void 0 : callbacks.onPartialData) === "function") {
|
|
102
|
+
onPartialData = callbacks.onPartialData;
|
|
103
|
+
}
|
|
88
104
|
// use the one from options or default to defaultBrowserBuilder
|
|
89
105
|
const { buildBrowser = defaultBrowserBuilder } = options;
|
|
90
106
|
// first get the sitemap
|
|
@@ -137,7 +153,7 @@ class ArachneWebService {
|
|
|
137
153
|
if (pageWait) {
|
|
138
154
|
(0, stentor_logger_1.log)().info(`Waiting for ${pageWaitTime} ms for page to completely load.`);
|
|
139
155
|
yield (0, arachne_1.autoScroll)(page, pageWaitTime);
|
|
140
|
-
(0, stentor_logger_1.log)().info(
|
|
156
|
+
(0, stentor_logger_1.log)().info("...and done waiting.");
|
|
141
157
|
}
|
|
142
158
|
// Get the raw HTML
|
|
143
159
|
// It is important to use page.content() versus response.text() since
|
|
@@ -145,73 +161,115 @@ class ArachneWebService {
|
|
|
145
161
|
// on the page while response.text() only gives you the HTML from the original GET
|
|
146
162
|
const html = yield page.content();
|
|
147
163
|
const $ = cheerio.load(html);
|
|
148
|
-
|
|
164
|
+
// SAVE HTML & MARKDOWN
|
|
165
|
+
(0, stentor_logger_1.log)().info(`Converting HTML to markdown...`);
|
|
166
|
+
// Convert to markdown
|
|
167
|
+
const markdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
168
|
+
stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"],
|
|
169
|
+
}).convert(html, true);
|
|
170
|
+
(0, stentor_logger_1.log)().debug(`Markdown: ${markdown.length} characters`);
|
|
171
|
+
if (options.markdownPath) {
|
|
172
|
+
const path = (0, path_1.resolve)(options.markdownPath, `${fileName}.md`);
|
|
173
|
+
(0, stentor_logger_1.log)().debug(`Writing markdown to ${path}...`);
|
|
174
|
+
yield fs.promises.writeFile(path, markdown);
|
|
175
|
+
(0, stentor_logger_1.log)().debug(`Markdown written.`);
|
|
176
|
+
}
|
|
177
|
+
if (options.htmlPath) {
|
|
178
|
+
const path = (0, path_1.resolve)(options.htmlPath, `${fileName}.html`);
|
|
179
|
+
(0, stentor_logger_1.log)().debug(`Writing HTML to ${path}...`);
|
|
180
|
+
yield fs.promises.writeFile(path, html);
|
|
181
|
+
(0, stentor_logger_1.log)().debug(`HTML written.`);
|
|
182
|
+
}
|
|
183
|
+
// DONE SAVING MARKDOWN & HTML
|
|
184
|
+
const description = $('meta[name="description"]').attr("content");
|
|
149
185
|
const htmlHasContactInfo = (0, extractContacts_1.hasContactInfo)(html);
|
|
150
186
|
if (htmlHasContactInfo) {
|
|
151
187
|
(0, stentor_logger_1.log)().info(`Contact information detected in the HTML.`);
|
|
152
188
|
}
|
|
153
189
|
const htmlExtractedContacts = (0, extractContacts_1.extractContacts)(html);
|
|
154
190
|
const things = new arachne_parsers_1.ArachneJSONLD(html).getThings();
|
|
155
|
-
|
|
156
|
-
// TODO: Why aren't we using this?
|
|
157
|
-
// let homeConstructionBusiness: HomeAndConstructionBusiness | undefined;
|
|
158
|
-
(0, stentor_logger_1.log)().info('Looking through JSON Schema...');
|
|
191
|
+
(0, stentor_logger_1.log)().info("Looking through JSON Schema...");
|
|
159
192
|
for (const thing of things) {
|
|
160
193
|
if ((0, arachne_parsers_1.isLocalBusiness)(thing)) {
|
|
161
194
|
(0, stentor_logger_1.log)().debug(`Found local business information`);
|
|
162
|
-
localBusiness = thing;
|
|
163
195
|
schema.localBusiness = thing;
|
|
164
196
|
}
|
|
165
197
|
else if ((0, arachne_parsers_1.isHomeConstructionBusiness)(thing)) {
|
|
166
198
|
(0, stentor_logger_1.log)().debug(`Found home construction business`);
|
|
167
|
-
// homeConstructionBusiness = thing;
|
|
168
199
|
schema.homeConstructionBusiness = thing;
|
|
169
200
|
}
|
|
201
|
+
else if ((0, arachne_parsers_1.isOrganization)(thing)) {
|
|
202
|
+
(0, stentor_logger_1.log)().debug(`Found organization information`);
|
|
203
|
+
schema.organization = thing;
|
|
204
|
+
}
|
|
170
205
|
else {
|
|
171
206
|
(0, stentor_logger_1.log)().debug(`Found JSON Schema ${thing["@id"]} ${thing["@type"]}`);
|
|
207
|
+
// console.log(thing);
|
|
172
208
|
}
|
|
173
209
|
}
|
|
174
|
-
// We
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
if (typeof localBusinessLogo === "string") {
|
|
178
|
-
try {
|
|
179
|
-
const square = yield (0, image_1.imageIsSquare)(localBusinessLogo, 75);
|
|
180
|
-
if (square) {
|
|
181
|
-
(0, stentor_logger_1.log)().debug(`Found a square logo from json+ld ${localBusinessLogo}`);
|
|
182
|
-
site.logo = localBusinessLogo;
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
catch (e) {
|
|
186
|
-
(0, stentor_logger_1.log)().debug(`Error determining if logo is square ${e}`);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
// See if we are installed
|
|
210
|
+
// We now go through and try to find data from what we have!
|
|
211
|
+
const businessData = yield (0, getBusinessData_1.getBusinessDataFromSchema)(schema);
|
|
212
|
+
business = Object.assign({}, businessData);
|
|
191
213
|
if (!site.logo) {
|
|
192
|
-
// try to get the
|
|
214
|
+
// try to get the
|
|
193
215
|
(0, stentor_logger_1.log)().debug(`Looking for logo from favicon`);
|
|
194
216
|
const favicon = yield (0, findBestFaviconURL_1.findBestFaviconURL)(page, request.url);
|
|
195
217
|
(0, stentor_logger_1.log)().debug(`Found a favicon ${favicon}, using it.`);
|
|
196
218
|
// favicons are typically square, lets use it?
|
|
197
219
|
site.logo = favicon;
|
|
198
220
|
}
|
|
199
|
-
|
|
221
|
+
if (!business.logo && (site === null || site === void 0 ? void 0 : site.logo)) {
|
|
222
|
+
(0, stentor_logger_1.log)().info(`Setting business logo from site logo.`);
|
|
223
|
+
business.logo = site.logo;
|
|
224
|
+
}
|
|
225
|
+
if ((0, stentor_utils_1.existsAndNotEmpty)(business.phoneNumbers)) {
|
|
226
|
+
(0, stentor_logger_1.log)().info(`Phone numbers from schema ${(_a = business.phoneNumbers) === null || _a === void 0 ? void 0 : _a.map((p) => p.number).join(", ")}`);
|
|
227
|
+
}
|
|
228
|
+
// If we have no phone numbers from the schema, but we have some from the HTML
|
|
229
|
+
if (!(0, stentor_utils_1.existsAndNotEmpty)(business.phoneNumbers) &&
|
|
230
|
+
(0, stentor_utils_1.existsAndNotEmpty)(htmlExtractedContacts.phoneNumbers)) {
|
|
231
|
+
(0, stentor_logger_1.log)().info(`Updating business phone numbers from HTML. ${htmlExtractedContacts.phoneNumbers.join(", ")}`);
|
|
232
|
+
business.phoneNumbers = htmlExtractedContacts.phoneNumbers.map((p) => ({ number: p }));
|
|
233
|
+
}
|
|
234
|
+
// If we have no emails from the schema, but we have some from the HTML
|
|
235
|
+
if (!business.email && (0, stentor_utils_1.existsAndNotEmpty)(htmlExtractedContacts.emails)) {
|
|
236
|
+
(0, stentor_logger_1.log)().info(`Updating business email from HTML. ${htmlExtractedContacts.emails.join(", ")}`);
|
|
237
|
+
business.email = htmlExtractedContacts.emails.map((e) => ({ email: e }));
|
|
238
|
+
}
|
|
239
|
+
if (!business.category && ((schema === null || schema === void 0 ? void 0 : schema.localBusiness) || (schema === null || schema === void 0 ? void 0 : schema.homeConstructionBusiness))) {
|
|
240
|
+
business.category = (0, getBusinessCategoryFromSchema_1.getBusinessCategoryFromSchema)((schema === null || schema === void 0 ? void 0 : schema.localBusiness) || (schema === null || schema === void 0 ? void 0 : schema.homeConstructionBusiness));
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* PARTIAL DATA AVAILABLE
|
|
244
|
+
*
|
|
245
|
+
* Initial site load will have basic schema data and business data.
|
|
246
|
+
*/
|
|
247
|
+
onPartialData({
|
|
248
|
+
url,
|
|
249
|
+
site,
|
|
250
|
+
detectedTechnologies,
|
|
251
|
+
business,
|
|
252
|
+
schema,
|
|
253
|
+
screenshots,
|
|
254
|
+
}, "AFTER_SITE_LOAD");
|
|
255
|
+
// Try to figure out primary color, requires a screenshot saved to disk to analyze
|
|
200
256
|
if (options.screenshotsPath) {
|
|
201
257
|
const path = (0, path_1.resolve)(options.screenshotsPath, `${fileName}.png`);
|
|
202
258
|
(0, stentor_logger_1.log)().debug(`Saving screenshot to ${path}`);
|
|
203
259
|
const captureBeyondViewport = options.viewportSize ? false : true;
|
|
204
260
|
const fullPage = options.viewportSize ? false : true;
|
|
205
|
-
yield page
|
|
261
|
+
yield page
|
|
262
|
+
.screenshot({
|
|
206
263
|
path,
|
|
207
264
|
fullPage,
|
|
208
|
-
captureBeyondViewport
|
|
209
|
-
})
|
|
265
|
+
captureBeyondViewport,
|
|
266
|
+
})
|
|
267
|
+
.catch((e) => {
|
|
210
268
|
(0, stentor_logger_1.log)().error(`Error saving screenshot`);
|
|
211
269
|
console.error(e);
|
|
212
270
|
});
|
|
213
271
|
screenshots.push({
|
|
214
|
-
filePath: path
|
|
272
|
+
filePath: path,
|
|
215
273
|
});
|
|
216
274
|
(0, stentor_logger_1.log)().debug(`Screenshot saved.`);
|
|
217
275
|
(0, stentor_logger_1.log)().debug(`Analyzing colors...`);
|
|
@@ -249,7 +307,7 @@ class ArachneWebService {
|
|
|
249
307
|
continue;
|
|
250
308
|
default:
|
|
251
309
|
}
|
|
252
|
-
const debugPreview =
|
|
310
|
+
const debugPreview = "\x1b[48;2;" + (0, image_1.hexToRGB)(current === null || current === void 0 ? void 0 : current.hex) + "m \x1b[0m";
|
|
253
311
|
debugSwatches = `${debugSwatches} ${swatch} (${current === null || current === void 0 ? void 0 : current.population}):${current === null || current === void 0 ? void 0 : current.hex} ${debugPreview} `;
|
|
254
312
|
}
|
|
255
313
|
}
|
|
@@ -261,9 +319,9 @@ class ArachneWebService {
|
|
|
261
319
|
}
|
|
262
320
|
site.primaryColor = mostPopular === null || mostPopular === void 0 ? void 0 : mostPopular.hex;
|
|
263
321
|
site.secondaryColor = (0, image_1.complementHexColor)(site.primaryColor);
|
|
264
|
-
const primaryPreview =
|
|
265
|
-
const secondaryPreview =
|
|
266
|
-
const vibrantPreview =
|
|
322
|
+
const primaryPreview = "\x1b[48;2;" + (0, image_1.hexToRGB)(site.primaryColor) + "m \x1b[0m";
|
|
323
|
+
const secondaryPreview = "\x1b[48;2;" + (0, image_1.hexToRGB)(site.secondaryColor) + "m \x1b[0m";
|
|
324
|
+
const vibrantPreview = "\x1b[48;2;" + (0, image_1.hexToRGB)(site.vibrantColor) + "m \x1b[0m";
|
|
267
325
|
(0, stentor_logger_1.log)().debug(`Primary color: ${site.primaryColor} ${primaryPreview}`);
|
|
268
326
|
(0, stentor_logger_1.log)().debug(`Secondary color: ${site.secondaryColor} ${secondaryPreview}`);
|
|
269
327
|
(0, stentor_logger_1.log)().debug(`Vibrant color: ${site.vibrantColor} ${vibrantPreview}`);
|
|
@@ -273,56 +331,62 @@ class ArachneWebService {
|
|
|
273
331
|
else {
|
|
274
332
|
(0, stentor_logger_1.log)().info(`screenshotsPath not provided, cannot determine primaryColor and secondaryColor`);
|
|
275
333
|
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
(0, stentor_logger_1.log)().debug(`HTML written.`);
|
|
291
|
-
}
|
|
334
|
+
// Done looking for colors
|
|
335
|
+
/**
|
|
336
|
+
* PARTIAL DATA AVAILABLE
|
|
337
|
+
*
|
|
338
|
+
* We now have primaryColor, secondaryColor, and screenshots available.
|
|
339
|
+
*/
|
|
340
|
+
onPartialData({
|
|
341
|
+
url,
|
|
342
|
+
site,
|
|
343
|
+
detectedTechnologies,
|
|
344
|
+
business,
|
|
345
|
+
schema,
|
|
346
|
+
screenshots,
|
|
347
|
+
}, "AFTER_COLORS");
|
|
292
348
|
let aiData = undefined;
|
|
293
349
|
const { openAIKey } = options;
|
|
294
350
|
// Use OpenAI to determine Call To Action
|
|
295
351
|
if (openAIKey) {
|
|
296
352
|
(0, stentor_logger_1.log)().debug(`Calling GenerativeAI for website data...`);
|
|
297
353
|
const genService = new GenerativeAIService_1.GenerativeAIService({
|
|
298
|
-
key: openAIKey,
|
|
354
|
+
key: openAIKey,
|
|
355
|
+
page,
|
|
299
356
|
});
|
|
300
357
|
aiData = yield genService.getWebsiteAI(markdown, {
|
|
301
358
|
url,
|
|
302
359
|
site,
|
|
303
|
-
schema
|
|
360
|
+
schema,
|
|
304
361
|
});
|
|
305
362
|
if (!aiData) {
|
|
306
363
|
(0, stentor_logger_1.log)().info(`No AI Data received from GenerativeAI.`);
|
|
307
364
|
// give it empty data so we can add to it later
|
|
308
365
|
aiData = {};
|
|
309
366
|
}
|
|
310
|
-
if (!(0, extractContacts_1.
|
|
311
|
-
(0, stentor_logger_1.log)().warn(`No
|
|
367
|
+
if (!(0, extractContacts_1.hasEmail)(markdown) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.email)) {
|
|
368
|
+
(0, stentor_logger_1.log)().warn(`No emails detected in the markdown but AI found some, removing AI emails.`);
|
|
369
|
+
aiData.email = [];
|
|
370
|
+
}
|
|
371
|
+
if (!(0, extractContacts_1.hasPhoneNumber)(markdown) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers)) {
|
|
372
|
+
(0, stentor_logger_1.log)().warn(`No phone numbers detected in the markdown but AI found some, removing AI phone numbers.`);
|
|
373
|
+
aiData.phone_numbers = [];
|
|
312
374
|
}
|
|
313
|
-
if (((
|
|
375
|
+
if (((_b = options === null || options === void 0 ? void 0 : options.dataOptions) === null || _b === void 0 ? void 0 : _b.preferContactUsPage) && ((_c = site.pages) === null || _c === void 0 ? void 0 : _c.contactUs)) {
|
|
314
376
|
(0, stentor_logger_1.log)().info(`Using Contact Us page for data.`);
|
|
315
377
|
const contactUsPageUrl = site.pages.contactUs;
|
|
316
378
|
yield page.goto(contactUsPageUrl);
|
|
317
379
|
const contactUsHTML = yield page.content();
|
|
318
|
-
const contactUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
380
|
+
const contactUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
381
|
+
stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"],
|
|
382
|
+
}).convert(contactUsHTML, true);
|
|
319
383
|
if ((0, extractContacts_1.hasContactInfo)(contactUsMarkdown)) {
|
|
320
384
|
const extractedContacts = (0, extractContacts_1.extractContacts)(contactUsMarkdown);
|
|
321
385
|
const contactUsData = yield genService.getWebsiteAI(contactUsMarkdown, {
|
|
322
|
-
url: contactUsPageUrl
|
|
386
|
+
url: contactUsPageUrl,
|
|
323
387
|
});
|
|
324
|
-
const contactEmail = (
|
|
325
|
-
const contactPhoneNumbers = (
|
|
388
|
+
const contactEmail = (_d = contactUsData === null || contactUsData === void 0 ? void 0 : contactUsData.email) === null || _d === void 0 ? void 0 : _d.filter((e) => e.email.length > 0);
|
|
389
|
+
const contactPhoneNumbers = (_e = contactUsData === null || contactUsData === void 0 ? void 0 : contactUsData.phone_numbers) === null || _e === void 0 ? void 0 : _e.filter((p) => p.number.length > 0);
|
|
326
390
|
if ((0, stentor_utils_1.existsAndNotEmpty)(contactPhoneNumbers) && aiData) {
|
|
327
391
|
(0, stentor_logger_1.log)().info(`Updating with contact us phone numbers.\n${JSON.stringify(contactPhoneNumbers)}`);
|
|
328
392
|
aiData.phone_numbers = contactPhoneNumbers;
|
|
@@ -342,17 +406,19 @@ class ArachneWebService {
|
|
|
342
406
|
(0, stentor_logger_1.log)().info(`No detected contact information in the Contact Us page`);
|
|
343
407
|
}
|
|
344
408
|
}
|
|
345
|
-
if ((
|
|
409
|
+
if ((_f = options === null || options === void 0 ? void 0 : options.dataOptions) === null || _f === void 0 ? void 0 : _f.preferHeaderInfo) {
|
|
346
410
|
(0, stentor_logger_1.log)().info(`Looking at header information for data.`);
|
|
347
|
-
const headerHTML = yield page.evaluate(() => { var _a; return (_a = document.querySelector(
|
|
411
|
+
const headerHTML = yield page.evaluate(() => { var _a; return (_a = document.querySelector("header")) === null || _a === void 0 ? void 0 : _a.outerHTML; });
|
|
348
412
|
if (headerHTML) {
|
|
349
|
-
const headerMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
413
|
+
const headerMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
414
|
+
stripTags: ["script", "head", "img", "iframe", "style"],
|
|
415
|
+
}).convert(headerHTML, true);
|
|
350
416
|
if ((0, extractContacts_1.hasContactInfo)(headerMarkdown)) {
|
|
351
417
|
const headerData = yield genService.getWebsiteAI(headerMarkdown, {
|
|
352
|
-
url
|
|
418
|
+
url,
|
|
353
419
|
});
|
|
354
|
-
const headerEmails = (
|
|
355
|
-
const headerPhoneNumbers = (
|
|
420
|
+
const headerEmails = (_g = headerData === null || headerData === void 0 ? void 0 : headerData.email) === null || _g === void 0 ? void 0 : _g.filter((e) => e.email.length > 0);
|
|
421
|
+
const headerPhoneNumbers = (_h = headerData === null || headerData === void 0 ? void 0 : headerData.phone_numbers) === null || _h === void 0 ? void 0 : _h.filter((p) => p.number.length > 0);
|
|
356
422
|
if ((0, stentor_utils_1.existsAndNotEmpty)(headerPhoneNumbers) && aiData) {
|
|
357
423
|
(0, stentor_logger_1.log)().info(`Updating with header phone numbers.\n${JSON.stringify(headerPhoneNumbers)}`);
|
|
358
424
|
aiData.phone_numbers = headerPhoneNumbers;
|
|
@@ -367,12 +433,83 @@ class ArachneWebService {
|
|
|
367
433
|
}
|
|
368
434
|
}
|
|
369
435
|
}
|
|
436
|
+
// We need to look at the contact information from the AI and make sure it was in the original html text
|
|
437
|
+
// TODO: Filter these?
|
|
438
|
+
if (aiData.email && (0, stentor_utils_1.existsAndNotEmpty)(aiData.email)) {
|
|
439
|
+
const emails = aiData.email.map((e) => e.email);
|
|
440
|
+
const foundEmails = emails.filter((e) => htmlExtractedContacts.emails.includes(e));
|
|
441
|
+
if (foundEmails.length > 0) {
|
|
442
|
+
(0, stentor_logger_1.log)().info(`Found ${foundEmails.length} emails from AI in the HTML.`);
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
if (aiData.phone_numbers && (0, stentor_utils_1.existsAndNotEmpty)(aiData.phone_numbers)) {
|
|
446
|
+
const phoneNumbers = aiData.phone_numbers.map((p) => p.number);
|
|
447
|
+
const foundPhoneNumbers = phoneNumbers.filter((p) => htmlExtractedContacts.phoneNumbers.includes(p));
|
|
448
|
+
if (foundPhoneNumbers.length > 0) {
|
|
449
|
+
(0, stentor_logger_1.log)().info(`Found ${foundPhoneNumbers.length} phone numbers from AI in the HTML.`);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
370
452
|
}
|
|
371
453
|
else {
|
|
372
454
|
(0, stentor_logger_1.log)().info(`OpenAI key not provided, not calling GenerativeAIService.`);
|
|
373
455
|
}
|
|
456
|
+
// Find some AI overrides (if we have the aiData)
|
|
457
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.organization_name) {
|
|
458
|
+
business.name = aiData.organization_name;
|
|
459
|
+
}
|
|
460
|
+
if ((aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers)) {
|
|
461
|
+
(0, stentor_logger_1.log)().info(`Updating business phone numbers from AI data. ${(_j = aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers) === null || _j === void 0 ? void 0 : _j.map((p) => p.number).join(", ")}`);
|
|
462
|
+
business.phoneNumbers = aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers;
|
|
463
|
+
}
|
|
464
|
+
if ((aiData === null || aiData === void 0 ? void 0 : aiData.email) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.email)) {
|
|
465
|
+
(0, stentor_logger_1.log)().info(`Updating business email from AI data. ${(_k = aiData === null || aiData === void 0 ? void 0 : aiData.email) === null || _k === void 0 ? void 0 : _k.map((p) => p.email).join(", ")}`);
|
|
466
|
+
business.email = aiData === null || aiData === void 0 ? void 0 : aiData.email;
|
|
467
|
+
}
|
|
468
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.industry_category) {
|
|
469
|
+
// It isn't perfect so we need to clean it
|
|
470
|
+
const category = aiData.industry_category; //.replace("_", "-").replace(" ", "-").toLowerCase() as BusinessCategory;
|
|
471
|
+
business.category = category;
|
|
472
|
+
}
|
|
473
|
+
// if no business addresses from schema, look on AI data
|
|
474
|
+
if (!(0, stentor_utils_1.existsAndNotEmpty)(business.addresses) &&
|
|
475
|
+
(aiData === null || aiData === void 0 ? void 0 : aiData.addresses) &&
|
|
476
|
+
(0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.addresses)) {
|
|
477
|
+
(0, stentor_logger_1.log)().info(`Updating business addresses from AI data.`);
|
|
478
|
+
business.addresses = aiData.addresses;
|
|
479
|
+
}
|
|
480
|
+
// Set some items
|
|
481
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.description) {
|
|
482
|
+
site.description = aiData.description;
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
site.description = description;
|
|
486
|
+
}
|
|
487
|
+
if (typeof (aiData === null || aiData === void 0 ? void 0 : aiData.parked_domain) === "boolean") {
|
|
488
|
+
site.parkedDomain = aiData.parked_domain;
|
|
489
|
+
}
|
|
490
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.calls_to_action) {
|
|
491
|
+
site.callsToAction = aiData.calls_to_action;
|
|
492
|
+
}
|
|
493
|
+
else {
|
|
494
|
+
site.callsToAction = (0, getCallToAction_1.getCallToAction)(html);
|
|
495
|
+
}
|
|
496
|
+
/**
|
|
497
|
+
* PARTIAL DATA AVAILABLE
|
|
498
|
+
*
|
|
499
|
+
* We now have primaryColor, secondaryColor, and screenshots available.
|
|
500
|
+
*/
|
|
501
|
+
onPartialData({
|
|
502
|
+
url,
|
|
503
|
+
site,
|
|
504
|
+
detectedTechnologies,
|
|
505
|
+
business,
|
|
506
|
+
schema,
|
|
507
|
+
screenshots,
|
|
508
|
+
}, "AFTER_AI");
|
|
374
509
|
(0, stentor_logger_1.log)().debug(`Preparing detectors...`);
|
|
375
|
-
const detectors = (0, stentor_utils_1.existsAndNotEmpty)(options.detectors)
|
|
510
|
+
const detectors = (0, stentor_utils_1.existsAndNotEmpty)(options.detectors)
|
|
511
|
+
? options.detectors || []
|
|
512
|
+
: contants_1.DEFAULT_DETECTORS;
|
|
376
513
|
(0, stentor_logger_1.log)().info(`Detecting ${detectors.length} technologies...`);
|
|
377
514
|
// go through and detect each
|
|
378
515
|
for (const detector of detectors) {
|
|
@@ -405,64 +542,25 @@ class ArachneWebService {
|
|
|
405
542
|
console.error(e);
|
|
406
543
|
}
|
|
407
544
|
}
|
|
408
|
-
// We now go through and try to find data from what we have!
|
|
409
|
-
const businessData = (0, getBusinessData_1.getBusinessData)(schema.localBusiness || schema.homeConstructionBusiness);
|
|
410
|
-
business = Object.assign({}, businessData);
|
|
411
|
-
if ((0, stentor_utils_1.existsAndNotEmpty)(business.phoneNumbers)) {
|
|
412
|
-
(0, stentor_logger_1.log)().info(`Phone numbers from schema ${(_h = business.phoneNumbers) === null || _h === void 0 ? void 0 : _h.map((p) => p.number).join(", ")}`);
|
|
413
|
-
}
|
|
414
|
-
// Find some AI overrides (if we have the aiData)
|
|
415
|
-
if (aiData === null || aiData === void 0 ? void 0 : aiData.organization_name) {
|
|
416
|
-
business.name = aiData.organization_name;
|
|
417
|
-
}
|
|
418
|
-
if ((aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers)) {
|
|
419
|
-
(0, stentor_logger_1.log)().info(`Updating business phone numbers from AI data. ${(_j = aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers) === null || _j === void 0 ? void 0 : _j.map((p) => p.number).join(", ")}`);
|
|
420
|
-
business.phoneNumbers = aiData === null || aiData === void 0 ? void 0 : aiData.phone_numbers;
|
|
421
|
-
}
|
|
422
|
-
// If we have no phone numbers from the schema, but we have some from the HTML
|
|
423
|
-
if (!(0, stentor_utils_1.existsAndNotEmpty)(business.phoneNumbers) && (0, stentor_utils_1.existsAndNotEmpty)(htmlExtractedContacts.phoneNumbers)) {
|
|
424
|
-
(0, stentor_logger_1.log)().info(`Updating business phone numbers from HTML. ${htmlExtractedContacts.phoneNumbers.join(", ")}`);
|
|
425
|
-
business.phoneNumbers = htmlExtractedContacts.phoneNumbers.map((p) => ({ number: p }));
|
|
426
|
-
}
|
|
427
|
-
if ((aiData === null || aiData === void 0 ? void 0 : aiData.email) && (0, stentor_utils_1.existsAndNotEmpty)(aiData === null || aiData === void 0 ? void 0 : aiData.email)) {
|
|
428
|
-
(0, stentor_logger_1.log)().info(`Updating business email from AI data. ${(_k = aiData === null || aiData === void 0 ? void 0 : aiData.email) === null || _k === void 0 ? void 0 : _k.map((p) => p.email).join(", ")}`);
|
|
429
|
-
business.email = aiData === null || aiData === void 0 ? void 0 : aiData.email;
|
|
430
|
-
}
|
|
431
|
-
// If we have no emails from the schema, but we have some from the HTML
|
|
432
|
-
if (!business.email && (0, stentor_utils_1.existsAndNotEmpty)(htmlExtractedContacts.emails)) {
|
|
433
|
-
(0, stentor_logger_1.log)().info(`Updating business email from HTML. ${htmlExtractedContacts.emails.join(", ")}`);
|
|
434
|
-
business.email = htmlExtractedContacts.emails.map((e) => ({ email: e }));
|
|
435
|
-
}
|
|
436
|
-
if (aiData === null || aiData === void 0 ? void 0 : aiData.industry_category) {
|
|
437
|
-
// It isn't perfect so we need to clean it
|
|
438
|
-
const category = aiData.industry_category; //.replace("_", "-").replace(" ", "-").toLowerCase() as BusinessCategory;
|
|
439
|
-
business.category = category;
|
|
440
|
-
}
|
|
441
|
-
if (!business.category && ((schema === null || schema === void 0 ? void 0 : schema.localBusiness) || (schema === null || schema === void 0 ? void 0 : schema.homeConstructionBusiness))) {
|
|
442
|
-
business.category = (0, getBusinessCategoryFromSchema_1.getBusinessCategoryFromSchema)((schema === null || schema === void 0 ? void 0 : schema.localBusiness) || (schema === null || schema === void 0 ? void 0 : schema.homeConstructionBusiness));
|
|
443
|
-
}
|
|
444
|
-
// Set some items
|
|
445
|
-
if (aiData === null || aiData === void 0 ? void 0 : aiData.description) {
|
|
446
|
-
site.description = aiData.description;
|
|
447
|
-
}
|
|
448
|
-
else {
|
|
449
|
-
site.description = description;
|
|
450
|
-
}
|
|
451
|
-
if (typeof (aiData === null || aiData === void 0 ? void 0 : aiData.parked_domain) === "boolean") {
|
|
452
|
-
site.parkedDomain = aiData.parked_domain;
|
|
453
|
-
}
|
|
454
545
|
const godaddyParked = detectedTechnologies.find((d) => d.technology === arachne_parsers_1.GODADDY_PARKED_DOMAIN);
|
|
455
546
|
if (godaddyParked === null || godaddyParked === void 0 ? void 0 : godaddyParked.installed) {
|
|
456
547
|
// little override here, in case we find the parked domain
|
|
457
548
|
site.parkedDomain = true;
|
|
458
549
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
550
|
+
/**
|
|
551
|
+
* PARTIAL DATA AVAILABLE
|
|
552
|
+
*
|
|
553
|
+
* We now have primaryColor, secondaryColor, and screenshots available.
|
|
554
|
+
*/
|
|
555
|
+
onPartialData({
|
|
556
|
+
url,
|
|
557
|
+
site,
|
|
558
|
+
detectedTechnologies,
|
|
559
|
+
business,
|
|
560
|
+
schema,
|
|
561
|
+
screenshots,
|
|
562
|
+
}, "AFTER_DETECTORS");
|
|
563
|
+
}), // end of the page handler
|
|
466
564
|
});
|
|
467
565
|
yield crawler.run();
|
|
468
566
|
// see if things failed or not
|
|
@@ -478,7 +576,7 @@ class ArachneWebService {
|
|
|
478
576
|
screenshots,
|
|
479
577
|
business,
|
|
480
578
|
failed,
|
|
481
|
-
failedRequests
|
|
579
|
+
failedRequests,
|
|
482
580
|
};
|
|
483
581
|
});
|
|
484
582
|
}
|