@xapp/arachne-web-service 1.9.0 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +13 -1
- package/lib/ArachneWebService.js +186 -124
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/GenerativeAIService.js +31 -20
- package/lib/GenerativeAIService.js.map +1 -1
- package/lib/models.d.ts +6 -1
- package/lib/utils/extractContacts.d.ts +1 -1
- package/lib/utils/extractContacts.js +4 -2
- package/lib/utils/extractContacts.js.map +1 -1
- package/lib/utils/findBestFaviconURL.js +47 -15
- package/lib/utils/findBestFaviconURL.js.map +1 -1
- package/lib/utils/getBusinessCategoryFromSchema.d.ts +2 -2
- package/lib/utils/getBusinessCategoryFromSchema.js +16 -16
- package/lib/utils/getBusinessCategoryFromSchema.js.map +1 -1
- package/lib/utils/getBusinessData.d.ts +4 -4
- package/lib/utils/getBusinessData.js +63 -26
- package/lib/utils/getBusinessData.js.map +1 -1
- package/lib/utils/getSiteMap.js +14 -12
- package/lib/utils/getSiteMap.js.map +1 -1
- package/package.json +18 -18
|
@@ -34,18 +34,20 @@ class GenerativeAIService {
|
|
|
34
34
|
let maxTokens = webPrompt.maxTokens;
|
|
35
35
|
let tokens = webPrompt.tokens;
|
|
36
36
|
(0, stentor_logger_1.log)().info(`Raw markdown tokens: ${tokens} max: ${maxTokens}`);
|
|
37
|
-
// we will access later
|
|
37
|
+
// we will access later
|
|
38
38
|
let aboutUsMarkdown = "";
|
|
39
39
|
let aboutUsPrompt = undefined;
|
|
40
40
|
// first try if over max tokens
|
|
41
41
|
if (maxTokens < 0 && (pages === null || pages === void 0 ? void 0 : pages.aboutUs)) {
|
|
42
42
|
const aboutUs = pages.aboutUs;
|
|
43
|
-
// load about us
|
|
43
|
+
// load about us
|
|
44
44
|
(0, stentor_logger_1.log)().info(`Raw markdown has too many tokens, attempting to load about us page at ${aboutUs}...`);
|
|
45
|
-
yield this.page.goto(aboutUs, { waitUntil:
|
|
45
|
+
yield this.page.goto(aboutUs, { waitUntil: "networkidle2", timeout: 5000 });
|
|
46
46
|
(0, stentor_logger_1.log)().info(`Loaded...`);
|
|
47
47
|
const aboutUsHTML = yield this.page.content();
|
|
48
|
-
aboutUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
48
|
+
aboutUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
49
|
+
stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"],
|
|
50
|
+
}).convert(aboutUsHTML, true);
|
|
49
51
|
aboutUsPrompt = (0, getPrompt_1.getWebMarkdownPrompt)(aboutUsMarkdown, url);
|
|
50
52
|
(0, stentor_logger_1.log)().info(`About Us tokens: ${aboutUsPrompt.tokens} max: ${aboutUsPrompt.maxTokens}`);
|
|
51
53
|
if (aboutUsPrompt.maxTokens > 0) {
|
|
@@ -55,15 +57,18 @@ class GenerativeAIService {
|
|
|
55
57
|
tokens = aboutUsPrompt.tokens;
|
|
56
58
|
}
|
|
57
59
|
(0, stentor_logger_1.log)().info(`Returning to original page ${url}...`);
|
|
58
|
-
yield this.page.goto(url, { waitUntil:
|
|
60
|
+
yield this.page.goto(url, { waitUntil: "networkidle2", timeout: 5000 });
|
|
59
61
|
(0, stentor_logger_1.log)().info(`Loaded...`);
|
|
60
62
|
}
|
|
61
63
|
// try the article approach
|
|
62
64
|
if (maxTokens < 0 && url) {
|
|
63
65
|
(0, stentor_logger_1.log)().info(`Raw markdown has too many tokens, attempting to load original page as an article ${url}...`);
|
|
64
|
-
yield this.page.goto(url, { waitUntil:
|
|
66
|
+
yield this.page.goto(url, { waitUntil: "networkidle2", timeout: 5000 });
|
|
65
67
|
const mainHTML = yield this.page.content();
|
|
66
|
-
const mainArticleMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
68
|
+
const mainArticleMarkdown = new arachne_parsers_1.ArachneHTMLtoText({
|
|
69
|
+
stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"],
|
|
70
|
+
articleFocus: true,
|
|
71
|
+
}).convert(mainHTML, true);
|
|
67
72
|
const articlePrompt = (0, getPrompt_1.getWebMarkdownPrompt)(mainArticleMarkdown, url);
|
|
68
73
|
(0, stentor_logger_1.log)().info(`Article tokens: ${articlePrompt.tokens} max: ${articlePrompt.maxTokens}`);
|
|
69
74
|
if (articlePrompt.maxTokens > 0) {
|
|
@@ -117,10 +122,10 @@ class GenerativeAIService {
|
|
|
117
122
|
(0, stentor_logger_1.log)().info(`Final prompt is ${tokens} tokens, setting max_tokens to ${maxTokens}.`);
|
|
118
123
|
}
|
|
119
124
|
const options = {
|
|
120
|
-
method:
|
|
125
|
+
method: "POST",
|
|
121
126
|
headers: {
|
|
122
|
-
|
|
123
|
-
|
|
127
|
+
"Content-Type": "application/json",
|
|
128
|
+
Authorization: `Bearer ${this.key}`,
|
|
124
129
|
},
|
|
125
130
|
body: JSON.stringify({
|
|
126
131
|
model: "gpt-4o-mini-2024-07-18",
|
|
@@ -128,20 +133,21 @@ class GenerativeAIService {
|
|
|
128
133
|
messages: [
|
|
129
134
|
{
|
|
130
135
|
role: "system",
|
|
131
|
-
content: prompt
|
|
132
|
-
}
|
|
136
|
+
content: prompt,
|
|
137
|
+
},
|
|
133
138
|
],
|
|
134
139
|
temperature: 0.25,
|
|
135
140
|
max_tokens: maxTokens,
|
|
136
141
|
top_p: 1,
|
|
137
142
|
frequency_penalty: 0,
|
|
138
|
-
presence_penalty: 0
|
|
139
|
-
})
|
|
143
|
+
presence_penalty: 0,
|
|
144
|
+
}),
|
|
140
145
|
};
|
|
141
146
|
// eslint-disable-next-line no-console
|
|
142
147
|
console.time("open_ai_web");
|
|
143
148
|
let status = 999;
|
|
144
|
-
const openAIResponse = yield fetch(
|
|
149
|
+
const openAIResponse = yield fetch("https://api.openai.com/v1/chat/completions", options)
|
|
150
|
+
.then((response) => {
|
|
145
151
|
status = response.status;
|
|
146
152
|
if (status === 200) {
|
|
147
153
|
return response.json();
|
|
@@ -151,7 +157,8 @@ class GenerativeAIService {
|
|
|
151
157
|
return response.json();
|
|
152
158
|
}
|
|
153
159
|
return response;
|
|
154
|
-
})
|
|
160
|
+
})
|
|
161
|
+
.finally(() => {
|
|
155
162
|
// eslint-disable-next-line no-console
|
|
156
163
|
console.timeEnd("open_ai_web");
|
|
157
164
|
});
|
|
@@ -164,7 +171,7 @@ class GenerativeAIService {
|
|
|
164
171
|
// quick common error check, look for trailing commas before closing bracket
|
|
165
172
|
// Regex to find trailing commas before a closing brace or bracket
|
|
166
173
|
const trailingComma = /,(?=\s*[}\]])/g;
|
|
167
|
-
completion = completion.replace(trailingComma,
|
|
174
|
+
completion = completion.replace(trailingComma, "");
|
|
168
175
|
aiData = JSON.parse(completion);
|
|
169
176
|
(0, stentor_logger_1.log)().debug(`Parsed AI response....`);
|
|
170
177
|
(0, stentor_logger_1.log)().debug(JSON.stringify(aiData, undefined, 2));
|
|
@@ -186,12 +193,16 @@ class GenerativeAIService {
|
|
|
186
193
|
if (aiData && aiData.phone_numbers && (0, stentor_utils_1.existsAndNotEmpty)(aiData.phone_numbers)) {
|
|
187
194
|
const preLength = aiData.phone_numbers.length;
|
|
188
195
|
// clean for hallucinations
|
|
189
|
-
// numbers like 123-45678 or numbers like 555-
|
|
190
|
-
const mostLikelyRealPhoneNumbers = aiData.phone_numbers
|
|
196
|
+
// numbers like 123-45678 or numbers like 555-
|
|
197
|
+
const mostLikelyRealPhoneNumbers = aiData.phone_numbers
|
|
198
|
+
.filter((number) => !!number.number)
|
|
199
|
+
.filter((number) => !(0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
191
200
|
const postLength = mostLikelyRealPhoneNumbers.length;
|
|
192
201
|
if (postLength < preLength) {
|
|
193
202
|
const removedNumbers = aiData.phone_numbers.filter((number) => (0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
194
|
-
(0, stentor_logger_1.log)().info(`Removed ${preLength - postLength} hallucinated phone numbers: ${removedNumbers
|
|
203
|
+
(0, stentor_logger_1.log)().info(`Removed ${preLength - postLength} hallucinated phone numbers: ${removedNumbers
|
|
204
|
+
.map((number) => number.number)
|
|
205
|
+
.join(", ")}`);
|
|
195
206
|
}
|
|
196
207
|
aiData.phone_numbers = mostLikelyRealPhoneNumbers;
|
|
197
208
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GenerativeAIService.js","sourceRoot":"","sources":["../src/GenerativeAIService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,2DAA8E;AAC9E,mDAAqC;AAGrC,yDAAsD;AACtD,iDAAsF;AACtF,uDAAgE;AAChE,iDAAkD;AAoBlD,MAAa,mBAAmB;
|
|
1
|
+
{"version":3,"file":"GenerativeAIService.js","sourceRoot":"","sources":["../src/GenerativeAIService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,2DAA8E;AAC9E,mDAAqC;AAGrC,yDAAsD;AACtD,iDAAsF;AACtF,uDAAgE;AAChE,iDAAkD;AAoBlD,MAAa,mBAAmB;IAK5B,YAAY,KAAyC;QACjD,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEK,YAAY,CAAC,QAAgB,EAAE,IAAmB;;YACpD,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;YAEnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YAE9C,MAAM,KAAK,GAAG,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,KAAK,CAAC;YAC1B,MAAM,aAAa,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,aAAa,CAAC;YAC5C,MAAM,wBAAwB,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,wBAAwB,CAAC;YAElE,IAAI,MAAM,GAA0B,SAAS,CAAC;YAE9C,MAAM,SAAS,GAAG,IAAA,gCAAoB,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEtD,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;YACtC,IAAI,SAAS,GAAW,SAAS,CAAC,SAAS,CAAC;YAC5C,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;YAEtC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,wBAAwB,MAAM,SAAS,SAAS,EAAE,CAAC,CAAC;YAE/D,uBAAuB;YACvB,IAAI,eAAe,GAAW,EAAE,CAAC;YACjC,IAAI,aAAa,GAAuB,SAAS,CAAC;YAElD,+BAA+B;YAC/B,IAAI,SAAS,GAAG,CAAC,KAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,CAAA,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;gBAC9B,gBAAgB;gBAChB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,yEAAyE,OAAO,KAAK,CAAC,CAAC;gBAClG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5E,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAE9C,eAAe,GAAG,IAAI,mCAAiB,CAAC;oBACpC,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;iBACvE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBAE9B,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBAE3D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;oBACnC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;gBACD,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,CAAC;YAED,2BAA2B;YAC3B,IAAI,SAAS,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;gBACvB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oFAAoF,GAAG,KAAK,CAAC,CAAC;gBACzG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAE3C,MAAM,mBAAmB,GAAG,IAAI,mCAAiB,CAAC;oBAC9C,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;oBACpE,YAAY,EAAE,IAAI;iBACrB,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAE3B,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;gBAErE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEtF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAChD,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,iBAAiB;YACjB,IAAI,SAAS,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,wBAAwB,CAAC,EAAE,CAAC;gBAC/D,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,oBAAoB,SAAS,oEAAoE,CACpG,CAAC;gBACF,4CAA4C;gBAE5C,MAAM,gBAAgB,GAAG,IAAA,+BAAmB,EAAC,aAAa,IAAI,wBAAwB,EAAE,GAAG,CAAC,CAAC;gBAE7F,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,gBAAgB,CAAC,MAAM,SAAS,gBAAgB,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEhG,IAAI,gBAAgB,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBACjC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;oBACpE,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;oBACjC,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;oBACvC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;gBACrC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;gBACjE,sDAAsD;gBACtD,MAAM,eAAe,GAAG,IAAA,6BAAa,EAAC,QAAQ,CAAC,CAAC;gBAEhD,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBAEjE,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;oBAClE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;gBACpD,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;gBAEhF,MAAM,sBAAsB,GAAG,IAAA,6BAAa,EAAC,eAAe,CAAC,CAAC;gBAE9D,MAAM,oBAAoB,GAAG,IAAA,gCAAoB,EAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;gBAE/E,IAAI,oBAAoB,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;oBAC3E,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC;oBACrC,SAAS,GAAG,oBAAoB,CAAC,SAAS,CAAC;oBAC3C,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC;gBACzC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gCAAgC,GAAG,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBAC7E,OAAO,SAAS,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACJ,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,MAAM,kCAAkC,SAAS,GAAG,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,OAAO,GAAgB;gBACzB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACL,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE;iBACtC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACjB,KAAK,EAAE,wBAAwB;oBAC/B,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;oBACxC,QAAQ,EAAE;wBACN;4BACI,IAAI,EAAE,QAAQ;4BACd,OAAO,EAAE,MAAM;yBAClB;qBACJ;oBACD,WAAW,EAAE,IAAI;oBACjB,UAAU,EAAE,SAAS;oBACrB,KAAK,EAAE,CAAC;oBACR,iBAAiB,EAAE,CAAC;oBACpB,gBAAgB,EAAE,CAAC;iBACtB,CAAC;aACL,CAAC;YAEF,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5B,IAAI,MAAM,GAAW,GAAG,CAAC;YACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE,OAAO,CAAC;iBACpF,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACf,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzB,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACjB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC3B,CAAC;qBAAM,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACxB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;oBACpC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC3B,CAAC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACV,sCAAsC;gBACtC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YAEP,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;gBACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CACxH,CAAC;gBAEF,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAE9C,IAAI,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC7C,IAAI,CAAC;oBACD,4EAA4E;oBAC5E,kEAAkE;oBAClE,MAAM,aAAa,GAAG,gBAAgB,CAAC;oBACvC,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;oBAEnD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAChC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;oBACtC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;oBAC3C,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;gBACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;YAED,wEAAwE;YACxE,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,IAAA,iCAAiB,EAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5D,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrE,MAAM,CAAC,KAAK,GAAG,cAAc,CAAC;YAClC,CAAC;YAED,IAAI,MAAM,IAAI,MAAM,CAAC,aAAa,IAAI,IAAA,iCAAiB,EAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5E,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC;gBAC9C,2BAA2B;gBAC3B,8CAA8C;gBAE9C,MAAM,0BAA0B,GAAG,MAAM,CAAC,aAAa;qBAClD,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;qBACnC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAA,yCAAyB,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEnE,MAAM,UAAU,GAAG,0BAA0B,CAAC,MAAM,CAAC;gBAErD,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;oBACzB,MAAM,cAAc,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC1D,IAAA,yCAAyB,EAAC,MAAM,CAAC,MAAM,CAAC,CAC3C,CAAC;oBACF,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,WAAW,SAAS,GAAG,UAAU,gCAAgC,cAAc;yBAC1E,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;yBAC9B,IAAI,CAAC,IAAI,CAAC,EAAE,CACpB,CAAC;gBACN,CAAC;gBAED,MAAM,CAAC,aAAa,GAAG,0BAA0B,CAAC;YACtD,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;CACJ;AAnPD,kDAmPC"}
|
package/lib/models.d.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
-
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness, Organization } from "schema-dts";
|
|
3
3
|
import { SiteMapData } from "./utils/getSiteMap";
|
|
4
4
|
export interface WebsiteSchema {
|
|
5
5
|
localBusiness?: LocalBusiness;
|
|
6
6
|
homeConstructionBusiness?: HomeAndConstructionBusiness;
|
|
7
|
+
organization?: Organization;
|
|
7
8
|
}
|
|
8
9
|
export interface WebsiteDescriptiveData {
|
|
9
10
|
/**
|
|
@@ -111,6 +112,10 @@ export interface BusinessData {
|
|
|
111
112
|
* Name of the business
|
|
112
113
|
*/
|
|
113
114
|
name?: string;
|
|
115
|
+
/**
|
|
116
|
+
* A square image that most likely is the logo.
|
|
117
|
+
*/
|
|
118
|
+
logo?: string;
|
|
114
119
|
/**
|
|
115
120
|
* Category of the business.
|
|
116
121
|
*/
|
|
@@ -21,7 +21,7 @@ export declare function hasEmail(text: string): boolean;
|
|
|
21
21
|
*/
|
|
22
22
|
export declare function hasPhoneNumber(text: string): boolean;
|
|
23
23
|
/**
|
|
24
|
-
* Checks if the given text contains emails or phone numbers.
|
|
24
|
+
* Checks if the given text contains emails or phone numbers (filtering out hallucinated numbers).
|
|
25
25
|
*
|
|
26
26
|
* @param text The text to be checked.
|
|
27
27
|
* @returns An object containing arrays of found emails and phone numbers.
|
|
@@ -39,7 +39,7 @@ function hasPhoneNumber(text) {
|
|
|
39
39
|
return extractContacts(text).phoneNumbers.length > 0;
|
|
40
40
|
}
|
|
41
41
|
/**
|
|
42
|
-
* Checks if the given text contains emails or phone numbers.
|
|
42
|
+
* Checks if the given text contains emails or phone numbers (filtering out hallucinated numbers).
|
|
43
43
|
*
|
|
44
44
|
* @param text The text to be checked.
|
|
45
45
|
* @returns An object containing arrays of found emails and phone numbers.
|
|
@@ -49,7 +49,9 @@ function extractContacts(text) {
|
|
|
49
49
|
const phoneRegex = /\b\d{3}[-.]?\d{3}[-.]?\d{4}\b/g;
|
|
50
50
|
const foundEmails = text.match(emailRegex) || [];
|
|
51
51
|
// remove any emails that end in .png or .jpg, .webp, jpeg, avif, svg, gif
|
|
52
|
-
const filtered = foundEmails
|
|
52
|
+
const filtered = foundEmails
|
|
53
|
+
.filter((email) => !email.match(/\.(png|jpg|webp|jpeg|avif|svg|bmp|gif)$/))
|
|
54
|
+
.filter((email) => !(0, arachne_utils_1.isHallucinatedPhoneNumber)(email));
|
|
53
55
|
// and dedupe emails
|
|
54
56
|
const emails = Array.from(new Set(filtered));
|
|
55
57
|
const phoneNumbersMatches = text.match(phoneRegex) || [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractContacts.js","sourceRoot":"","sources":["../../src/utils/extractContacts.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAUlC,wCASC;AAQD,4BAEC;AAQD,wCAEC;AAQD,
|
|
1
|
+
{"version":3,"file":"extractContacts.js","sourceRoot":"","sources":["../../src/utils/extractContacts.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAUlC,wCASC;AAQD,4BAEC;AAQD,wCAEC;AAQD,0CAsBC;AAnED,uDAAgE;AAEhE;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,uCAAuC;IAEvC,MAAM,iBAAiB,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;IAEhD,IAAI,iBAAiB,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnF,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,QAAQ,CAAC,IAAY;IACjC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;AACzD,CAAC;AAED;;;;;GAKG;AACH,SAAgB,eAAe,CAAC,IAAY;IACxC,MAAM,UAAU,GAAG,sDAAsD,CAAC;IAC1E,MAAM,UAAU,GAAG,gCAAgC,CAAC;IAEpD,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IAEjD,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,WAAW;SACvB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;SAC1E,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,IAAA,yCAAyB,EAAC,KAAK,CAAC,CAAC,CAAC;IAE1D,oBAAoB;IACpB,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IAE7C,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;IAEzD,kDAAkD;IAClD,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAC,MAAM,CAChE,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAA,yCAAyB,EAAC,MAAM,CAAC,CACjD,CAAC;IAEF,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AACpC,CAAC"}
|
|
@@ -10,7 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
10
10
|
};
|
|
11
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
12
|
exports.findBestFaviconURL = findBestFaviconURL;
|
|
13
|
-
const getDOMElementHREF_1 = require("./getDOMElementHREF");
|
|
14
13
|
/**
|
|
15
14
|
* For the given page, find the best favicon URL.
|
|
16
15
|
*
|
|
@@ -22,19 +21,52 @@ const getDOMElementHREF_1 = require("./getDOMElementHREF");
|
|
|
22
21
|
*/
|
|
23
22
|
function findBestFaviconURL(page, pageUrl) {
|
|
24
23
|
return __awaiter(this, void 0, void 0, function* () {
|
|
25
|
-
|
|
26
|
-
const
|
|
27
|
-
`link[rel="icon"]`,
|
|
28
|
-
`link[rel="shortcut icon"]`
|
|
29
|
-
];
|
|
24
|
+
var _a;
|
|
25
|
+
const rootUrl = new URL(pageUrl).protocol + "//" + new URL(pageUrl).host;
|
|
30
26
|
let faviconUrlFromDocument = null;
|
|
31
|
-
for
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
27
|
+
// look for apple-touch-icon && msapplication-TileImage, these are high res icons
|
|
28
|
+
const highResIcons = yield page.evaluate(() => {
|
|
29
|
+
const icons = document.querySelectorAll(`link[rel="apple-touch-icon"], link[rel="msapplication-TileImage"], link[rel="icon"], link[rel="shortcut icon"]`);
|
|
30
|
+
// return an array of hrefs & sizes
|
|
31
|
+
return Array.from(icons).map((icon) => {
|
|
32
|
+
return {
|
|
33
|
+
href: icon.getAttribute("href"),
|
|
34
|
+
sizes: icon.getAttribute("sizes"),
|
|
35
|
+
};
|
|
36
|
+
});
|
|
37
|
+
});
|
|
38
|
+
// now, look for the highest res icon, not more than 192x192, if we can't find one we just default to the first one we find
|
|
39
|
+
let bestIcon = null;
|
|
40
|
+
for (const icon of highResIcons) {
|
|
41
|
+
if (icon.sizes) {
|
|
42
|
+
const sizes = icon.sizes.split("x");
|
|
43
|
+
if (sizes.length === 2) {
|
|
44
|
+
const width = parseInt(sizes[0]);
|
|
45
|
+
const height = parseInt(sizes[1]);
|
|
46
|
+
if (width <= 192 && height <= 192) {
|
|
47
|
+
if (bestIcon === null) {
|
|
48
|
+
bestIcon = icon;
|
|
49
|
+
}
|
|
50
|
+
else {
|
|
51
|
+
const bestIconSizes = (_a = bestIcon.sizes) === null || _a === void 0 ? void 0 : _a.split("x");
|
|
52
|
+
if (bestIconSizes) {
|
|
53
|
+
const bestWidth = parseInt(bestIconSizes[0]);
|
|
54
|
+
const bestHeight = parseInt(bestIconSizes[1]);
|
|
55
|
+
if (width > bestWidth && height > bestHeight) {
|
|
56
|
+
bestIcon = icon;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
35
62
|
}
|
|
36
|
-
|
|
37
|
-
|
|
63
|
+
}
|
|
64
|
+
if (bestIcon) {
|
|
65
|
+
faviconUrlFromDocument = bestIcon.href;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
// no high res icon found, just use the first one we found
|
|
69
|
+
faviconUrlFromDocument = highResIcons[0].href;
|
|
38
70
|
}
|
|
39
71
|
if (faviconUrlFromDocument === null) {
|
|
40
72
|
// No favicon link found in document, best URL is likley favicon.ico at root
|
|
@@ -47,13 +79,13 @@ function findBestFaviconURL(page, pageUrl) {
|
|
|
47
79
|
// absolute url
|
|
48
80
|
return faviconUrlFromDocument;
|
|
49
81
|
}
|
|
50
|
-
else if (faviconUrlFromDocument.substr(0, 1) ===
|
|
82
|
+
else if (faviconUrlFromDocument.substr(0, 1) === "/") {
|
|
51
83
|
// favicon relative to root
|
|
52
|
-
return
|
|
84
|
+
return rootUrl + faviconUrlFromDocument;
|
|
53
85
|
}
|
|
54
86
|
else {
|
|
55
87
|
// favicon relative to current (pageUrl) URL
|
|
56
|
-
return
|
|
88
|
+
return pageUrl + "/" + faviconUrlFromDocument;
|
|
57
89
|
}
|
|
58
90
|
});
|
|
59
91
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"findBestFaviconURL.js","sourceRoot":"","sources":["../../src/utils/findBestFaviconURL.ts"],"names":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"findBestFaviconURL.js","sourceRoot":"","sources":["../../src/utils/findBestFaviconURL.ts"],"names":[],"mappings":";;;;;;;;;;;AAYA,gDAuEC;AAhFD;;;;;;;;GAQG;AACH,SAAsB,kBAAkB,CAAC,IAAiB,EAAE,OAAe;;;QACvE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC;QAEzE,IAAI,sBAAsB,GAAkB,IAAI,CAAC;QAEjD,iFAAiF;QAEjF,MAAM,YAAY,GAAqD,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC5F,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CACnC,gHAAgH,CACnH,CAAC;YAEF,mCAAmC;YACnC,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;gBAClC,OAAO;oBACH,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;oBAC/B,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC;iBACpC,CAAC;YACN,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,2HAA2H;QAC3H,IAAI,QAAQ,GAA0D,IAAI,CAAC;QAC3E,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBACb,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBACpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACrB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBACjC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClC,IAAI,KAAK,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;wBAChC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;4BACpB,QAAQ,GAAG,IAAI,CAAC;wBACpB,CAAC;6BAAM,CAAC;4BACJ,MAAM,aAAa,GAAG,MAAA,QAAQ,CAAC,KAAK,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;4BACjD,IAAI,aAAa,EAAE,CAAC;gCAChB,MAAM,SAAS,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gCAC7C,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gCAC9C,IAAI,KAAK,GAAG,SAAS,IAAI,MAAM,GAAG,UAAU,EAAE,CAAC;oCAC3C,QAAQ,GAAG,IAAI,CAAC;gCACpB,CAAC;4BACL,CAAC;wBACL,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACX,sBAAsB,GAAG,QAAQ,CAAC,IAAI,CAAC;QAC3C,CAAC;aAAM,CAAC;YACJ,0DAA0D;YAC1D,sBAAsB,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClD,CAAC;QAED,IAAI,sBAAsB,KAAK,IAAI,EAAE,CAAC;YAClC,4EAA4E;YAC5E,OAAO,OAAO,GAAG,cAAc,CAAC;QACpC,CAAC;QAED,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/C,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,GAAG,sBAAsB,EAAE,CAAC;QACnE,CAAC;aAAM,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;YACxD,eAAe;YACf,OAAO,sBAAsB,CAAC;QAClC,CAAC;aAAM,IAAI,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YACrD,2BAA2B;YAC3B,OAAO,OAAO,GAAG,sBAAsB,CAAC;QAC5C,CAAC;aAAM,CAAC;YACJ,4CAA4C;YAC5C,OAAO,OAAO,GAAG,GAAG,GAAG,sBAAsB,CAAC;QAClD,CAAC;IACL,CAAC;CAAA"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
-
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness, Organization } from "schema-dts";
|
|
3
3
|
import { BusinessCategory } from "../models";
|
|
4
|
-
export declare function getBusinessCategoryFromSchema(schema: LocalBusiness | HomeAndConstructionBusiness | undefined): BusinessCategory | undefined;
|
|
4
|
+
export declare function getBusinessCategoryFromSchema(schema: LocalBusiness | HomeAndConstructionBusiness | Organization | undefined): BusinessCategory | undefined;
|
|
@@ -15,28 +15,28 @@ function getBusinessCategoryFromSchema(schema) {
|
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
17
|
switch (type) {
|
|
18
|
-
case
|
|
19
|
-
return
|
|
20
|
-
case
|
|
21
|
-
return
|
|
18
|
+
case "ProfessionalService":
|
|
19
|
+
return "consulting";
|
|
20
|
+
case "FinancialService":
|
|
21
|
+
return "financial-services";
|
|
22
22
|
case "Electrician":
|
|
23
23
|
case "GeneralContractor":
|
|
24
24
|
case "Plumber":
|
|
25
25
|
case "HVACBusiness":
|
|
26
26
|
case "RoofingContractor":
|
|
27
27
|
case "HousePainter":
|
|
28
|
-
case
|
|
29
|
-
return
|
|
30
|
-
case
|
|
31
|
-
return
|
|
32
|
-
case
|
|
33
|
-
return
|
|
34
|
-
case
|
|
35
|
-
return
|
|
36
|
-
case
|
|
37
|
-
return
|
|
38
|
-
case
|
|
39
|
-
return
|
|
28
|
+
case "HomeAndConstructionBusiness":
|
|
29
|
+
return "home-services-contractors";
|
|
30
|
+
case "InsuranceAgency":
|
|
31
|
+
return "insurance";
|
|
32
|
+
case "LegalService":
|
|
33
|
+
return "legal-services";
|
|
34
|
+
case "RealEstateAgent":
|
|
35
|
+
return "real-estate";
|
|
36
|
+
case "InternetCafe":
|
|
37
|
+
return "technology-services";
|
|
38
|
+
case "TravelAgency":
|
|
39
|
+
return "travel";
|
|
40
40
|
default:
|
|
41
41
|
return undefined;
|
|
42
42
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getBusinessCategoryFromSchema.js","sourceRoot":"","sources":["../../src/utils/getBusinessCategoryFromSchema.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAMlC,
|
|
1
|
+
{"version":3,"file":"getBusinessCategoryFromSchema.js","sourceRoot":"","sources":["../../src/utils/getBusinessCategoryFromSchema.ts"],"names":[],"mappings":";AAAA,kCAAkC;;AAMlC,sEAuCC;AAzCD,2DAAgD;AAEhD,SAAgB,6BAA6B,CACzC,MAA8E;IAE9E,IAAI,IAAI,GAAW,EAAE,CAAC;IACtB,IAAI,IAAA,yBAAO,EAAC,MAAM,CAAC,EAAE,CAAC;QAClB,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC5B,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACJ,IAAI,GAAG,UAAU,CAAC;QACtB,CAAC;IACL,CAAC;IAED,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,qBAAqB;YACtB,OAAO,YAAY,CAAC;QACxB,KAAK,kBAAkB;YACnB,OAAO,oBAAoB,CAAC;QAChC,KAAK,aAAa,CAAC;QACnB,KAAK,mBAAmB,CAAC;QACzB,KAAK,SAAS,CAAC;QACf,KAAK,cAAc,CAAC;QACpB,KAAK,mBAAmB,CAAC;QACzB,KAAK,cAAc,CAAC;QACpB,KAAK,6BAA6B;YAC9B,OAAO,2BAA2B,CAAC;QACvC,KAAK,iBAAiB;YAClB,OAAO,WAAW,CAAC;QACvB,KAAK,cAAc;YACf,OAAO,gBAAgB,CAAC;QAC5B,KAAK,iBAAiB;YAClB,OAAO,aAAa,CAAC;QACzB,KAAK,cAAc;YACf,OAAO,qBAAqB,CAAC;QACjC,KAAK,cAAc;YACf,OAAO,QAAQ,CAAC;QACpB;YACI,OAAO,SAAS,CAAC;IACzB,CAAC;AACL,CAAC"}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
import {
|
|
3
|
-
import { BusinessData } from "../models";
|
|
1
|
+
import { HomeAndConstructionBusiness, LocalBusiness, Organization } from "schema-dts";
|
|
2
|
+
import { BusinessData, WebsiteSchema } from "../models";
|
|
4
3
|
/**
|
|
5
4
|
* Will attempt to extract business data from a schema.org schema.
|
|
6
5
|
*
|
|
7
6
|
* @param schema
|
|
8
7
|
* @returns
|
|
9
8
|
*/
|
|
10
|
-
export declare function getBusinessData(schema: LocalBusiness | HomeAndConstructionBusiness | undefined): Partial<BusinessData
|
|
9
|
+
export declare function getBusinessData(schema: LocalBusiness | HomeAndConstructionBusiness | Organization | undefined): Promise<Partial<BusinessData>>;
|
|
10
|
+
export declare function getBusinessDataFromSchema(schema: WebsiteSchema): Promise<Partial<BusinessData>>;
|
|
@@ -1,7 +1,21 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.getBusinessData = getBusinessData;
|
|
13
|
+
exports.getBusinessDataFromSchema = getBusinessDataFromSchema;
|
|
14
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
4
15
|
const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
16
|
+
const stentor_logger_1 = require("stentor-logger");
|
|
17
|
+
const image_1 = require("./image");
|
|
18
|
+
const getBusinessCategoryFromSchema_1 = require("./getBusinessCategoryFromSchema");
|
|
5
19
|
/**
|
|
6
20
|
* Will attempt to extract business data from a schema.org schema.
|
|
7
21
|
*
|
|
@@ -9,37 +23,60 @@ const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
|
9
23
|
* @returns
|
|
10
24
|
*/
|
|
11
25
|
function getBusinessData(schema) {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const partial = {};
|
|
16
|
-
if (schema && typeof schema === "object") {
|
|
17
|
-
// try to get it from JSON?
|
|
18
|
-
if (typeof schema.name === "string") {
|
|
19
|
-
partial.name = schema === null || schema === void 0 ? void 0 : schema.name;
|
|
26
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
27
|
+
if (!schema) {
|
|
28
|
+
return {};
|
|
20
29
|
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
const partial = {};
|
|
31
|
+
if (schema && typeof schema === "object") {
|
|
32
|
+
// try to get it from JSON?
|
|
33
|
+
if (typeof schema.name === "string") {
|
|
34
|
+
partial.name = schema === null || schema === void 0 ? void 0 : schema.name;
|
|
35
|
+
}
|
|
36
|
+
if (typeof schema.telephone === "string") {
|
|
37
|
+
partial.phoneNumbers = [{ number: schema.telephone }];
|
|
38
|
+
}
|
|
39
|
+
if (typeof schema.logo === "string") {
|
|
40
|
+
try {
|
|
41
|
+
const square = yield (0, image_1.imageIsSquare)(schema.logo, 75);
|
|
42
|
+
if (square) {
|
|
43
|
+
(0, stentor_logger_1.log)().debug(`Found a square logo from json+ld ${schema.logo}`);
|
|
44
|
+
partial.logo = schema.logo;
|
|
45
|
+
}
|
|
33
46
|
}
|
|
34
|
-
|
|
35
|
-
|
|
47
|
+
catch (e) {
|
|
48
|
+
(0, stentor_logger_1.log)().debug(`Error determining if logo is square ${e}`);
|
|
36
49
|
}
|
|
37
|
-
|
|
38
|
-
|
|
50
|
+
}
|
|
51
|
+
if ((0, arachne_parsers_1.isTyped)(schema.address)) {
|
|
52
|
+
const address = schema.address;
|
|
53
|
+
if ((0, arachne_parsers_1.isPostalAddress)(address)) {
|
|
54
|
+
const bizAddress = {};
|
|
55
|
+
if (address.addressLocality && typeof address.addressLocality === "string") {
|
|
56
|
+
bizAddress.locality = address.addressLocality;
|
|
57
|
+
}
|
|
58
|
+
if (address.streetAddress && typeof address.streetAddress === "string") {
|
|
59
|
+
bizAddress.streetAddress = address.streetAddress;
|
|
60
|
+
}
|
|
61
|
+
if (address.postalCode && typeof address.postalCode === "string") {
|
|
62
|
+
bizAddress.postalCode = address.postalCode;
|
|
63
|
+
}
|
|
64
|
+
if (Object.keys(bizAddress).length > 0) {
|
|
65
|
+
partial.addresses = [bizAddress];
|
|
66
|
+
}
|
|
39
67
|
}
|
|
40
68
|
}
|
|
41
69
|
}
|
|
42
|
-
|
|
43
|
-
|
|
70
|
+
partial.category = (0, getBusinessCategoryFromSchema_1.getBusinessCategoryFromSchema)(schema);
|
|
71
|
+
return partial;
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
function getBusinessDataFromSchema(schema) {
|
|
75
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
const localBusiness = yield getBusinessData(schema.localBusiness);
|
|
77
|
+
const organization = yield getBusinessData(schema.organization);
|
|
78
|
+
const homeAndConstructionBusiness = yield getBusinessData(schema.homeConstructionBusiness);
|
|
79
|
+
return Object.assign(Object.assign(Object.assign({}, homeAndConstructionBusiness), localBusiness), organization);
|
|
80
|
+
});
|
|
44
81
|
}
|
|
45
82
|
//# sourceMappingURL=getBusinessData.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"getBusinessData.js","sourceRoot":"","sources":["../../src/utils/getBusinessData.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"getBusinessData.js","sourceRoot":"","sources":["../../src/utils/getBusinessData.ts"],"names":[],"mappings":";;;;;;;;;;;AAgBA,0CA0DC;AAED,8DAUC;AAtFD,kCAAkC;AAClC,2DAAiE;AAEjE,mDAAqC;AAIrC,mCAAwC;AACxC,mFAAgF;AAEhF;;;;;GAKG;AACH,SAAsB,eAAe,CACjC,MAA8E;;QAE9E,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAA0B,EAAE,CAAC;QAE1C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;YACvC,2BAA2B;YAC3B,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CAAC;YAChC,CAAC;YAED,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;gBACvC,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAClC,IAAI,CAAC;oBACD,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBACpD,IAAI,MAAM,EAAE,CAAC;wBACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;wBAC/D,OAAO,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;oBAC/B,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uCAAuC,CAAC,EAAE,CAAC,CAAC;gBAC5D,CAAC;YACL,CAAC;YAED,IAAI,IAAA,yBAAO,EAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;gBAE/B,IAAI,IAAA,iCAAe,EAAC,OAAO,CAAC,EAAE,CAAC;oBAC3B,MAAM,UAAU,GAAoB,EAAE,CAAC;oBACvC,IAAI,OAAO,CAAC,eAAe,IAAI,OAAO,OAAO,CAAC,eAAe,KAAK,QAAQ,EAAE,CAAC;wBACzE,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,eAAe,CAAC;oBAClD,CAAC;oBAED,IAAI,OAAO,CAAC,aAAa,IAAI,OAAO,OAAO,CAAC,aAAa,KAAK,QAAQ,EAAE,CAAC;wBACrE,UAAU,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;oBACrD,CAAC;oBAED,IAAI,OAAO,CAAC,UAAU,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;wBAC/D,UAAU,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;oBAC/C,CAAC;oBAED,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACrC,OAAO,CAAC,SAAS,GAAG,CAAC,UAAU,CAAC,CAAC;oBACrC,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,CAAC,QAAQ,GAAG,IAAA,6DAA6B,EAAC,MAAM,CAAC,CAAC;QAEzD,OAAO,OAAO,CAAC;IACnB,CAAC;CAAA;AAED,SAAsB,yBAAyB,CAAC,MAAqB;;QACjE,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QAClE,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAChE,MAAM,2BAA2B,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,wBAAwB,CAAC,CAAC;QAE3F,qDACO,2BAA2B,GAC3B,aAAa,GACb,YAAY,EACjB;IACN,CAAC;CAAA"}
|
package/lib/utils/getSiteMap.js
CHANGED
|
@@ -61,19 +61,21 @@ const fetchSitemap = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
|
61
61
|
}, TIMEOUT);
|
|
62
62
|
// this fails
|
|
63
63
|
try {
|
|
64
|
-
const text = yield fetch(url, { signal })
|
|
64
|
+
const text = yield fetch(url, { signal })
|
|
65
|
+
.then((response) => {
|
|
65
66
|
if (!response.ok) {
|
|
66
67
|
throw new Error(`Sitemap returned non-200 status code: ${response.status}`);
|
|
67
68
|
}
|
|
68
69
|
return response.text();
|
|
69
|
-
})
|
|
70
|
+
})
|
|
71
|
+
.finally(() => {
|
|
70
72
|
clearTimeout(timeoutId);
|
|
71
73
|
});
|
|
72
74
|
return text;
|
|
73
75
|
}
|
|
74
76
|
catch (error) {
|
|
75
77
|
console.error(`Failed to fetch sitemap: ${error}`);
|
|
76
|
-
return
|
|
78
|
+
return "";
|
|
77
79
|
}
|
|
78
80
|
});
|
|
79
81
|
/**
|
|
@@ -91,22 +93,22 @@ function getSitemap(sitemapUrl) {
|
|
|
91
93
|
const normalizedSitemapUrl = (0, arachne_1.normalizeURL)(sitemapUrl);
|
|
92
94
|
const xmlData = yield fetchSitemap(normalizedSitemapUrl);
|
|
93
95
|
if (!xmlData) {
|
|
94
|
-
console.error(
|
|
96
|
+
console.error("No sitemap data retrieved.");
|
|
95
97
|
return [];
|
|
96
98
|
}
|
|
97
99
|
const $ = cheerio.load(xmlData, {
|
|
98
|
-
xmlMode: true
|
|
100
|
+
xmlMode: true,
|
|
99
101
|
});
|
|
100
102
|
let allUrls = [];
|
|
101
103
|
// Extract all urls
|
|
102
|
-
$(
|
|
103
|
-
const loc = $(url).find(
|
|
104
|
+
$("url").each((i, url) => {
|
|
105
|
+
const loc = $(url).find("loc").text();
|
|
104
106
|
allUrls.push(loc);
|
|
105
107
|
});
|
|
106
108
|
// Extract sitemaps and crawl recursively
|
|
107
|
-
const sitemapNodes = $(
|
|
109
|
+
const sitemapNodes = $("sitemap").toArray();
|
|
108
110
|
for (const sitemapNode of sitemapNodes) {
|
|
109
|
-
const loc = $(sitemapNode).find(
|
|
111
|
+
const loc = $(sitemapNode).find("loc").text();
|
|
110
112
|
// prevent an infinite loop by making sure loc is not the same as the original sitemapUrl
|
|
111
113
|
const normalizedLoc = (0, arachne_1.normalizeURL)(loc);
|
|
112
114
|
if (normalizedLoc !== normalizedSitemapUrl) {
|
|
@@ -135,11 +137,11 @@ function getSiteMapData(url) {
|
|
|
135
137
|
sitemapUrl.pathname = `${originalPath}sitemap.xml`;
|
|
136
138
|
}
|
|
137
139
|
catch (e) {
|
|
138
|
-
(0, stentor_logger_1.log)().debug(`
|
|
140
|
+
(0, stentor_logger_1.log)().debug(`Unable to parse sitemap url from ${url}.`);
|
|
139
141
|
return undefined;
|
|
140
142
|
}
|
|
141
143
|
if (!sitemapUrl) {
|
|
142
|
-
(0, stentor_logger_1.log)().debug(`
|
|
144
|
+
(0, stentor_logger_1.log)().debug(`Unable to parse sitemap url from ${url}.`);
|
|
143
145
|
return undefined;
|
|
144
146
|
}
|
|
145
147
|
(0, stentor_logger_1.log)().debug(`Loading sitemap at ${sitemapUrl.toString()}`);
|
|
@@ -214,7 +216,7 @@ function getSiteMapData(url) {
|
|
|
214
216
|
aboutUs,
|
|
215
217
|
contactUs,
|
|
216
218
|
faqPage,
|
|
217
|
-
paymentPage
|
|
219
|
+
paymentPage,
|
|
218
220
|
};
|
|
219
221
|
});
|
|
220
222
|
}
|