@xapp/arachne-web-service 1.18.6 → 1.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +32 -3
- package/lib/ArachneWebService.js +162 -61
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/GenerativeAIService.d.ts +17 -0
- package/lib/GenerativeAIService.js +70 -26
- package/lib/GenerativeAIService.js.map +1 -1
- package/lib/contants.d.ts +13 -1
- package/lib/contants.js +72 -1
- package/lib/contants.js.map +1 -1
- package/lib/models.d.ts +72 -1
- package/lib/models.js +10 -0
- package/lib/models.js.map +1 -1
- package/lib/utils/aboutUs.d.ts +34 -0
- package/lib/utils/aboutUs.js +144 -0
- package/lib/utils/aboutUs.js.map +1 -0
- package/lib/utils/getPrompt.d.ts +8 -0
- package/lib/utils/getPrompt.js +27 -0
- package/lib/utils/getPrompt.js.map +1 -1
- package/lib/utils/getSiteMap.d.ts +34 -1
- package/lib/utils/getSiteMap.js +127 -3
- package/lib/utils/getSiteMap.js.map +1 -1
- package/lib/utils/pagePatterns.d.ts +16 -1
- package/lib/utils/pagePatterns.js +21 -1
- package/lib/utils/pagePatterns.js.map +1 -1
- package/lib/utils/subpageDetection.d.ts +140 -0
- package/lib/utils/subpageDetection.js +259 -0
- package/lib/utils/subpageDetection.js.map +1 -0
- package/package.json +5 -5
|
@@ -14,6 +14,7 @@ const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
|
14
14
|
const stentor_logger_1 = require("stentor-logger");
|
|
15
15
|
const cleanMarkdown_1 = require("./utils/cleanMarkdown");
|
|
16
16
|
const getPrompt_1 = require("./utils/getPrompt");
|
|
17
|
+
const aboutUs_1 = require("./utils/aboutUs");
|
|
17
18
|
const arachne_utils_1 = require("@xapp/arachne-utils");
|
|
18
19
|
const stentor_utils_1 = require("stentor-utils");
|
|
19
20
|
class GenerativeAIService {
|
|
@@ -123,6 +124,70 @@ class GenerativeAIService {
|
|
|
123
124
|
else {
|
|
124
125
|
(0, stentor_logger_1.log)().info(`Final prompt is ${tokens} tokens, setting max_tokens to ${maxTokens}.`);
|
|
125
126
|
}
|
|
127
|
+
aiData = yield this.complete(prompt, maxTokens, "open_ai_web");
|
|
128
|
+
// Some clean up before returning, make sure we don't have empty strings
|
|
129
|
+
if (aiData && aiData.email && (0, stentor_utils_1.existsAndNotEmpty)(aiData.email)) {
|
|
130
|
+
const filteredEmails = aiData.email.filter((email) => !!email.email);
|
|
131
|
+
aiData.email = filteredEmails;
|
|
132
|
+
}
|
|
133
|
+
if (aiData && aiData.phone_numbers && (0, stentor_utils_1.existsAndNotEmpty)(aiData.phone_numbers)) {
|
|
134
|
+
const preLength = aiData.phone_numbers.length;
|
|
135
|
+
// clean for hallucinations
|
|
136
|
+
// numbers like 123-45678 or numbers like 555-
|
|
137
|
+
const mostLikelyRealPhoneNumbers = aiData.phone_numbers
|
|
138
|
+
.filter((number) => !!number.number)
|
|
139
|
+
.filter((number) => !(0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
140
|
+
const postLength = mostLikelyRealPhoneNumbers.length;
|
|
141
|
+
if (postLength < preLength) {
|
|
142
|
+
const removedNumbers = aiData.phone_numbers.filter((number) => (0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
143
|
+
(0, stentor_logger_1.log)().info(`Removed ${preLength - postLength} hallucinated phone numbers: ${removedNumbers
|
|
144
|
+
.map((number) => number.number)
|
|
145
|
+
.join(", ")}`);
|
|
146
|
+
}
|
|
147
|
+
aiData.phone_numbers = mostLikelyRealPhoneNumbers;
|
|
148
|
+
}
|
|
149
|
+
return aiData;
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Reads an about-us page for founding year, ownership, service areas, team size and licences.
|
|
154
|
+
*
|
|
155
|
+
* Adds one OpenAI call. Values the page does not state are dropped, see `sanitizeAboutUsAI()`.
|
|
156
|
+
*
|
|
157
|
+
* @param markdown The about-us page as markdown
|
|
158
|
+
* @param url URL of the about-us page
|
|
159
|
+
* @returns The data found, or undefined when the call fails or the page is empty or too long
|
|
160
|
+
*/
|
|
161
|
+
getAboutUsAI(markdown, url) {
|
|
162
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
163
|
+
if (!markdown.trim()) {
|
|
164
|
+
(0, stentor_logger_1.log)().info(`About us page ${url} is empty, not calling OpenAI.`);
|
|
165
|
+
return undefined;
|
|
166
|
+
}
|
|
167
|
+
let aboutUsPrompt = (0, getPrompt_1.getAboutUsPrompt)(markdown, url);
|
|
168
|
+
if (aboutUsPrompt.maxTokens <= 0) {
|
|
169
|
+
aboutUsPrompt = (0, getPrompt_1.getAboutUsPrompt)((0, cleanMarkdown_1.cleanMarkdown)(markdown), url);
|
|
170
|
+
}
|
|
171
|
+
if (aboutUsPrompt.maxTokens <= 0) {
|
|
172
|
+
(0, stentor_logger_1.log)().warn(`Unable to read about us page ${url}, too many tokens ${aboutUsPrompt.tokens}`);
|
|
173
|
+
return undefined;
|
|
174
|
+
}
|
|
175
|
+
(0, stentor_logger_1.log)().info(`🔍Reading about us page ${url}, ${aboutUsPrompt.tokens} tokens`);
|
|
176
|
+
const aiData = yield this.complete(aboutUsPrompt.prompt, aboutUsPrompt.maxTokens, "open_ai_about_us");
|
|
177
|
+
if (!aiData) {
|
|
178
|
+
return undefined;
|
|
179
|
+
}
|
|
180
|
+
return (0, aboutUs_1.sanitizeAboutUsAI)(aiData, markdown);
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Sends a prompt to OpenAI and parses the JSON answer.
|
|
185
|
+
*
|
|
186
|
+
* @returns The parsed answer, or undefined when the call fails or the answer is not JSON
|
|
187
|
+
*/
|
|
188
|
+
complete(prompt, maxTokens, timerLabel) {
|
|
189
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
190
|
+
let result = undefined;
|
|
126
191
|
const options = {
|
|
127
192
|
method: "POST",
|
|
128
193
|
headers: {
|
|
@@ -146,7 +211,7 @@ class GenerativeAIService {
|
|
|
146
211
|
}),
|
|
147
212
|
};
|
|
148
213
|
// eslint-disable-next-line no-console
|
|
149
|
-
console.time(
|
|
214
|
+
console.time(timerLabel);
|
|
150
215
|
let status = 999;
|
|
151
216
|
const openAIResponse = yield fetch("https://api.openai.com/v1/chat/completions", options)
|
|
152
217
|
.then((response) => {
|
|
@@ -162,7 +227,7 @@ class GenerativeAIService {
|
|
|
162
227
|
})
|
|
163
228
|
.finally(() => {
|
|
164
229
|
// eslint-disable-next-line no-console
|
|
165
|
-
console.timeEnd(
|
|
230
|
+
console.timeEnd(timerLabel);
|
|
166
231
|
});
|
|
167
232
|
if (status === 200) {
|
|
168
233
|
const usage = openAIResponse.usage;
|
|
@@ -174,9 +239,9 @@ class GenerativeAIService {
|
|
|
174
239
|
// Regex to find trailing commas before a closing brace or bracket
|
|
175
240
|
const trailingComma = /,(?=\s*[}\]])/g;
|
|
176
241
|
completion = completion.replace(trailingComma, "");
|
|
177
|
-
|
|
242
|
+
result = JSON.parse(completion);
|
|
178
243
|
(0, stentor_logger_1.log)().debug(`Parsed AI response....`);
|
|
179
|
-
(0, stentor_logger_1.log)().debug(JSON.stringify(
|
|
244
|
+
(0, stentor_logger_1.log)().debug(JSON.stringify(result, undefined, 2));
|
|
180
245
|
}
|
|
181
246
|
catch (e) {
|
|
182
247
|
(0, stentor_logger_1.log)().warn(`Unable to parse AI response.`);
|
|
@@ -187,28 +252,7 @@ class GenerativeAIService {
|
|
|
187
252
|
(0, stentor_logger_1.log)().warn(`Unabled to get website info from OpenAI status:${status}`);
|
|
188
253
|
(0, stentor_logger_1.log)().debug(openAIResponse);
|
|
189
254
|
}
|
|
190
|
-
|
|
191
|
-
if (aiData && aiData.email && (0, stentor_utils_1.existsAndNotEmpty)(aiData.email)) {
|
|
192
|
-
const filteredEmails = aiData.email.filter((email) => !!email.email);
|
|
193
|
-
aiData.email = filteredEmails;
|
|
194
|
-
}
|
|
195
|
-
if (aiData && aiData.phone_numbers && (0, stentor_utils_1.existsAndNotEmpty)(aiData.phone_numbers)) {
|
|
196
|
-
const preLength = aiData.phone_numbers.length;
|
|
197
|
-
// clean for hallucinations
|
|
198
|
-
// numbers like 123-45678 or numbers like 555-
|
|
199
|
-
const mostLikelyRealPhoneNumbers = aiData.phone_numbers
|
|
200
|
-
.filter((number) => !!number.number)
|
|
201
|
-
.filter((number) => !(0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
202
|
-
const postLength = mostLikelyRealPhoneNumbers.length;
|
|
203
|
-
if (postLength < preLength) {
|
|
204
|
-
const removedNumbers = aiData.phone_numbers.filter((number) => (0, arachne_utils_1.isHallucinatedPhoneNumber)(number.number));
|
|
205
|
-
(0, stentor_logger_1.log)().info(`Removed ${preLength - postLength} hallucinated phone numbers: ${removedNumbers
|
|
206
|
-
.map((number) => number.number)
|
|
207
|
-
.join(", ")}`);
|
|
208
|
-
}
|
|
209
|
-
aiData.phone_numbers = mostLikelyRealPhoneNumbers;
|
|
210
|
-
}
|
|
211
|
-
return aiData;
|
|
255
|
+
return result;
|
|
212
256
|
});
|
|
213
257
|
}
|
|
214
258
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GenerativeAIService.js","sourceRoot":"","sources":["../src/GenerativeAIService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,2DAAuG;AACvG,mDAAqC;AAGrC,yDAAsD;AACtD,
|
|
1
|
+
{"version":3,"file":"GenerativeAIService.js","sourceRoot":"","sources":["../src/GenerativeAIService.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,2DAAuG;AACvG,mDAAqC;AAGrC,yDAAsD;AACtD,iDAAwG;AACxG,6CAA4E;AAC5E,uDAAgE;AAChE,iDAAkD;AAqBlD,MAAa,mBAAmB;IAK5B,YAAY,KAAyC;QACjD,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEK,YAAY,CAAC,QAAgB,EAAE,IAAmB;;YACpD,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;YAEnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,4BAA4B,GAAG,EAAE,CAAC,CAAC;YAE9C,MAAM,KAAK,GAAG,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,KAAK,CAAC;YAC1B,MAAM,aAAa,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,aAAa,CAAC;YAC5C,MAAM,wBAAwB,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,wBAAwB,CAAC;YAElE,IAAI,MAAM,GAA0B,SAAS,CAAC;YAE9C,MAAM,SAAS,GAAG,IAAA,gCAAoB,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEtD,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;YACtC,IAAI,SAAS,GAAW,SAAS,CAAC,SAAS,CAAC;YAC5C,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;YAEtC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,wBAAwB,MAAM,SAAS,SAAS,EAAE,CAAC,CAAC;YAE/D,uBAAuB;YACvB,IAAI,eAAe,GAAW,EAAE,CAAC;YACjC,IAAI,aAAa,GAAuB,SAAS,CAAC;YAElD,+BAA+B;YAC/B,IAAI,SAAS,GAAG,CAAC,KAAI,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,CAAA,EAAE,CAAC;gBAClC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;gBAC9B,gBAAgB;gBAChB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,yEAAyE,OAAO,KAAK,CAAC,CAAC;gBAClG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAC5E,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBACxB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAE9C,eAAe,GAAG,IAAI,mCAAiB,CAAC;oBACpC,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;oBACpE,cAAc,EAAE,yCAAuB;iBAC1C,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;gBAE9B,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBAE3D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEvF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;oBACnC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;gBACD,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,GAAG,KAAK,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5B,CAAC;YAED,2BAA2B;YAC3B,IAAI,SAAS,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;gBACvB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oFAAoF,GAAG,KAAK,CAAC,CAAC;gBACzG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExE,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBAE3C,MAAM,mBAAmB,GAAG,IAAI,mCAAiB,CAAC;oBAC9C,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC;oBACpE,cAAc,EAAE,yCAAuB;oBACvC,YAAY,EAAE,IAAI;iBACrB,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAE3B,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;gBAErE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEtF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAChD,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,iBAAiB;YACjB,IAAI,SAAS,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,wBAAwB,CAAC,EAAE,CAAC;gBAC/D,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,oBAAoB,SAAS,oEAAoE,CACpG,CAAC;gBACF,4CAA4C;gBAE5C,MAAM,gBAAgB,GAAG,IAAA,+BAAmB,EAAC,aAAa,IAAI,wBAAwB,EAAE,GAAG,CAAC,CAAC;gBAE7F,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,gBAAgB,CAAC,MAAM,SAAS,gBAAgB,CAAC,SAAS,EAAE,CAAC,CAAC;gBAEhG,IAAI,gBAAgB,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBACjC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;oBACpE,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;oBACjC,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;oBACvC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;gBACrC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAChB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC;gBACjE,sDAAsD;gBACtD,MAAM,eAAe,GAAG,IAAA,6BAAa,EAAC,QAAQ,CAAC,CAAC;gBAEhD,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;gBAEjE,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBAC9B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;oBAClE,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;oBAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;oBACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,GAAG,CAAC,IAAI,aAAa,IAAI,eAAe,EAAE,CAAC;gBACpD,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mEAAmE,CAAC,CAAC;gBAEhF,MAAM,sBAAsB,GAAG,IAAA,6BAAa,EAAC,eAAe,CAAC,CAAC;gBAE9D,MAAM,oBAAoB,GAAG,IAAA,gCAAoB,EAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;gBAE/E,IAAI,oBAAoB,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;oBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;oBAC3E,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC;oBACrC,SAAS,GAAG,oBAAoB,CAAC,SAAS,CAAC;oBAC3C,MAAM,GAAG,oBAAoB,CAAC,MAAM,CAAC;gBACzC,CAAC;YACL,CAAC;YAED,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBACjB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gCAAgC,GAAG,qBAAqB,MAAM,EAAE,CAAC,CAAC;gBAC7E,OAAO,SAAS,CAAC;YACrB,CAAC;iBAAM,CAAC;gBACJ,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,MAAM,kCAAkC,SAAS,GAAG,CAAC,CAAC;YACxF,CAAC;YAED,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAY,MAAM,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC;YAE1E,wEAAwE;YACxE,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,IAAI,IAAA,iCAAiB,EAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC5D,MAAM,cAAc,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBACrE,MAAM,CAAC,KAAK,GAAG,cAAc,CAAC;YAClC,CAAC;YAED,IAAI,MAAM,IAAI,MAAM,CAAC,aAAa,IAAI,IAAA,iCAAiB,EAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC5E,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC;gBAC9C,2BAA2B;gBAC3B,8CAA8C;gBAE9C,MAAM,0BAA0B,GAAG,MAAM,CAAC,aAAa;qBAClD,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;qBACnC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAA,yCAAyB,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;gBAEnE,MAAM,UAAU,GAAG,0BAA0B,CAAC,MAAM,CAAC;gBAErD,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;oBACzB,MAAM,cAAc,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAC1D,IAAA,yCAAyB,EAAC,MAAM,CAAC,MAAM,CAAC,CAC3C,CAAC;oBACF,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,WAAW,SAAS,GAAG,UAAU,gCAAgC,cAAc;yBAC1E,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC;yBAC9B,IAAI,CAAC,IAAI,CAAC,EAAE,CACpB,CAAC;gBACN,CAAC;gBAED,MAAM,CAAC,aAAa,GAAG,0BAA0B,CAAC;YACtD,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;IAED;;;;;;;;OAQG;IACG,YAAY,CAAC,QAAgB,EAAE,GAAW;;YAC5C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACnB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,iBAAiB,GAAG,gCAAgC,CAAC,CAAC;gBACjE,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAI,aAAa,GAAG,IAAA,4BAAgB,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAEpD,IAAI,aAAa,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;gBAC/B,aAAa,GAAG,IAAA,4BAAgB,EAAC,IAAA,6BAAa,EAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;YACnE,CAAC;YAED,IAAI,aAAa,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;gBAC/B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gCAAgC,GAAG,qBAAqB,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC3F,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,2BAA2B,GAAG,KAAK,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;YAE7E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAY,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,CAAC;YAEjH,IAAI,CAAC,MAAM,EAAE,CAAC;gBACV,OAAO,SAAS,CAAC;YACrB,CAAC;YAED,OAAO,IAAA,2BAAiB,EAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/C,CAAC;KAAA;IAED;;;;OAIG;IACW,QAAQ,CAAI,MAAc,EAAE,SAAiB,EAAE,UAAkB;;YAC3E,IAAI,MAAM,GAAkB,SAAS,CAAC;YAEtC,MAAM,OAAO,GAAgB;gBACzB,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACL,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,GAAG,EAAE;iBACtC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACjB,KAAK,EAAE,wBAAwB;oBAC/B,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;oBACxC,QAAQ,EAAE;wBACN;4BACI,IAAI,EAAE,QAAQ;4BACd,OAAO,EAAE,MAAM;yBAClB;qBACJ;oBACD,WAAW,EAAE,IAAI;oBACjB,UAAU,EAAE,SAAS;oBACrB,KAAK,EAAE,CAAC;oBACR,iBAAiB,EAAE,CAAC;oBACpB,gBAAgB,EAAE,CAAC;iBACtB,CAAC;aACL,CAAC;YAEF,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACzB,IAAI,MAAM,GAAW,GAAG,CAAC;YACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,4CAA4C,EAAE,OAAO,CAAC;iBACpF,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACf,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzB,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACjB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC3B,CAAC;qBAAM,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;oBACxB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;oBACpC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC3B,CAAC;gBACD,OAAO,QAAQ,CAAC;YACpB,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACV,sCAAsC;gBACtC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAChC,CAAC,CAAC,CAAC;YAEP,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;gBACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CACxH,CAAC;gBAEF,MAAM,WAAW,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAE9C,IAAI,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC;gBAC7C,IAAI,CAAC;oBACD,4EAA4E;oBAC5E,kEAAkE;oBAClE,MAAM,aAAa,GAAG,gBAAgB,CAAC;oBACvC,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;oBAEnD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;oBAChC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;oBACtC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;gBACtD,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;oBAC3C,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBAC3B,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;gBACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,MAAM,CAAC;QAClB,CAAC;KAAA;CACJ;AAvSD,kDAuSC"}
|
package/lib/contants.d.ts
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
/*! Copyright (c) 2024, XAPP AI */
|
|
2
|
-
import { AbstractDetector, Evaluator, EvaluatorDetector } from "@xapp/arachne-parsers";
|
|
2
|
+
import { AbstractDetector, DetectorResult, Evaluator, EvaluatorDetector } from "@xapp/arachne-parsers";
|
|
3
3
|
export type DetectorConstructor = new (html: string | cheerio.Root, page: Evaluator) => AbstractDetector | EvaluatorDetector;
|
|
4
4
|
/** Caller-supplied detectors or DEFAULT_DETECTORS. Explicit `[]` skips the chain. */
|
|
5
5
|
export declare function resolveDetectors(input: DetectorConstructor[] | undefined): DetectorConstructor[];
|
|
6
|
+
/**
|
|
7
|
+
* Runs each detector on the page in turn, handing every detector's results to `onResults` as soon as it finishes,
|
|
8
|
+
* so a caller that stops waiting keeps the results gathered so far.
|
|
9
|
+
*
|
|
10
|
+
* A detector that throws is logged and skipped.
|
|
11
|
+
*
|
|
12
|
+
* @param detectors Detectors to run, in order
|
|
13
|
+
* @param html HTML of the page
|
|
14
|
+
* @param page The live page the HTML came from
|
|
15
|
+
* @param onResults Receives each detector's results
|
|
16
|
+
*/
|
|
17
|
+
export declare function runDetectorChain(detectors: DetectorConstructor[], html: string, page: Evaluator, onResults: (results: DetectorResult[]) => void): Promise<void>;
|
|
6
18
|
export declare const DEFAULT_DETECTORS: DetectorConstructor[];
|
package/lib/contants.js
CHANGED
|
@@ -1,28 +1,100 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
2
11
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
12
|
exports.DEFAULT_DETECTORS = void 0;
|
|
4
13
|
exports.resolveDetectors = resolveDetectors;
|
|
14
|
+
exports.runDetectorChain = runDetectorChain;
|
|
5
15
|
/*! Copyright (c) 2024, XAPP AI */
|
|
6
16
|
const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
17
|
+
const stentor_logger_1 = require("stentor-logger");
|
|
7
18
|
/** Caller-supplied detectors or DEFAULT_DETECTORS. Explicit `[]` skips the chain. */
|
|
8
19
|
function resolveDetectors(input) {
|
|
9
20
|
return Array.isArray(input) ? input : exports.DEFAULT_DETECTORS;
|
|
10
21
|
}
|
|
22
|
+
/**
|
|
23
|
+
* Runs each detector on the page in turn, handing every detector's results to `onResults` as soon as it finishes,
|
|
24
|
+
* so a caller that stops waiting keeps the results gathered so far.
|
|
25
|
+
*
|
|
26
|
+
* A detector that throws is logged and skipped.
|
|
27
|
+
*
|
|
28
|
+
* @param detectors Detectors to run, in order
|
|
29
|
+
* @param html HTML of the page
|
|
30
|
+
* @param page The live page the HTML came from
|
|
31
|
+
* @param onResults Receives each detector's results
|
|
32
|
+
*/
|
|
33
|
+
function runDetectorChain(detectors, html, page, onResults) {
|
|
34
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
35
|
+
// go through and detect each
|
|
36
|
+
for (const detector of detectors) {
|
|
37
|
+
try {
|
|
38
|
+
if (process.env.DEBUG_DETECTORS === "true") {
|
|
39
|
+
(0, stentor_logger_1.log)().info(`Running detector ${detector.name}...`);
|
|
40
|
+
}
|
|
41
|
+
// monitor the time of each detector
|
|
42
|
+
const start = Date.now();
|
|
43
|
+
const detectorResult = yield new detector(html, page).detect();
|
|
44
|
+
// if one takes more than 500ms we log debug, 600ms we warn
|
|
45
|
+
const end = Date.now();
|
|
46
|
+
const duration = end - start;
|
|
47
|
+
if (process.env.DEBUG_DETECTORS === "true") {
|
|
48
|
+
(0, stentor_logger_1.log)().info(`Detector ${detector.name} took ${duration}ms to run.`);
|
|
49
|
+
const installed = detectorResult.filter((d) => d.installed);
|
|
50
|
+
const installedTechnologies = installed.map((d) => d.technology).join(", ");
|
|
51
|
+
(0, stentor_logger_1.log)().info(`Detector ${detector.name} found ${installed.length} installed items. ${installedTechnologies}`);
|
|
52
|
+
}
|
|
53
|
+
if (duration > 500) {
|
|
54
|
+
(0, stentor_logger_1.log)().debug(`Detector ${detector.name} took ${duration}ms to run.`);
|
|
55
|
+
}
|
|
56
|
+
if (duration > 600) {
|
|
57
|
+
(0, stentor_logger_1.log)().warn(`Detector ${detector.name} took ${duration}ms to run.`);
|
|
58
|
+
}
|
|
59
|
+
onResults(detectorResult);
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
(0, stentor_logger_1.log)().error(`Error detecting technology ${detector.name}`);
|
|
63
|
+
console.error(e);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
}
|
|
11
68
|
exports.DEFAULT_DETECTORS = [
|
|
12
69
|
arachne_parsers_1.XAPPDetector,
|
|
13
70
|
// Home Service Tools
|
|
14
71
|
arachne_parsers_1.AvocaDetector,
|
|
72
|
+
arachne_parsers_1.BooksyDetector,
|
|
73
|
+
arachne_parsers_1.BoulevardDetector,
|
|
15
74
|
arachne_parsers_1.CallRailDetector,
|
|
16
75
|
arachne_parsers_1.ContractorCommerceDetector,
|
|
76
|
+
arachne_parsers_1.DemandforceDetector,
|
|
17
77
|
arachne_parsers_1.HouseCallProDetector,
|
|
18
78
|
arachne_parsers_1.FieldPulseDetector,
|
|
19
79
|
arachne_parsers_1.JobberDetector,
|
|
80
|
+
arachne_parsers_1.MindbodyDetector,
|
|
20
81
|
arachne_parsers_1.ScheduleEngineDetector,
|
|
21
82
|
arachne_parsers_1.ScorpionDetector,
|
|
22
83
|
arachne_parsers_1.ServiceTitanDetector,
|
|
84
|
+
arachne_parsers_1.SquareAppointmentsDetector,
|
|
23
85
|
arachne_parsers_1.SurefireDetector,
|
|
24
86
|
arachne_parsers_1.ThryvDetector,
|
|
87
|
+
arachne_parsers_1.VagaroDetector,
|
|
25
88
|
arachne_parsers_1.WorkizDetector,
|
|
89
|
+
// Salon & Spa
|
|
90
|
+
arachne_parsers_1.AestheticsProDetector,
|
|
91
|
+
arachne_parsers_1.DaySmartSalonDetector,
|
|
92
|
+
arachne_parsers_1.JaneDetector,
|
|
93
|
+
arachne_parsers_1.MangomintDetector,
|
|
94
|
+
arachne_parsers_1.MeevoDetector,
|
|
95
|
+
arachne_parsers_1.PatientNowDetector,
|
|
96
|
+
arachne_parsers_1.PhorestDetector,
|
|
97
|
+
arachne_parsers_1.SalonBizDetector,
|
|
26
98
|
// Chat
|
|
27
99
|
arachne_parsers_1.AutoBotAIDetector,
|
|
28
100
|
arachne_parsers_1.ApexChatDetector,
|
|
@@ -76,7 +148,6 @@ exports.DEFAULT_DETECTORS = [
|
|
|
76
148
|
arachne_parsers_1.AttributionDetector,
|
|
77
149
|
// Other
|
|
78
150
|
arachne_parsers_1.NiceJobDetector,
|
|
79
|
-
arachne_parsers_1.CallRailDetector,
|
|
80
151
|
arachne_parsers_1.GooglereCAPTCHADetector,
|
|
81
152
|
arachne_parsers_1.NitroPackDetector,
|
|
82
153
|
arachne_parsers_1.YextDetector,
|
package/lib/contants.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"contants.js","sourceRoot":"","sources":["../src/contants.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"contants.js","sourceRoot":"","sources":["../src/contants.ts"],"names":[],"mappings":";;;;;;;;;;;;AAyFA,4CAEC;AAaD,4CAwCC;AAhJD,kCAAkC;AAClC,2DAgF+B;AAC/B,mDAAqC;AAMrC,qFAAqF;AACrF,SAAgB,gBAAgB,CAAC,KAAwC;IACrE,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,yBAAiB,CAAC;AAC5D,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAsB,gBAAgB,CAClC,SAAgC,EAChC,IAAY,EACZ,IAAe,EACf,SAA8C;;QAE9C,6BAA6B;QAC7B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,EAAE,CAAC;oBACzC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC;gBACvD,CAAC;gBACD,oCAAoC;gBACpC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,MAAM,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBAC/D,2DAA2D;gBAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,QAAQ,GAAG,GAAG,GAAG,KAAK,CAAC;gBAE7B,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,KAAK,MAAM,EAAE,CAAC;oBACzC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,YAAY,QAAQ,CAAC,IAAI,SAAS,QAAQ,YAAY,CAAC,CAAC;oBACnE,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;oBAC5D,MAAM,qBAAqB,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5E,IAAA,oBAAG,GAAE,CAAC,IAAI,CACN,YAAY,QAAQ,CAAC,IAAI,UAAU,SAAS,CAAC,MAAM,qBAAqB,qBAAqB,EAAE,CAClG,CAAC;gBACN,CAAC;gBAED,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;oBACjB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,YAAY,QAAQ,CAAC,IAAI,SAAS,QAAQ,YAAY,CAAC,CAAC;gBACxE,CAAC;gBACD,IAAI,QAAQ,GAAG,GAAG,EAAE,CAAC;oBACjB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,YAAY,QAAQ,CAAC,IAAI,SAAS,QAAQ,YAAY,CAAC,CAAC;gBACvE,CAAC;gBACD,SAAS,CAAC,cAAc,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACT,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,8BAA8B,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC3D,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC;QACL,CAAC;IACL,CAAC;CAAA;AAEY,QAAA,iBAAiB,GAA0B;IACpD,8BAAY;IACZ,qBAAqB;IACrB,+BAAa;IACb,gCAAc;IACd,mCAAiB;IACjB,kCAAgB;IAChB,4CAA0B;IAC1B,qCAAmB;IACnB,sCAAoB;IACpB,oCAAkB;IAClB,gCAAc;IACd,kCAAgB;IAChB,wCAAsB;IACtB,kCAAgB;IAChB,sCAAoB;IACpB,4CAA0B;IAC1B,kCAAgB;IAChB,+BAAa;IACb,gCAAc;IACd,gCAAc;IACd,cAAc;IACd,uCAAqB;IACrB,uCAAqB;IACrB,8BAAY;IACZ,mCAAiB;IACjB,+BAAa;IACb,oCAAkB;IAClB,iCAAe;IACf,kCAAgB;IAChB,OAAO;IACP,mCAAiB;IACjB,kCAAgB;IAChB,iCAAe;IACf,kCAAgB;IAChB,oCAAkB;IAClB,iCAAe;IACf,kCAAgB;IAChB,+BAAa;IACb,mCAAiB;IACjB,iCAAe;IACf,kCAAgB;IAChB,kCAAgB;IAChB,uCAAqB;IACrB,qCAAmB;IACnB,mCAAiB;IACjB,gCAAc;IACd,2CAAyB;IACzB,+BAAa;IACb,gCAAc;IACd,kCAAgB;IAChB,kCAAgB;IAChB,sBAAsB;IACtB,gCAAc;IACd,iCAAe;IACf,MAAM;IACN,gCAAc;IACd,8BAAY;IACZ,gCAAc;IACd,iCAAe;IACf,wCAAsB;IACtB,gCAAc;IACd,gCAAc;IACd,sCAAoB;IACpB,qCAAmB;IACnB,iCAAe;IACf,6BAAW;IACX,mCAAiB;IACjB,WAAW;IACX,gCAAc;IACd,mCAAiB;IACjB,cAAc;IACd,iFAAiF;IACjF,iCAAe;IACf,mCAAiB;IACjB,0CAAwB;IACxB,yCAAuB;IACvB,qCAAmB;IACnB,qCAAmB;IACnB,cAAc;IACd,qCAAmB;IACnB,QAAQ;IACR,iCAAe;IACf,yCAAuB;IACvB,mCAAiB;IACjB,8BAAY;CACf,CAAC"}
|
package/lib/models.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/*! Copyright (c) 2023, XAPP AI */
|
|
2
2
|
import { HomeAndConstructionBusiness, LocalBusiness, Organization } from "schema-dts";
|
|
3
|
-
import { ProfileLink } from "@xapp/arachne-parsers";
|
|
3
|
+
import { DetectorResult, ProfileLink } from "@xapp/arachne-parsers";
|
|
4
4
|
import { SiteMapData } from "./utils/getSiteMap";
|
|
5
5
|
export interface WebsiteSchema {
|
|
6
6
|
localBusiness?: LocalBusiness;
|
|
@@ -189,4 +189,75 @@ export interface BusinessData {
|
|
|
189
189
|
* (e.g., Facebook, BBB, HomeAdvisor, Yelp)
|
|
190
190
|
*/
|
|
191
191
|
profileLinks?: ProfileLink[];
|
|
192
|
+
/**
|
|
193
|
+
* The year the business was founded, as stated on its about-us page.
|
|
194
|
+
*
|
|
195
|
+
* Only read when `dataOptions.readAboutUsPage` is true.
|
|
196
|
+
*/
|
|
197
|
+
foundedYear?: number;
|
|
198
|
+
/**
|
|
199
|
+
* How the business describes its ownership, e.g. family-owned or veteran-owned, from its about-us page.
|
|
200
|
+
*
|
|
201
|
+
* Only read when `dataOptions.readAboutUsPage` is true.
|
|
202
|
+
*/
|
|
203
|
+
ownership?: BusinessOwnership[];
|
|
204
|
+
/**
|
|
205
|
+
* Cities, counties or regions the business says it serves, from its about-us page.
|
|
206
|
+
*
|
|
207
|
+
* Only read when `dataOptions.readAboutUsPage` is true.
|
|
208
|
+
*/
|
|
209
|
+
serviceAreas?: string[];
|
|
210
|
+
/**
|
|
211
|
+
* Number of people on the team, as stated on the about-us page.
|
|
212
|
+
*
|
|
213
|
+
* Only read when `dataOptions.readAboutUsPage` is true.
|
|
214
|
+
*/
|
|
215
|
+
teamSize?: number;
|
|
216
|
+
}
|
|
217
|
+
export type BusinessOwnership = "family-owned" | "veteran-owned" | "woman-owned" | "minority-owned" | "locally-owned" | "employee-owned" | "franchise";
|
|
218
|
+
export declare const BUSINESS_OWNERSHIP_TYPES: readonly BusinessOwnership[];
|
|
219
|
+
/**
|
|
220
|
+
* Discovered pages that subpage detection can visit.
|
|
221
|
+
*/
|
|
222
|
+
export type SubpageType = "bookingPage" | "contactUs" | "aboutUs";
|
|
223
|
+
/**
|
|
224
|
+
* Where a detection was made.
|
|
225
|
+
*/
|
|
226
|
+
export interface DetectionSource {
|
|
227
|
+
/**
|
|
228
|
+
* The page type the detector ran on; "homepage" is the URL the crawl started from.
|
|
229
|
+
*/
|
|
230
|
+
pageType: "homepage" | SubpageType;
|
|
231
|
+
/**
|
|
232
|
+
* URL of that page.
|
|
233
|
+
*/
|
|
234
|
+
url: string;
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* A detector result, with where it was found when subpage detection is on.
|
|
238
|
+
*/
|
|
239
|
+
export interface SourcedDetectorResult extends DetectorResult {
|
|
240
|
+
/**
|
|
241
|
+
* The page the detection came from. Only set when `subpageDetection` is on.
|
|
242
|
+
*/
|
|
243
|
+
source?: DetectionSource;
|
|
244
|
+
}
|
|
245
|
+
export interface SubpageDetectionOptions {
|
|
246
|
+
/**
|
|
247
|
+
* Which discovered page types to visit and run the detectors on, in order. Empty or absent = off.
|
|
248
|
+
*
|
|
249
|
+
* A page is only visited when it was discovered (sitemap first, homepage links second) and is on the
|
|
250
|
+
* same site as the crawled URL. A page shared by two types is visited once.
|
|
251
|
+
*/
|
|
252
|
+
pages?: SubpageType[];
|
|
253
|
+
/**
|
|
254
|
+
* Budget in milliseconds for each subpage: navigation plus the detector chain. Defaults to 10,000.
|
|
255
|
+
*
|
|
256
|
+
* All subpages together are also bounded by what remains of `pageHandlerTimeout`.
|
|
257
|
+
*/
|
|
258
|
+
pageTimeoutMs?: number;
|
|
259
|
+
/**
|
|
260
|
+
* Most subpages visited in one crawl. Defaults to 3.
|
|
261
|
+
*/
|
|
262
|
+
maxPages?: number;
|
|
192
263
|
}
|
package/lib/models.js
CHANGED
|
@@ -1,3 +1,13 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BUSINESS_OWNERSHIP_TYPES = void 0;
|
|
4
|
+
exports.BUSINESS_OWNERSHIP_TYPES = [
|
|
5
|
+
"family-owned",
|
|
6
|
+
"veteran-owned",
|
|
7
|
+
"woman-owned",
|
|
8
|
+
"minority-owned",
|
|
9
|
+
"locally-owned",
|
|
10
|
+
"employee-owned",
|
|
11
|
+
"franchise",
|
|
12
|
+
];
|
|
3
13
|
//# sourceMappingURL=models.js.map
|
package/lib/models.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":""}
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":";;;AA8Oa,QAAA,wBAAwB,GAAiC;IAClE,cAAc;IACd,eAAe;IACf,aAAa;IACb,gBAAgB;IAChB,eAAe;IACf,gBAAgB;IAChB,WAAW;CACd,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/*! Copyright (c) 2026, XAPP AI */
|
|
2
|
+
import { BusinessData } from "../models";
|
|
3
|
+
/**
|
|
4
|
+
* The raw answer to the about-us prompt, see `getAboutUsPrompt()`.
|
|
5
|
+
*/
|
|
6
|
+
export interface AboutUsAI {
|
|
7
|
+
founded_year?: unknown;
|
|
8
|
+
ownership?: unknown;
|
|
9
|
+
service_areas?: unknown;
|
|
10
|
+
team_size?: unknown;
|
|
11
|
+
licenses?: unknown;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Business data read from an about-us page.
|
|
15
|
+
*/
|
|
16
|
+
export type AboutUsData = Pick<BusinessData, "foundedYear" | "ownership" | "serviceAreas" | "teamSize" | "licenses">;
|
|
17
|
+
/**
|
|
18
|
+
* Validates the LLM's answer against the page it read.
|
|
19
|
+
*
|
|
20
|
+
* Anything the page does not literally state is dropped: the founding year, team size, each service area and each
|
|
21
|
+
* licence number must appear in the markdown. Ownership must be one of BUSINESS_OWNERSHIP_TYPES.
|
|
22
|
+
*
|
|
23
|
+
* @param ai The parsed answer
|
|
24
|
+
* @param markdown The about-us page the answer was drawn from
|
|
25
|
+
*/
|
|
26
|
+
export declare function sanitizeAboutUsAI(ai: AboutUsAI | undefined, markdown: string): AboutUsData;
|
|
27
|
+
/**
|
|
28
|
+
* Adds about-us data to the business without overwriting what is already known: each field is only filled when
|
|
29
|
+
* empty, and licences are added when their number is new (compared as `licenseKey()` does), once per number.
|
|
30
|
+
*
|
|
31
|
+
* @param business Business data, mutated
|
|
32
|
+
* @param data From `sanitizeAboutUsAI()`
|
|
33
|
+
*/
|
|
34
|
+
export declare function mergeAboutUsData(business: BusinessData, data: AboutUsData): void;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.sanitizeAboutUsAI = sanitizeAboutUsAI;
|
|
4
|
+
exports.mergeAboutUsData = mergeAboutUsData;
|
|
5
|
+
/*! Copyright (c) 2026, XAPP AI */
|
|
6
|
+
const models_1 = require("../models");
|
|
7
|
+
/** Earliest founding year accepted; older answers are almost always a misread. */
|
|
8
|
+
const MIN_FOUNDED_YEAR = 1800;
|
|
9
|
+
const MAX_SERVICE_AREAS = 25;
|
|
10
|
+
function toInteger(value) {
|
|
11
|
+
const n = typeof value === "string" && value.trim() !== "" ? Number(value) : value;
|
|
12
|
+
return typeof n === "number" && Number.isInteger(n) ? n : undefined;
|
|
13
|
+
}
|
|
14
|
+
function escapeRegExp(value) {
|
|
15
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Identity of a licence number, compared the way `sanitizeAboutUsAI()` checks the number is on the page:
|
|
19
|
+
* case-insensitively, with whitespace significant. So "cfc1234567" and "CFC1234567" are one licence, while
|
|
20
|
+
* "CFC 1234567" is another.
|
|
21
|
+
*/
|
|
22
|
+
function licenseKey(number) {
|
|
23
|
+
return number.toLowerCase();
|
|
24
|
+
}
|
|
25
|
+
/** The number appears on its own in the text, e.g. 12 in "team of 12" but not in "2012". */
|
|
26
|
+
function statesNumber(text, n) {
|
|
27
|
+
return new RegExp(`(^|[^0-9])${n}([^0-9]|$)`).test(text);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Validates the LLM's answer against the page it read.
|
|
31
|
+
*
|
|
32
|
+
* Anything the page does not literally state is dropped: the founding year, team size, each service area and each
|
|
33
|
+
* licence number must appear in the markdown. Ownership must be one of BUSINESS_OWNERSHIP_TYPES.
|
|
34
|
+
*
|
|
35
|
+
* @param ai The parsed answer
|
|
36
|
+
* @param markdown The about-us page the answer was drawn from
|
|
37
|
+
*/
|
|
38
|
+
function sanitizeAboutUsAI(ai, markdown) {
|
|
39
|
+
const data = {};
|
|
40
|
+
if (!ai || typeof ai !== "object") {
|
|
41
|
+
return data;
|
|
42
|
+
}
|
|
43
|
+
const lowerMarkdown = markdown.toLowerCase();
|
|
44
|
+
const foundedYear = toInteger(ai.founded_year);
|
|
45
|
+
if (foundedYear !== undefined &&
|
|
46
|
+
foundedYear >= MIN_FOUNDED_YEAR &&
|
|
47
|
+
foundedYear <= new Date().getFullYear() &&
|
|
48
|
+
statesNumber(markdown, foundedYear)) {
|
|
49
|
+
data.foundedYear = foundedYear;
|
|
50
|
+
}
|
|
51
|
+
if (Array.isArray(ai.ownership)) {
|
|
52
|
+
const ownership = new Set();
|
|
53
|
+
for (const value of ai.ownership) {
|
|
54
|
+
const normalized = typeof value === "string" ? value.trim().toLowerCase() : "";
|
|
55
|
+
if (models_1.BUSINESS_OWNERSHIP_TYPES.includes(normalized)) {
|
|
56
|
+
ownership.add(normalized);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (ownership.size > 0) {
|
|
60
|
+
data.ownership = [...ownership];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (Array.isArray(ai.service_areas)) {
|
|
64
|
+
const areas = [];
|
|
65
|
+
for (const value of ai.service_areas) {
|
|
66
|
+
if (typeof value !== "string")
|
|
67
|
+
continue;
|
|
68
|
+
const area = value.trim();
|
|
69
|
+
if (!area || !lowerMarkdown.includes(area.toLowerCase()))
|
|
70
|
+
continue;
|
|
71
|
+
if (areas.some((existing) => existing.toLowerCase() === area.toLowerCase()))
|
|
72
|
+
continue;
|
|
73
|
+
areas.push(area);
|
|
74
|
+
if (areas.length >= MAX_SERVICE_AREAS)
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
if (areas.length > 0) {
|
|
78
|
+
data.serviceAreas = areas;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const teamSize = toInteger(ai.team_size);
|
|
82
|
+
if (teamSize !== undefined && teamSize > 0 && statesNumber(markdown, teamSize)) {
|
|
83
|
+
data.teamSize = teamSize;
|
|
84
|
+
}
|
|
85
|
+
if (Array.isArray(ai.licenses)) {
|
|
86
|
+
const seen = new Set();
|
|
87
|
+
const licenses = ai.licenses.filter((license) => {
|
|
88
|
+
if (!license ||
|
|
89
|
+
typeof license.number !== "string" ||
|
|
90
|
+
license.number.length === 0 ||
|
|
91
|
+
!new RegExp(escapeRegExp(license.number), "i").test(markdown)) {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
// one entry per number, the first one given wins
|
|
95
|
+
const key = licenseKey(license.number);
|
|
96
|
+
if (seen.has(key)) {
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
seen.add(key);
|
|
100
|
+
return true;
|
|
101
|
+
});
|
|
102
|
+
if (licenses.length > 0) {
|
|
103
|
+
data.licenses = licenses;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return data;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Adds about-us data to the business without overwriting what is already known: each field is only filled when
|
|
110
|
+
* empty, and licences are added when their number is new (compared as `licenseKey()` does), once per number.
|
|
111
|
+
*
|
|
112
|
+
* @param business Business data, mutated
|
|
113
|
+
* @param data From `sanitizeAboutUsAI()`
|
|
114
|
+
*/
|
|
115
|
+
function mergeAboutUsData(business, data) {
|
|
116
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
117
|
+
if (business.foundedYear === undefined && data.foundedYear !== undefined) {
|
|
118
|
+
business.foundedYear = data.foundedYear;
|
|
119
|
+
}
|
|
120
|
+
if (!((_a = business.ownership) === null || _a === void 0 ? void 0 : _a.length) && ((_b = data.ownership) === null || _b === void 0 ? void 0 : _b.length)) {
|
|
121
|
+
business.ownership = data.ownership;
|
|
122
|
+
}
|
|
123
|
+
if (!((_c = business.serviceAreas) === null || _c === void 0 ? void 0 : _c.length) && ((_d = data.serviceAreas) === null || _d === void 0 ? void 0 : _d.length)) {
|
|
124
|
+
business.serviceAreas = data.serviceAreas;
|
|
125
|
+
}
|
|
126
|
+
if (business.teamSize === undefined && data.teamSize !== undefined) {
|
|
127
|
+
business.teamSize = data.teamSize;
|
|
128
|
+
}
|
|
129
|
+
if ((_e = data.licenses) === null || _e === void 0 ? void 0 : _e.length) {
|
|
130
|
+
const known = new Set(((_f = business.licenses) !== null && _f !== void 0 ? _f : []).map((license) => licenseKey(license.number)));
|
|
131
|
+
const added = data.licenses.filter((license) => {
|
|
132
|
+
const key = licenseKey(license.number);
|
|
133
|
+
if (known.has(key)) {
|
|
134
|
+
return false;
|
|
135
|
+
}
|
|
136
|
+
known.add(key);
|
|
137
|
+
return true;
|
|
138
|
+
});
|
|
139
|
+
if (added.length > 0) {
|
|
140
|
+
business.licenses = [...((_g = business.licenses) !== null && _g !== void 0 ? _g : []), ...added];
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=aboutUs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"aboutUs.js","sourceRoot":"","sources":["../../src/utils/aboutUs.ts"],"names":[],"mappings":";;AAwDA,8CA6EC;AASD,4CA2BC;AAzKD,kCAAkC;AAClC,sCAAuG;AAkBvG,kFAAkF;AAClF,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,SAAS,SAAS,CAAC,KAAc;IAC7B,MAAM,CAAC,GAAG,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACnF,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACxE,CAAC;AAED,SAAS,YAAY,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,MAAc;IAC9B,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC;AAChC,CAAC;AAED,4FAA4F;AAC5F,SAAS,YAAY,CAAC,IAAY,EAAE,CAAS;IACzC,OAAO,IAAI,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7D,CAAC;AAED;;;;;;;;GAQG;AACH,SAAgB,iBAAiB,CAAC,EAAyB,EAAE,QAAgB;IACzE,MAAM,IAAI,GAAgB,EAAE,CAAC;IAE7B,IAAI,CAAC,EAAE,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,MAAM,aAAa,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE7C,MAAM,WAAW,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;IAC/C,IACI,WAAW,KAAK,SAAS;QACzB,WAAW,IAAI,gBAAgB;QAC/B,WAAW,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,EACrC,CAAC;QACC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAqB,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,SAAS,EAAE,CAAC;YAC/B,MAAM,UAAU,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,IAAK,iCAA8C,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;gBACvE,SAAS,CAAC,GAAG,CAAC,UAA+B,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;QACD,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,GAAG,CAAC,GAAG,SAAS,CAAC,CAAC;QACpC,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;QAClC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC;YACnC,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,SAAS;YACxC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAAE,SAAS;YACnE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;gBAAE,SAAS;YACtF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,IAAI,KAAK,CAAC,MAAM,IAAI,iBAAiB;gBAAE,MAAM;QACjD,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,GAAG,CAAC,IAAI,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC;QAC7E,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAC7B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,QAAQ,GAAsB,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAA8B,EAAE;YAC3F,IACI,CAAC,OAAO;gBACR,OAAO,OAAO,CAAC,MAAM,KAAK,QAAQ;gBAClC,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;gBAC3B,CAAC,IAAI,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,EAC/D,CAAC;gBACC,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,iDAAiD;YACjD,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC7B,CAAC;IACL,CAAC;IAED,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,gBAAgB,CAAC,QAAsB,EAAE,IAAiB;;IACtE,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACvE,QAAQ,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC,CAAA,MAAA,QAAQ,CAAC,SAAS,0CAAE,MAAM,CAAA,KAAI,MAAA,IAAI,CAAC,SAAS,0CAAE,MAAM,CAAA,EAAE,CAAC;QACxD,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;IACxC,CAAC;IACD,IAAI,CAAC,CAAA,MAAA,QAAQ,CAAC,YAAY,0CAAE,MAAM,CAAA,KAAI,MAAA,IAAI,CAAC,YAAY,0CAAE,MAAM,CAAA,EAAE,CAAC;QAC9D,QAAQ,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;IAC9C,CAAC;IACD,IAAI,QAAQ,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACjE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IACtC,CAAC;IACD,IAAI,MAAA,IAAI,CAAC,QAAQ,0CAAE,MAAM,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,MAAA,QAAQ,CAAC,QAAQ,mCAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACvC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjB,OAAO,KAAK,CAAC;YACjB,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnB,QAAQ,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAA,QAAQ,CAAC,QAAQ,mCAAI,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,CAAC;QACjE,CAAC;IACL,CAAC;AACL,CAAC"}
|
package/lib/utils/getPrompt.d.ts
CHANGED
|
@@ -23,4 +23,12 @@ export declare function getTokens(prompt: string, modelLimit?: number): {
|
|
|
23
23
|
maxTokens: number;
|
|
24
24
|
};
|
|
25
25
|
export declare function getWebMarkdownPrompt(markdown: string, url: string): Prompt;
|
|
26
|
+
/**
|
|
27
|
+
* A narrow prompt for an about-us page: founding year, ownership, service areas, team size and licences, which the
|
|
28
|
+
* homepage often does not state.
|
|
29
|
+
*
|
|
30
|
+
* @param markdown The about-us page as markdown
|
|
31
|
+
* @param url URL of the about-us page
|
|
32
|
+
*/
|
|
33
|
+
export declare function getAboutUsPrompt(markdown: string, url: string): Prompt;
|
|
26
34
|
export declare function getJSONSchemaPrompt(json: any, url: string): Prompt;
|