@xapp/arachne-web-service 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +17 -37
- package/lib/ArachneWebService.js +160 -24
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/index.d.ts +1 -0
- package/lib/index.js +1 -0
- package/lib/index.js.map +1 -1
- package/lib/models.d.ts +106 -0
- package/lib/models.js +3 -0
- package/lib/models.js.map +1 -0
- package/lib/utils/getBusinessData.d.ts +4 -0
- package/lib/utils/getBusinessData.js +16 -0
- package/lib/utils/getBusinessData.js.map +1 -0
- package/lib/utils/getColors.d.ts +0 -0
- package/lib/utils/getColors.js +4 -0
- package/lib/utils/getColors.js.map +1 -0
- package/lib/utils/getPrompt.d.ts +11 -0
- package/lib/utils/getPrompt.js +25 -0
- package/lib/utils/getPrompt.js.map +1 -0
- package/lib/utils/getSiteMap.d.ts +13 -0
- package/lib/utils/getSiteMap.js +86 -0
- package/lib/utils/getSiteMap.js.map +1 -0
- package/lib/utils/image.d.ts +1 -0
- package/lib/utils/image.js +11 -1
- package/lib/utils/image.js.map +1 -1
- package/package.json +11 -11
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/*! Copyright (c) 2023, XAPP AI */
|
|
2
2
|
import "isomorphic-fetch";
|
|
3
3
|
import { DetectorResult } from "@xapp/arachne-parsers";
|
|
4
|
-
import {
|
|
4
|
+
import { WebsiteSchema, WebsiteDescriptiveData, ScreenshotLocation, BusinessData } from "./models";
|
|
5
5
|
export interface ArachneAppGetInfoOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Provide an openAI key to use OpenAI to determine the call to action of the website.
|
|
@@ -15,10 +15,17 @@ export interface ArachneAppGetInfoOptions {
|
|
|
15
15
|
* If not provided, you will not get primaryColor, secondaryColor, or images.
|
|
16
16
|
*/
|
|
17
17
|
screenshotsPath?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Provide a path for saving the markdown of the website
|
|
20
|
+
*/
|
|
21
|
+
markdownPath?: string;
|
|
18
22
|
/**
|
|
19
23
|
* Run headless, true by default.
|
|
20
24
|
*/
|
|
21
25
|
headless?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Override the viewport size for the browser window.
|
|
28
|
+
*/
|
|
22
29
|
viewportSize?: {
|
|
23
30
|
width: number;
|
|
24
31
|
height: number;
|
|
@@ -32,50 +39,22 @@ export interface ArachneAppGetInfoOptions {
|
|
|
32
39
|
*/
|
|
33
40
|
pageHandlerTimeout?: number;
|
|
34
41
|
}
|
|
35
|
-
export interface
|
|
36
|
-
localBusiness?: LocalBusiness;
|
|
37
|
-
}
|
|
38
|
-
export interface WebsiteDescriptiveData {
|
|
39
|
-
/**
|
|
40
|
-
* A square image that most likely is the logo.
|
|
41
|
-
*/
|
|
42
|
-
logo?: string;
|
|
43
|
-
/**
|
|
44
|
-
* Primary color detected on the website
|
|
45
|
-
*/
|
|
46
|
-
primaryColor?: string;
|
|
47
|
-
/**
|
|
48
|
-
* Secondary color that complements the primary color on the website.
|
|
49
|
-
*/
|
|
50
|
-
secondaryColor?: string;
|
|
42
|
+
export interface WebsiteData {
|
|
51
43
|
/**
|
|
52
|
-
*
|
|
44
|
+
* Descriptive metadata for the website.
|
|
53
45
|
*/
|
|
54
|
-
|
|
55
|
-
callToAction: string;
|
|
56
|
-
url?: string;
|
|
57
|
-
}[];
|
|
46
|
+
site: WebsiteDescriptiveData;
|
|
58
47
|
/**
|
|
59
|
-
* A
|
|
48
|
+
* A list of detected technologies
|
|
60
49
|
*/
|
|
61
|
-
|
|
50
|
+
detectedTechnologies: DetectorResult[];
|
|
62
51
|
/**
|
|
63
|
-
*
|
|
52
|
+
* Information related to the organization
|
|
64
53
|
*/
|
|
65
|
-
|
|
66
|
-
number: string;
|
|
67
|
-
purpose?: string;
|
|
68
|
-
}[];
|
|
69
|
-
}
|
|
70
|
-
export interface ScreenshotLocation {
|
|
54
|
+
business: BusinessData;
|
|
71
55
|
/**
|
|
72
|
-
*
|
|
56
|
+
* JSON+LD schema detected on the website
|
|
73
57
|
*/
|
|
74
|
-
filePath: string;
|
|
75
|
-
}
|
|
76
|
-
export interface WebsiteData {
|
|
77
|
-
site: WebsiteDescriptiveData;
|
|
78
|
-
detectedTechnologies: DetectorResult[];
|
|
79
58
|
schema: WebsiteSchema;
|
|
80
59
|
/**
|
|
81
60
|
* The screenshots that were taken from the website.
|
|
@@ -85,6 +64,7 @@ export interface WebsiteData {
|
|
|
85
64
|
export declare class ArachneWebService {
|
|
86
65
|
/**
|
|
87
66
|
* Get data on the provided website.
|
|
67
|
+
*
|
|
88
68
|
* @param url
|
|
89
69
|
* @param options
|
|
90
70
|
* @returns
|
package/lib/ArachneWebService.js
CHANGED
|
@@ -1,4 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
26
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
27
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -16,19 +39,24 @@ exports.ArachneWebService = void 0;
|
|
|
16
39
|
/*! Copyright (c) 2023, XAPP AI */
|
|
17
40
|
require("isomorphic-fetch"); // must be at top, making available to all
|
|
18
41
|
const path_1 = require("path");
|
|
42
|
+
const fs = __importStar(require("fs"));
|
|
43
|
+
const cheerio = __importStar(require("cheerio"));
|
|
19
44
|
const arachne_1 = require("@xapp/arachne");
|
|
20
45
|
const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
21
46
|
const stentor_logger_1 = require("stentor-logger");
|
|
22
47
|
const stentor_utils_1 = require("stentor-utils");
|
|
23
48
|
const node_vibrant_1 = __importDefault(require("node-vibrant"));
|
|
24
|
-
const { encode } = require('gpt-3-encoder');
|
|
25
49
|
const autoScroll_1 = require("./utils/autoScroll");
|
|
26
50
|
const findBestFaviconURL_1 = require("./utils/findBestFaviconURL");
|
|
27
51
|
const image_1 = require("./utils/image");
|
|
28
52
|
const isLambda_1 = require("./utils/isLambda");
|
|
53
|
+
const getSiteMap_1 = require("./utils/getSiteMap");
|
|
54
|
+
const getPrompt_1 = require("./utils/getPrompt");
|
|
55
|
+
const getBusinessData_1 = require("./utils/getBusinessData");
|
|
29
56
|
class ArachneWebService {
|
|
30
57
|
/**
|
|
31
58
|
* Get data on the provided website.
|
|
59
|
+
*
|
|
32
60
|
* @param url
|
|
33
61
|
* @param options
|
|
34
62
|
* @returns
|
|
@@ -39,6 +67,30 @@ class ArachneWebService {
|
|
|
39
67
|
const detectedTechnologies = [];
|
|
40
68
|
const schema = {};
|
|
41
69
|
const screenshots = [];
|
|
70
|
+
const business = {};
|
|
71
|
+
// first get the sitemap
|
|
72
|
+
const siteUrl = new URL(url);
|
|
73
|
+
const originalPath = siteUrl.pathname;
|
|
74
|
+
const sitemapUrl = new URL(url);
|
|
75
|
+
sitemapUrl.pathname = `${originalPath}sitemap.xml`;
|
|
76
|
+
(0, stentor_logger_1.log)().debug(`Loading sitemap at ${sitemapUrl.toString()}`);
|
|
77
|
+
const sitemap = yield (0, getSiteMap_1.getSitemap)(sitemapUrl.toString());
|
|
78
|
+
// we could have been passed a URL that already has a pathname, we filter by those first
|
|
79
|
+
const filtered = sitemap.filter((value) => {
|
|
80
|
+
const siteUrl = new URL(value);
|
|
81
|
+
const sitePath = siteUrl.pathname;
|
|
82
|
+
return sitePath.startsWith(originalPath);
|
|
83
|
+
});
|
|
84
|
+
(0, stentor_logger_1.log)().debug(`Sitemap has ${sitemap.length} pages, filtered to ${filtered.length}`);
|
|
85
|
+
// find about us page
|
|
86
|
+
const aboutUs = filtered.find((value) => {
|
|
87
|
+
const url = new URL(value);
|
|
88
|
+
const path = url.pathname;
|
|
89
|
+
// provided by OpenAI
|
|
90
|
+
const aboutUsRegex = /\/(about(-us)?|company\/about|who-we-are|our-story|info\/about|en\/about|about\/our-company)(\/)?$/i;
|
|
91
|
+
return aboutUsRegex.test(path);
|
|
92
|
+
});
|
|
93
|
+
(0, stentor_logger_1.log)().debug(`Found an about us page at ${aboutUs}`);
|
|
42
94
|
const queue = new arachne_1.MemoryRequestQueue();
|
|
43
95
|
queue.addRequest({ url });
|
|
44
96
|
const launchOptions = {
|
|
@@ -57,7 +109,7 @@ class ArachneWebService {
|
|
|
57
109
|
}
|
|
58
110
|
let browser;
|
|
59
111
|
try {
|
|
60
|
-
(0, stentor_logger_1.log)().debug('Looking for chrome-aws-lambda');
|
|
112
|
+
(0, stentor_logger_1.log)().debug('Looking for chrome-aws-lambda...');
|
|
61
113
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
62
114
|
const chromium = require('@sparticuz/chrome-aws-lambda');
|
|
63
115
|
browser = yield chromium.puppeteer.launch({
|
|
@@ -80,6 +132,8 @@ class ArachneWebService {
|
|
|
80
132
|
pageHandlerTimeout,
|
|
81
133
|
pageHandler: (page, request) => __awaiter(this, void 0, void 0, function* () {
|
|
82
134
|
const title = yield page.title();
|
|
135
|
+
const u = new URL(url);
|
|
136
|
+
const fileName = `${u.hostname}${u.pathname}`.replace(/[./]/g, "_");
|
|
83
137
|
(0, stentor_logger_1.log)().info(`Loaded ${request.url} : ${title}`);
|
|
84
138
|
if (pageWait) {
|
|
85
139
|
(0, stentor_logger_1.log)().info(`Waiting for ${pageWaitTime} ms for page to completely load.`);
|
|
@@ -91,6 +145,8 @@ class ArachneWebService {
|
|
|
91
145
|
// page.content() gets you the HTML after it has been modified by the JavaScript
|
|
92
146
|
// on the page while response.text() only gives you the HTML from the original GET
|
|
93
147
|
const html = yield page.content();
|
|
148
|
+
const $ = cheerio.load(html);
|
|
149
|
+
const description = $('meta[name="description"]').attr('content');
|
|
94
150
|
const detectors = [
|
|
95
151
|
new arachne_parsers_1.XAPPDetector(html),
|
|
96
152
|
new arachne_parsers_1.SurefireDetector(html)
|
|
@@ -98,12 +154,22 @@ class ArachneWebService {
|
|
|
98
154
|
detectedTechnologies.push(...new arachne_parsers_1.MultiDetector(html, detectors).detect());
|
|
99
155
|
const things = new arachne_parsers_1.ArachneJSONLD(html).getThings();
|
|
100
156
|
let localBusiness;
|
|
157
|
+
let homeConstructionBusiness;
|
|
158
|
+
(0, stentor_logger_1.log)().info('Looking through JSON Schema...');
|
|
101
159
|
for (const thing of things) {
|
|
102
160
|
if ((0, arachne_parsers_1.isLocalBusiness)(thing)) {
|
|
103
161
|
(0, stentor_logger_1.log)().debug(`Found local business information`);
|
|
104
162
|
localBusiness = thing;
|
|
105
163
|
schema.localBusiness = thing;
|
|
106
164
|
}
|
|
165
|
+
else if ((0, arachne_parsers_1.isHomeConstructionBusiness)(thing)) {
|
|
166
|
+
(0, stentor_logger_1.log)().debug(`Found home construction business`);
|
|
167
|
+
homeConstructionBusiness = thing;
|
|
168
|
+
schema.homeConstructionBusiness = thing;
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
(0, stentor_logger_1.log)().debug(`Found JSON Schema ${thing["@id"]} ${thing["@type"]}`);
|
|
172
|
+
}
|
|
107
173
|
}
|
|
108
174
|
// We are going to try to figure out the icon!
|
|
109
175
|
if (localBusiness && typeof localBusiness !== "string") {
|
|
@@ -127,9 +193,7 @@ class ArachneWebService {
|
|
|
127
193
|
}
|
|
128
194
|
// Try to figure out primary color, yup
|
|
129
195
|
if (options.screenshotsPath) {
|
|
130
|
-
const
|
|
131
|
-
const imageName = `${u.hostname}${u.pathname}`.replace(/[./]/g, "_");
|
|
132
|
-
const path = (0, path_1.resolve)(options.screenshotsPath, `${imageName}.png`);
|
|
196
|
+
const path = (0, path_1.resolve)(options.screenshotsPath, `${fileName}.png`);
|
|
133
197
|
(0, stentor_logger_1.log)().debug(`Saving screenshot to ${path}`);
|
|
134
198
|
const captureBeyondViewport = options.viewportSize ? false : true;
|
|
135
199
|
const fullPage = options.viewportSize ? false : true;
|
|
@@ -141,7 +205,8 @@ class ArachneWebService {
|
|
|
141
205
|
screenshots.push({
|
|
142
206
|
filePath: path
|
|
143
207
|
});
|
|
144
|
-
(0, stentor_logger_1.log)().debug(`Screenshot saved
|
|
208
|
+
(0, stentor_logger_1.log)().debug(`Screenshot saved.`);
|
|
209
|
+
(0, stentor_logger_1.log)().debug(`Analyzing colors...`);
|
|
145
210
|
const v = new node_vibrant_1.default(path);
|
|
146
211
|
const palette = yield v.getPalette();
|
|
147
212
|
let swatchName = undefined;
|
|
@@ -158,7 +223,18 @@ class ArachneWebService {
|
|
|
158
223
|
swatchName = swatch;
|
|
159
224
|
mostPopular = current;
|
|
160
225
|
}
|
|
161
|
-
|
|
226
|
+
switch (swatch) {
|
|
227
|
+
case "LightVibrant":
|
|
228
|
+
case "Vibrant":
|
|
229
|
+
site.vibrantColor = current === null || current === void 0 ? void 0 : current.hex;
|
|
230
|
+
continue;
|
|
231
|
+
case "DarkVibrant":
|
|
232
|
+
site.darkVibrantColor = current === null || current === void 0 ? void 0 : current.hex;
|
|
233
|
+
continue;
|
|
234
|
+
default:
|
|
235
|
+
}
|
|
236
|
+
const debugPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(current === null || current === void 0 ? void 0 : current.hex) + 'm \x1b[0m';
|
|
237
|
+
debugSwatches = `${debugSwatches} ${swatch} (${current === null || current === void 0 ? void 0 : current.population}):${current === null || current === void 0 ? void 0 : current.hex} ${debugPreview} `;
|
|
162
238
|
}
|
|
163
239
|
}
|
|
164
240
|
if (swatchName) {
|
|
@@ -169,24 +245,64 @@ class ArachneWebService {
|
|
|
169
245
|
}
|
|
170
246
|
site.primaryColor = mostPopular === null || mostPopular === void 0 ? void 0 : mostPopular.hex;
|
|
171
247
|
site.secondaryColor = (0, image_1.complementHexColor)(site.primaryColor);
|
|
172
|
-
(0,
|
|
173
|
-
(0,
|
|
248
|
+
const primaryPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.primaryColor) + 'm \x1b[0m';
|
|
249
|
+
const secondaryPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.secondaryColor) + 'm \x1b[0m';
|
|
250
|
+
const vibrantPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.vibrantColor) + 'm \x1b[0m';
|
|
251
|
+
(0, stentor_logger_1.log)().debug(`Primary color: ${site.primaryColor} ${primaryPreview}`);
|
|
252
|
+
(0, stentor_logger_1.log)().debug(`Secondary color: ${site.secondaryColor} ${secondaryPreview}`);
|
|
253
|
+
(0, stentor_logger_1.log)().debug(`Vibrant color: ${site.vibrantColor} ${vibrantPreview}`);
|
|
254
|
+
(0, stentor_logger_1.log)().debug(`All swatches are: ${debugSwatches}`);
|
|
174
255
|
}
|
|
175
256
|
else {
|
|
176
257
|
(0, stentor_logger_1.log)().info(`screenshotsPath not provided, cannot determine primaryColor and secondaryColor`);
|
|
177
258
|
}
|
|
259
|
+
// Convert to markdown
|
|
260
|
+
const markdown = new arachne_parsers_1.ArachneHTMLtoText({ stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"] }).convert(html, true);
|
|
261
|
+
if (options.markdownPath) {
|
|
262
|
+
const path = (0, path_1.resolve)(options.markdownPath, `${fileName}.md`);
|
|
263
|
+
(0, stentor_logger_1.log)().debug(`Writing markdown to ${path}...`);
|
|
264
|
+
yield fs.promises.writeFile(path, markdown);
|
|
265
|
+
(0, stentor_logger_1.log)().debug(`Written.`);
|
|
266
|
+
}
|
|
267
|
+
let aiData = undefined;
|
|
178
268
|
const { openAIKey = process.env.OPENAI_API_KEY } = options;
|
|
179
269
|
// Use OpenAI to determine Call To Action
|
|
180
270
|
if (openAIKey) {
|
|
181
271
|
(0, stentor_logger_1.log)().info(`Calling OpenAI....`);
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
272
|
+
const webPrompt = (0, getPrompt_1.getWebMarkdownPrompt)(markdown, url);
|
|
273
|
+
let prompt = webPrompt.prompt;
|
|
274
|
+
let maxTokens = webPrompt.maxTokens;
|
|
275
|
+
let tokens = webPrompt.tokens;
|
|
276
|
+
// first try if over max tokens
|
|
277
|
+
if (maxTokens < 0 && aboutUs) {
|
|
278
|
+
// load about us
|
|
279
|
+
(0, stentor_logger_1.log)().info(`Too many tokens, attempting to load about us page at ${aboutUs}...`);
|
|
280
|
+
yield page.goto(aboutUs);
|
|
281
|
+
(0, stentor_logger_1.log)().info(`Loaded...`);
|
|
282
|
+
const aboutUsHTML = yield page.content();
|
|
283
|
+
const aboutUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({ stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"] }).convert(aboutUsHTML, true);
|
|
284
|
+
const aboutUsPrompt = (0, getPrompt_1.getWebMarkdownPrompt)(aboutUsMarkdown, url);
|
|
285
|
+
(0, stentor_logger_1.log)().info(`About Us tokens: ${aboutUsPrompt.tokens} max: ${aboutUsPrompt.maxTokens}`);
|
|
286
|
+
if (aboutUsPrompt.maxTokens > 0) {
|
|
287
|
+
(0, stentor_logger_1.log)().info(`Using about us page.`);
|
|
288
|
+
prompt = aboutUsPrompt.prompt;
|
|
289
|
+
maxTokens = aboutUsPrompt.maxTokens;
|
|
290
|
+
tokens = aboutUsPrompt.tokens;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (maxTokens < 0 && localBusiness || homeConstructionBusiness) {
|
|
294
|
+
(0, stentor_logger_1.log)().info(`Too many tokens, attempting to use LocalBusiness schema to generate description.`);
|
|
295
|
+
// try second prompt where we send the JSON.
|
|
296
|
+
const jsonSchemaPrompt = (0, getPrompt_1.getJSONSchemaPrompt)(localBusiness || homeConstructionBusiness, url);
|
|
297
|
+
(0, stentor_logger_1.log)().info(`JSON Schema tokens: ${jsonSchemaPrompt.tokens} max: ${jsonSchemaPrompt.maxTokens}`);
|
|
298
|
+
if (jsonSchemaPrompt.maxTokens > 0) {
|
|
299
|
+
(0, stentor_logger_1.log)().info(`LocalBusiness JSON fits within token limit, using it.`);
|
|
300
|
+
prompt = jsonSchemaPrompt.prompt;
|
|
301
|
+
maxTokens = jsonSchemaPrompt.maxTokens;
|
|
302
|
+
tokens = jsonSchemaPrompt.tokens;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
(0, stentor_logger_1.log)().info(`Final prompt is ${tokens} tokens, setting max_tokens to ${maxTokens}.`);
|
|
190
306
|
const options = {
|
|
191
307
|
method: 'POST',
|
|
192
308
|
headers: {
|
|
@@ -197,7 +313,7 @@ class ArachneWebService {
|
|
|
197
313
|
model: "text-davinci-003",
|
|
198
314
|
prompt,
|
|
199
315
|
temperature: 0.25,
|
|
200
|
-
max_tokens,
|
|
316
|
+
max_tokens: maxTokens,
|
|
201
317
|
top_p: 1,
|
|
202
318
|
frequency_penalty: 0,
|
|
203
319
|
presence_penalty: 0
|
|
@@ -211,7 +327,7 @@ class ArachneWebService {
|
|
|
211
327
|
if (status === 200) {
|
|
212
328
|
return response.json();
|
|
213
329
|
}
|
|
214
|
-
return;
|
|
330
|
+
return response;
|
|
215
331
|
}).finally(() => {
|
|
216
332
|
// eslint-disable-next-line no-console
|
|
217
333
|
console.timeEnd("open_ai_web");
|
|
@@ -221,17 +337,36 @@ class ArachneWebService {
|
|
|
221
337
|
(0, stentor_logger_1.log)().info(`OpenAI token usage: prompt ${usage.prompt_tokens} completion ${usage.completion_tokens} total ${usage.total_tokens}`);
|
|
222
338
|
// Add the leading { as it is included in the query
|
|
223
339
|
const completion = `{\n${openAIResponse.choices[0].text}`;
|
|
224
|
-
|
|
340
|
+
aiData = JSON.parse(completion);
|
|
341
|
+
(0, stentor_logger_1.log)().debug(JSON.stringify(aiData, undefined, 2));
|
|
225
342
|
// eslint-disable-next-line no-console
|
|
226
|
-
site.callsToAction = siteData.calls_to_action;
|
|
227
|
-
site.description = siteData.description;
|
|
228
|
-
site.phoneNumbers = siteData.phone_numbers;
|
|
343
|
+
// site.callsToAction = siteData.calls_to_action;
|
|
344
|
+
// site.description = siteData.description;
|
|
345
|
+
// site.phoneNumbers = siteData.phone_numbers;
|
|
229
346
|
}
|
|
230
347
|
else {
|
|
231
348
|
(0, stentor_logger_1.log)().warn(`Unabled to get website info from OpenAI status:${status}`);
|
|
232
349
|
(0, stentor_logger_1.log)().debug(openAIResponse);
|
|
233
350
|
}
|
|
234
351
|
}
|
|
352
|
+
const businessData = (0, getBusinessData_1.getBusinessData)(schema.localBusiness || schema.homeConstructionBusiness);
|
|
353
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.organization_name) {
|
|
354
|
+
business.name = aiData.organization_name;
|
|
355
|
+
}
|
|
356
|
+
else if (businessData.name) {
|
|
357
|
+
business.name = businessData.name;
|
|
358
|
+
}
|
|
359
|
+
// Set some items
|
|
360
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.description) {
|
|
361
|
+
site.description = aiData.description;
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
// what is our alternative here?
|
|
365
|
+
site.description = description;
|
|
366
|
+
}
|
|
367
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.calls_to_action) {
|
|
368
|
+
site.callsToAction = aiData.calls_to_action;
|
|
369
|
+
}
|
|
235
370
|
else {
|
|
236
371
|
// Try to figure out the call to action
|
|
237
372
|
// TODO: Can we use our model export and a simple NLU to better find possible matches?
|
|
@@ -254,7 +389,8 @@ class ArachneWebService {
|
|
|
254
389
|
site,
|
|
255
390
|
detectedTechnologies,
|
|
256
391
|
schema,
|
|
257
|
-
screenshots
|
|
392
|
+
screenshots,
|
|
393
|
+
business
|
|
258
394
|
};
|
|
259
395
|
});
|
|
260
396
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ArachneWebService.js","sourceRoot":"","sources":["../src/ArachneWebService.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AACrE,+BAA+B;AAE/B,2CAAyF;AACzF,2DAU+B;AAE/B,mDAAqC;AACrC,iDAA8D;AAG9D,gEAAmC;AAGnC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAG5C,mDAAgD;AAChD,mEAAgE;AAChE,yCAAkE;AAClE,+CAA4C;AAoF5C,MAAa,iBAAiB;IAC1B;;;;;OAKG;IACU,cAAc,CAAC,GAAW,EAAE,UAAoC,EAAE;;YAE3E,MAAM,IAAI,GAA2B,EAAE,CAAC;YACxC,MAAM,oBAAoB,GAAqB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAkB,EAAE,CAAC;YACjC,MAAM,WAAW,GAAyB,EAAE,CAAC;YAE7C,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;YACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1B,MAAM,aAAa,GAAyE;gBACxF,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,OAAO,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC;YAEF,MAAM,QAAQ,GAAY,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAC9E,MAAM,YAAY,GAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAA,mBAAQ,GAAE,EAAE;gBACb,8BAA8B;gBAC9B,aAAa,CAAC,cAAc,GAAG,wBAAwB,CAAC;aAC3D;YAED,IAAI,OAAO,CAAC,YAAY,EAAE;gBACtB,aAAa,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC;gBACrD,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;aACvG;YAED,IAAI,OAAuD,CAAC;YAE5D,IAAI;gBACA,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAE7C,8DAA8D;gBAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;gBAEzD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;oBACtC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,eAAe,EAAE,QAAQ,CAAC,eAAe;oBACzC,cAAc,EAAE,MAAM,QAAQ,CAAC,cAAc;oBAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,iBAAiB,EAAE,IAAI;iBAC1B,CAAC,CAAC;aACN;YAAC,OAAO,CAAC,EAAE;gBACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;aAChG;YAED,MAAM,kBAAkB,GAAW,OAAO,OAAO,CAAC,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvH,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;gBAC5B,OAAO,EAAE,IAAI;gBACb,aAAa;gBACb,KAAK;gBACL,OAAO;gBACP,kBAAkB;gBAClB,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,EAAE;oBAE9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;oBAE/C,IAAI,QAAQ,EAAE;wBACV,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,eAAe,YAAY,kCAAkC,CAAC,CAAC;wBAC1E,MAAM,IAAA,uBAAU,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;wBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;qBACtC;oBAED,mBAAmB;oBACnB,qEAAqE;oBACrE,gFAAgF;oBAChF,kFAAkF;oBAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAElC,MAAM,SAAS,GAAuB;wBAClC,IAAI,8BAAY,CAAC,IAAI,CAAC;wBACtB,IAAI,kCAAgB,CAAC,IAAI,CAAC;qBAC7B,CAAC;oBAEF,oBAAoB,CAAC,IAAI,CAAC,GAAG,IAAI,+BAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE1E,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;oBAEnD,IAAI,aAAwC,CAAC;oBAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;wBACxB,IAAI,IAAA,iCAAe,EAAC,KAAK,CAAC,EAAE;4BACxB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC;yBAChC;qBACJ;oBAED,8CAA8C;oBAC9C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;wBACpD,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC;wBAC7C,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;4BACvC,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;4BAC1D,IAAI,MAAM,EAAE;gCACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,iBAAiB,EAAE,CAAC,CAAC;gCACrE,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;6BACjC;yBACJ;qBACJ;oBAED,0BAA0B;oBAC1B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBACZ,kBAAkB;wBAClB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;wBAE7C,MAAM,OAAO,GAAG,MAAM,IAAA,uCAAkB,EAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,OAAO,aAAa,CAAC,CAAC;wBACrD,8CAA8C;wBAC9C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;qBACvB;oBAED,uCAAuC;oBACvC,IAAI,OAAO,CAAC,eAAe,EAAE;wBACzB,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;wBACvB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBACrE,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,eAAe,EAAE,GAAG,SAAS,MAAM,CAAC,CAAC;wBAClE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;wBAE5C,MAAM,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAErD,MAAM,IAAI,CAAC,UAAU,CAAC;4BAClB,IAAI;4BACJ,QAAQ;4BACR,qBAAqB;yBACxB,CAAC,CAAC;wBAEH,WAAW,CAAC,IAAI,CAAC;4BACb,QAAQ,EAAE,IAAI;yBACjB,CAAC,CAAC;wBAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;wBAElD,MAAM,CAAC,GAAG,IAAI,sBAAO,CAAC,IAAI,CAAC,CAAC;wBAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;wBACrC,IAAI,UAAU,GAAuB,SAAS,CAAC;wBAC/C,IAAI,WAAW,CAAC;wBAChB,IAAI,aAAa,GAAG,EAAE,CAAC;wBAEvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;4BAC1B,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE;gCACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gCAChC,IAAI,CAAC,WAAW,EAAE;oCACd,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;qCAAM,IAAI,OAAO,IAAI,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE;oCAC/D,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;gCACD,aAAa,GAAG,GAAG,aAAa,IAAI,MAAM,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,EAAE,CAAA;6BAC/D;yBACJ;wBACD,IAAI,UAAU,EAAE;4BACZ,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,UAAU,IAAI,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,EAAE,CAAC,CAAC;yBACxE;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;yBACxD;wBAED,IAAI,CAAC,YAAY,GAAG,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,CAAC;wBACrC,IAAI,CAAC,cAAc,GAAG,IAAA,0BAAkB,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;wBAC9D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,aAAa,EAAE,CAAC,CAAC;qBAC7D;yBAAM;wBACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;qBAChG;oBAED,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC;oBAC3D,yCAAyC;oBACzC,IAAI,SAAS,EAAE;wBAEX,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;wBACjC,sBAAsB;wBACtB,MAAM,QAAQ,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;wBAC1G,uBAAuB;wBACvB,MAAM,MAAM,GAAG,GAAG,QAAQ,8EAA8E,GAAG,+yCAA+yC,CAAC;wBAE35C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;wBACrC,MAAM,iBAAiB,GAAG,GAAG,CAAC;wBAC9B,MAAM,UAAU,GAAW,IAAI,GAAG,iBAAiB,GAAG,MAAM,CAAC;wBAE7D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,aAAa,MAAM,kCAAkC,UAAU,EAAE,CAAC,CAAC;wBAE9E,MAAM,OAAO,GAAgB;4BACzB,MAAM,EAAE,MAAM;4BACd,OAAO,EAAE;gCACL,cAAc,EAAE,kBAAkB;gCAClC,eAAe,EAAE,UAAU,SAAS,EAAE;6BACzC;4BACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACjB,KAAK,EAAE,kBAAkB;gCACzB,MAAM;gCACN,WAAW,EAAE,IAAI;gCACjB,UAAU;gCACV,KAAK,EAAE,CAAC;gCACR,iBAAiB,EAAE,CAAC;gCACpB,gBAAgB,EAAE,CAAC;6BACtB,CAAC;yBACL,CAAA;wBAED,sCAAsC;wBACtC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;wBAC5B,IAAI,MAAM,GAAW,GAAG,CAAC;wBACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;4BACnG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;4BACzB,IAAI,MAAM,KAAK,GAAG,EAAE;gCAChB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;6BAC1B;4BACD,OAAO;wBACX,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;4BACZ,sCAAsC;4BACtC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;wBACnC,CAAC,CAAC,CAAC;wBAEH,IAAI,MAAM,KAAK,GAAG,EAAE;4BAUhB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;4BACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;4BAEjI,mDAAmD;4BACnD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC1D,MAAM,QAAQ,GAAc,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BACnD,sCAAsC;4BAEtC,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC,eAAe,CAAC;4BAC9C,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;4BACxC,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC;yBAC9C;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;4BACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;yBAC/B;qBACJ;yBAAM;wBACH,uCAAuC;wBACvC,sFAAsF;wBACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mHAAmH,CAAC,CAAC;wBAE9I,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAElC,sBAAsB;wBACtB,IAAI,OAAO,IAAI,IAAA,iCAAiB,EAAC,OAAO,CAAC,EAAE;4BACvC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAC;4BACtD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;4BACvC,MAAM,OAAO,GAAW,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAU,EAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAC7E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,OAAO,GAAG,CAAC,CAAC;4BACtC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;yBACpD;qBACJ;gBACL,CAAC,CAAA;gBACD,0BAA0B;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;YAEpB,OAAO;gBACH,IAAI;gBACJ,oBAAoB;gBACpB,MAAM;gBACN,WAAW;aACd,CAAC;QACN,CAAC;KAAA;CACJ;AArRD,8CAqRC"}
|
|
1
|
+
{"version":3,"file":"ArachneWebService.js","sourceRoot":"","sources":["../src/ArachneWebService.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AACrE,+BAA+B;AAC/B,uCAAyB;AACzB,iDAAmC;AAEnC,2CAAyF;AACzF,2DAW+B;AAE/B,mDAAqC;AACrC,iDAA8D;AAG9D,gEAAmC;AAInC,mDAAgD;AAChD,mEAAgE;AAChE,yCAA4E;AAC5E,+CAA4C;AAC5C,mDAAgD;AAChD,iDAA8E;AAC9E,6DAA0D;AA+D1D,MAAa,iBAAiB;IAC1B;;;;;;OAMG;IACU,cAAc,CAAC,GAAW,EAAE,UAAoC,EAAE;;YAE3E,MAAM,IAAI,GAA2B,EAAE,CAAC;YACxC,MAAM,oBAAoB,GAAqB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAkB,EAAE,CAAC;YACjC,MAAM,WAAW,GAAyB,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAiB,EAAE,CAAC;YAElC,wBAAwB;YAExB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEtC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,UAAU,CAAC,QAAQ,GAAG,GAAG,YAAY,aAAa,CAAC;YACnD,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,sBAAsB,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE3D,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAU,EAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAExD,wFAAwF;YACxF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAClC,OAAO,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,eAAe,OAAO,CAAC,MAAM,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnF,qBAAqB;YACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;gBACpC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC1B,qBAAqB;gBACrB,MAAM,YAAY,GAAG,qGAAqG,CAAC;gBAE3H,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;YAEpD,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;YACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1B,MAAM,aAAa,GAAyE;gBACxF,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,OAAO,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC;YAEF,MAAM,QAAQ,GAAY,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAC9E,MAAM,YAAY,GAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAA,mBAAQ,GAAE,EAAE;gBACb,8BAA8B;gBAC9B,aAAa,CAAC,cAAc,GAAG,wBAAwB,CAAC;aAC3D;YAED,IAAI,OAAO,CAAC,YAAY,EAAE;gBACtB,aAAa,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC;gBACrD,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;aACvG;YAED,IAAI,OAAuD,CAAC;YAE5D,IAAI;gBACA,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;gBAEhD,8DAA8D;gBAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;gBAEzD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;oBACtC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,eAAe,EAAE,QAAQ,CAAC,eAAe;oBACzC,cAAc,EAAE,MAAM,QAAQ,CAAC,cAAc;oBAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,iBAAiB,EAAE,IAAI;iBAC1B,CAAC,CAAC;aACN;YAAC,OAAO,CAAC,EAAE;gBACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;aAChG;YAED,MAAM,kBAAkB,GAAW,OAAO,OAAO,CAAC,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvH,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;gBAC5B,OAAO,EAAE,IAAI;gBACb,aAAa;gBACb,KAAK;gBACL,OAAO;gBACP,kBAAkB;gBAClB,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,EAAE;oBAE9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACpE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;oBAE/C,IAAI,QAAQ,EAAE;wBACV,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,eAAe,YAAY,kCAAkC,CAAC,CAAC;wBAC1E,MAAM,IAAA,uBAAU,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;wBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;qBACtC;oBAED,mBAAmB;oBACnB,qEAAqE;oBACrE,gFAAgF;oBAChF,kFAAkF;oBAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAE3C,MAAM,WAAW,GAAG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAElE,MAAM,SAAS,GAAuB;wBAClC,IAAI,8BAAY,CAAC,IAAI,CAAC;wBACtB,IAAI,kCAAgB,CAAC,IAAI,CAAC;qBAC7B,CAAC;oBAEF,oBAAoB,CAAC,IAAI,CAAC,GAAG,IAAI,+BAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE1E,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;oBAEnD,IAAI,aAAwC,CAAC;oBAC7C,IAAI,wBAAiE,CAAC;oBAEtE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;oBAC7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;wBACxB,IAAI,IAAA,iCAAe,EAAC,KAAK,CAAC,EAAE;4BACxB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC;yBAChC;6BAAM,IAAI,IAAA,4CAA0B,EAAC,KAAK,CAAC,EAAE;4BAC1C,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,wBAAwB,GAAG,KAAK,CAAC;4BACjC,MAAM,CAAC,wBAAwB,GAAG,KAAK,CAAC;yBAC3C;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;yBACtE;qBACJ;oBAED,8CAA8C;oBAC9C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;wBACpD,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC;wBAC7C,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;4BACvC,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;4BAC1D,IAAI,MAAM,EAAE;gCACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,iBAAiB,EAAE,CAAC,CAAC;gCACrE,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;6BACjC;yBACJ;qBACJ;oBAED,0BAA0B;oBAC1B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBACZ,kBAAkB;wBAClB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;wBAE7C,MAAM,OAAO,GAAG,MAAM,IAAA,uCAAkB,EAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,OAAO,aAAa,CAAC,CAAC;wBACrD,8CAA8C;wBAC9C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;qBACvB;oBAED,uCAAuC;oBACvC,IAAI,OAAO,CAAC,eAAe,EAAE;wBAEzB,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,eAAe,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC;wBAEjE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;wBAE5C,MAAM,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAErD,MAAM,IAAI,CAAC,UAAU,CAAC;4BAClB,IAAI;4BACJ,QAAQ;4BACR,qBAAqB;yBACxB,CAAC,CAAC;wBAEH,WAAW,CAAC,IAAI,CAAC;4BACb,QAAQ,EAAE,IAAI;yBACjB,CAAC,CAAC;wBACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;wBAEjC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;wBACnC,MAAM,CAAC,GAAG,IAAI,sBAAO,CAAC,IAAI,CAAC,CAAC;wBAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;wBACrC,IAAI,UAAU,GAAuB,SAAS,CAAC;wBAC/C,IAAI,WAAW,CAAC;wBAChB,IAAI,aAAa,GAAG,EAAE,CAAC;wBAEvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;4BAC1B,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE;gCACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gCAChC,IAAI,CAAC,WAAW,EAAE;oCACd,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;qCAAM,IAAI,OAAO,IAAI,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE;oCAC/D,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;gCAED,QAAQ,MAAM,EAAE;oCACZ,KAAK,cAAc,CAAC;oCACpB,KAAK,SAAS;wCACV,IAAI,CAAC,YAAY,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC;wCACjC,SAAS;oCACb,KAAK,aAAa;wCACd,IAAI,CAAC,gBAAgB,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC;wCACrC,SAAS;oCACb,QAAQ;iCACX;gCAED,MAAM,YAAY,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC,GAAG,gBAAgB,CAAC;gCAC9E,aAAa,GAAG,GAAG,aAAa,IAAI,MAAM,KAAK,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAK,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,IAAI,YAAY,GAAG,CAAA;6BACzG;yBACJ;wBACD,IAAI,UAAU,EAAE;4BACZ,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,UAAU,IAAI,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,EAAE,CAAC,CAAC;yBACxE;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;yBACxD;wBAED,IAAI,CAAC,YAAY,GAAG,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,CAAC;wBACrC,IAAI,CAAC,cAAc,GAAG,IAAA,0BAAkB,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAE5D,MAAM,cAAc,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAC;wBACrF,MAAM,gBAAgB,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,cAAc,CAAC,GAAG,gBAAgB,CAAC;wBACzF,MAAM,cAAc,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAA;wBAEpF,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,YAAY,IAAI,cAAc,EAAE,CAAC,CAAC;wBACrE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,cAAc,IAAI,gBAAgB,EAAE,CAAC,CAAC;wBAC3E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,YAAY,IAAI,cAAc,EAAE,CAAC,CAAC;wBACrE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,aAAa,EAAE,CAAC,CAAC;qBACrD;yBAAM;wBACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;qBAChG;oBAED,sBAAsB;oBACtB,MAAM,QAAQ,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAErI,IAAI,OAAO,CAAC,YAAY,EAAE;wBACtB,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;wBAC7D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,IAAI,KAAK,CAAC,CAAC;wBAC9C,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;wBAC5C,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;qBAC1B;oBAUD,IAAI,MAAM,GAA0B,SAAS,CAAC;oBAE9C,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC;oBAC3D,yCAAyC;oBACzC,IAAI,SAAS,EAAE;wBAEX,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;wBAEjC,MAAM,SAAS,GAAG,IAAA,gCAAoB,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBAEtD,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;wBACtC,IAAI,SAAS,GAAW,SAAS,CAAC,SAAS,CAAC;wBAC5C,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;wBAEtC,+BAA+B;wBAC/B,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,EAAE;4BAC1B,iBAAiB;4BACjB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,wDAAwD,OAAO,KAAK,CAAC,CAAC;4BACjF,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;4BACzB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BACxB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;4BACzC,MAAM,eAAe,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;4BAEnJ,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;4BAEjE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;4BAEvF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE;gCAC7B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;gCACnC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gCAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;gCACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;6BACjC;yBACJ;wBAED,IAAI,SAAS,GAAG,CAAC,IAAI,aAAa,IAAI,wBAAwB,EAAE;4BAC5D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;4BAC/F,4CAA4C;4BAE5C,MAAM,gBAAgB,GAAG,IAAA,+BAAmB,EAAC,aAAa,IAAI,wBAAwB,EAAE,GAAG,CAAC,CAAC;4BAE7F,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,gBAAgB,CAAC,MAAM,SAAS,gBAAgB,CAAC,SAAS,EAAE,CAAC,CAAC;4BAEhG,IAAI,gBAAgB,CAAC,SAAS,GAAG,CAAC,EAAE;gCAChC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAA;gCACnE,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;gCACjC,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;gCACvC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;6BACpC;yBACJ;wBAED,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,MAAM,kCAAkC,SAAS,GAAG,CAAC,CAAC;wBAEpF,MAAM,OAAO,GAAgB;4BACzB,MAAM,EAAE,MAAM;4BACd,OAAO,EAAE;gCACL,cAAc,EAAE,kBAAkB;gCAClC,eAAe,EAAE,UAAU,SAAS,EAAE;6BACzC;4BACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACjB,KAAK,EAAE,kBAAkB;gCACzB,MAAM;gCACN,WAAW,EAAE,IAAI;gCACjB,UAAU,EAAE,SAAS;gCACrB,KAAK,EAAE,CAAC;gCACR,iBAAiB,EAAE,CAAC;gCACpB,gBAAgB,EAAE,CAAC;6BACtB,CAAC;yBACL,CAAA;wBAED,sCAAsC;wBACtC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;wBAC5B,IAAI,MAAM,GAAW,GAAG,CAAC;wBACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;4BACnG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;4BACzB,IAAI,MAAM,KAAK,GAAG,EAAE;gCAChB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;6BAC1B;4BACD,OAAO,QAAQ,CAAC;wBACpB,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;4BACZ,sCAAsC;4BACtC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;wBACnC,CAAC,CAAC,CAAC;wBAEH,IAAI,MAAM,KAAK,GAAG,EAAE;4BAEhB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;4BACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;4BAEjI,mDAAmD;4BACnD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC1D,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BAEhC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;4BAClD,sCAAsC;4BAEtC,iDAAiD;4BACjD,4CAA4C;4BAC5C,8CAA8C;yBACjD;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;4BACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;yBAC/B;qBACJ;oBAED,MAAM,YAAY,GAAG,IAAA,iCAAe,EAAC,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,wBAAwB,CAAC,CAAC;oBAE9F,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,EAAE;wBAC3B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC;qBAC5C;yBAAM,IAAI,YAAY,CAAC,IAAI,EAAE;wBAC1B,QAAQ,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC;qBACrC;oBAED,iBAAiB;oBACjB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,WAAW,EAAE;wBACrB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;qBACzC;yBAAM;wBACH,gCAAgC;wBAChC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;qBAClC;oBAED,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,eAAe,EAAE;wBACzB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC;qBAC/C;yBAAM;wBACH,uCAAuC;wBACvC,sFAAsF;wBACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mHAAmH,CAAC,CAAC;wBAE9I,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAElC,sBAAsB;wBACtB,IAAI,OAAO,IAAI,IAAA,iCAAiB,EAAC,OAAO,CAAC,EAAE;4BACvC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAC;4BACtD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;4BACvC,MAAM,OAAO,GAAW,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAU,EAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAC7E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,OAAO,GAAG,CAAC,CAAC;4BACtC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;yBACpD;qBACJ;gBACL,CAAC,CAAA;gBACD,0BAA0B;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;YAEpB,OAAO;gBACH,IAAI;gBACJ,oBAAoB;gBACpB,MAAM;gBACN,WAAW;gBACX,QAAQ;aACX,CAAC;QACN,CAAC;KAAA;CACJ;AA7ZD,8CA6ZC"}
|
package/lib/index.d.ts
CHANGED
package/lib/index.js
CHANGED
|
@@ -16,4 +16,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/*! Copyright (c) 2023, XAPP AI */
|
|
18
18
|
__exportStar(require("./ArachneWebService"), exports);
|
|
19
|
+
__exportStar(require("./models"), exports);
|
|
19
20
|
//# sourceMappingURL=index.js.map
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,sDAAoC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,sDAAoC;AACpC,2CAAyB"}
|
package/lib/models.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
3
|
+
export interface WebsiteSchema {
|
|
4
|
+
localBusiness?: LocalBusiness;
|
|
5
|
+
homeConstructionBusiness?: HomeAndConstructionBusiness;
|
|
6
|
+
}
|
|
7
|
+
export interface WebsiteDescriptiveData {
|
|
8
|
+
/**
|
|
9
|
+
* A square image that most likely is the logo.
|
|
10
|
+
*/
|
|
11
|
+
logo?: string;
|
|
12
|
+
/**
|
|
13
|
+
* Calls to action detected on the website such as "Free Quote" or "Schedule Consultation"
|
|
14
|
+
*/
|
|
15
|
+
callsToAction?: {
|
|
16
|
+
callToAction: string;
|
|
17
|
+
url?: string;
|
|
18
|
+
}[];
|
|
19
|
+
/**
|
|
20
|
+
* A description of the organization found on the website
|
|
21
|
+
*/
|
|
22
|
+
description?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Phone numbers
|
|
25
|
+
*/
|
|
26
|
+
phoneNumbers?: {
|
|
27
|
+
number: string;
|
|
28
|
+
purpose?: string;
|
|
29
|
+
}[];
|
|
30
|
+
/**
|
|
31
|
+
* Primary (most popular) color detected on the website
|
|
32
|
+
*/
|
|
33
|
+
primaryColor?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Secondary color that complements the primary color on the website.
|
|
36
|
+
*/
|
|
37
|
+
secondaryColor?: string;
|
|
38
|
+
/**
|
|
39
|
+
* Vibrant color, excellent for highlight
|
|
40
|
+
*/
|
|
41
|
+
vibrantColor?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Dark vibrant color
|
|
44
|
+
*/
|
|
45
|
+
darkVibrantColor?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface ScreenshotLocation {
|
|
48
|
+
/**
|
|
49
|
+
* The local file path that the screenshot was saved to.
|
|
50
|
+
*/
|
|
51
|
+
filePath: string;
|
|
52
|
+
}
|
|
53
|
+
export interface BusinessHourPeriod {
|
|
54
|
+
open: {
|
|
55
|
+
day: "SUNDAY" | "MONDAY" | "TUESDAY" | "WEDNESDAY" | "THURSDAY" | "FRIDAY" | "SATURDAY";
|
|
56
|
+
time: string;
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
type HOME_CONSTRUCTION_SERVICES = "plumbing" | "hvac" | "roofing" | "remodelling" | "";
|
|
60
|
+
type HOME_SERVICES = "moving" | "cleaning";
|
|
61
|
+
type PROFESSIONAL_SERVICES = "legal";
|
|
62
|
+
export interface BusinessAddress {
|
|
63
|
+
formated?: string;
|
|
64
|
+
streetAddress?: string;
|
|
65
|
+
postalCode?: string;
|
|
66
|
+
/**
|
|
67
|
+
* City, Town or Locality. This is typically from the "city" of "city", "state"
|
|
68
|
+
*/
|
|
69
|
+
locality?: string;
|
|
70
|
+
region?: string;
|
|
71
|
+
country?: string;
|
|
72
|
+
}
|
|
73
|
+
export interface BusinessAreasServed {
|
|
74
|
+
locality?: string;
|
|
75
|
+
}
|
|
76
|
+
export interface BusinessData {
|
|
77
|
+
/**
|
|
78
|
+
* Name of the business
|
|
79
|
+
*/
|
|
80
|
+
name?: string;
|
|
81
|
+
services?: (HOME_SERVICES | HOME_CONSTRUCTION_SERVICES | PROFESSIONAL_SERVICES)[];
|
|
82
|
+
/**
|
|
83
|
+
* How the business describes themselves
|
|
84
|
+
*/
|
|
85
|
+
typeRaw?: string;
|
|
86
|
+
/**
|
|
87
|
+
* Phone numbers
|
|
88
|
+
*/
|
|
89
|
+
phoneNumbers?: {
|
|
90
|
+
number: string;
|
|
91
|
+
purpose?: string;
|
|
92
|
+
}[];
|
|
93
|
+
/**
|
|
94
|
+
* Primary business address
|
|
95
|
+
*/
|
|
96
|
+
address?: BusinessAddress;
|
|
97
|
+
serviceArea?: BusinessAreasServed;
|
|
98
|
+
geo?: {
|
|
99
|
+
lat: string;
|
|
100
|
+
lon: string;
|
|
101
|
+
};
|
|
102
|
+
hours?: {
|
|
103
|
+
periods: [];
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
export {};
|
package/lib/models.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
3
|
+
import { BusinessData } from "../models";
|
|
4
|
+
export declare function getBusinessData(schema: LocalBusiness | HomeAndConstructionBusiness | undefined): Partial<BusinessData>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getBusinessData = void 0;
|
|
4
|
+
function getBusinessData(schema) {
|
|
5
|
+
if (!schema) {
|
|
6
|
+
return {};
|
|
7
|
+
}
|
|
8
|
+
const partial = {};
|
|
9
|
+
// try to get it from JSON?
|
|
10
|
+
if (schema && typeof schema === "object" && typeof (schema === null || schema === void 0 ? void 0 : schema.name) === "string") {
|
|
11
|
+
partial.name = schema === null || schema === void 0 ? void 0 : schema.name;
|
|
12
|
+
}
|
|
13
|
+
return partial;
|
|
14
|
+
}
|
|
15
|
+
exports.getBusinessData = getBusinessData;
|
|
16
|
+
//# sourceMappingURL=getBusinessData.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getBusinessData.js","sourceRoot":"","sources":["../../src/utils/getBusinessData.ts"],"names":[],"mappings":";;;AAIA,SAAgB,eAAe,CAAC,MAA+D;IAE3F,IAAI,CAAC,MAAM,EAAE;QACT,OAAO,EAAE,CAAC;KACb;IAED,MAAM,OAAO,GAA0B,EAAE,CAAC;IAE1C,2BAA2B;IAC3B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CAAA,KAAK,QAAQ,EAAE;QAC1E,OAAO,CAAC,IAAI,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CAAC;KAC/B;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAdD,0CAcC"}
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getColors.js","sourceRoot":"","sources":["../../src/utils/getColors.ts"],"names":[],"mappings":";AACA,SAAS;AAER,kCAAkC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface Prompt {
|
|
2
|
+
prompt: string;
|
|
3
|
+
tokens: number;
|
|
4
|
+
maxTokens: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function getTokens(prompt: string, response?: number): {
|
|
7
|
+
tokens: number;
|
|
8
|
+
maxTokens: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function getWebMarkdownPrompt(markdown: string, url: string): Prompt;
|
|
11
|
+
export declare function getJSONSchemaPrompt(json: any, url: string): Prompt;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getJSONSchemaPrompt = exports.getWebMarkdownPrompt = exports.getTokens = void 0;
|
|
4
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
5
|
+
const { encode } = require('gpt-3-encoder');
|
|
6
|
+
function getTokens(prompt, response = 500) {
|
|
7
|
+
const tokens = encode(prompt).length;
|
|
8
|
+
const maxTokens = 4000 - response - tokens;
|
|
9
|
+
return { tokens, maxTokens };
|
|
10
|
+
}
|
|
11
|
+
exports.getTokens = getTokens;
|
|
12
|
+
function getWebMarkdownPrompt(markdown, url) {
|
|
13
|
+
const prompt = `Your task is to analyze the contents of a website (provided in markdown below) found at ${url} and provide information about the organization from what is on the website. You will only provide information available through the website content about the services they provide, contact information and a description of the business.\n\n===========\n${markdown}\n===========\nPlease print out in JSON format information about the website above with the following keys:\n\n\"organization_name\" - The name of the organization for the website.\n\"phone_numbers\" - An array of phone numbers for the organization. If available, include what they are used for such as \"customer support\" or \"sales\", if unknown then leave it blank. Use an object with keys: number and purpose.\n\"addresses\" - An array of addresses for the organization.\n\"services\" - An array of the top five unique most common services the organization provides, they should be concinse and no more than a few words such as \"Roofing\", \"Conversational AI Services\", \"Bathroom Remodelling\"\n\"calls_to_action\" - An array of the top five unique calls to action found on the site and their fully formed URL if they have one. Good examples of calls to actions are \"Request Demo\", \"Get Started\", \"Free Quote\", \"Schedule Consultation\", \"Sign Up\". Bad examples are \"Learn More\" or \"Follow us on Facebook\". Use an object with keys: url & callToAction in the array. For the url please combine it properly with the website's URL.\n\"description\" - A description of what the organization does, include the name of the organization, their services, their location, phone number, email. Keep it concise and to a few sentences and around 480 characters.\n \n{`;
|
|
14
|
+
const tokens = getTokens(prompt);
|
|
15
|
+
return Object.assign({ prompt }, tokens);
|
|
16
|
+
}
|
|
17
|
+
exports.getWebMarkdownPrompt = getWebMarkdownPrompt;
|
|
18
|
+
function getJSONSchemaPrompt(json, url) {
|
|
19
|
+
const jsonString = JSON.stringify(json);
|
|
20
|
+
const prompt = `${jsonString}\n====\nAbove is the JSON schema for a website found at ${url}. Please print out in JSON format information about the website above with the following keys:\n\n\"organization_name\" - The name of the organization for the website.\n\"description\" - A description of what the organization does, include the name of the organization, their services, their location, phone number, email and hours if available. Keep it concise and to a few sentences and around 480 characters.\n \n{`;
|
|
21
|
+
const tokens = getTokens(prompt);
|
|
22
|
+
return Object.assign({ prompt }, tokens);
|
|
23
|
+
}
|
|
24
|
+
exports.getJSONSchemaPrompt = getJSONSchemaPrompt;
|
|
25
|
+
//# sourceMappingURL=getPrompt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getPrompt.js","sourceRoot":"","sources":["../../src/utils/getPrompt.ts"],"names":[],"mappings":";;;AAAA,kCAAkC;AAClC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAS5C,SAAgB,SAAS,CAAC,MAAc,EAAE,WAAmB,GAAG;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrC,MAAM,SAAS,GAAW,IAAI,GAAG,QAAQ,GAAG,MAAM,CAAC;IAEnD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AALD,8BAKC;AAGD,SAAgB,oBAAoB,CAAC,QAAgB,EAAE,GAAW;IAE9D,MAAM,MAAM,GAAG,2FAA2F,GAAG,iQAAiQ,QAAQ,w2CAAw2C,CAAC;IAC/tD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACjC,uBACI,MAAM,IACH,MAAM,EACZ;AACL,CAAC;AARD,oDAQC;AAED,SAAgB,mBAAmB,CAAC,IAAS,EAAE,GAAW;IAEtD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,GAAG,UAAU,2DAA2D,GAAG,saAAsa,CAAC;IACjgB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,IACH,MAAM,EACZ;AACL,CAAC;AAVD,kDAUC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import "isomorphic-fetch";
|
|
3
|
+
/**
|
|
4
|
+
* Crawls a webpages sitemap and returns a list of all the URLs found within.
|
|
5
|
+
*
|
|
6
|
+
* It will additionally crawl and sitemaps listed within the original.
|
|
7
|
+
*
|
|
8
|
+
* @note Written with the help of OpenAI
|
|
9
|
+
*
|
|
10
|
+
* @param sitemapUrl
|
|
11
|
+
* @returns
|
|
12
|
+
*/
|
|
13
|
+
export declare function getSitemap(sitemapUrl: string): Promise<string[]>;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
28
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
29
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
30
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
31
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
+
exports.getSitemap = void 0;
|
|
36
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
37
|
+
require("isomorphic-fetch"); // must be at top, making available to all
|
|
38
|
+
const cheerio = __importStar(require("cheerio"));
|
|
39
|
+
const fetchSitemap = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
+
try {
|
|
41
|
+
const response = yield fetch(url);
|
|
42
|
+
return response.text();
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
console.error(`Failed to fetch sitemap: ${error}`);
|
|
46
|
+
return '';
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
/**
|
|
50
|
+
* Crawls a webpages sitemap and returns a list of all the URLs found within.
|
|
51
|
+
*
|
|
52
|
+
* It will additionally crawl and sitemaps listed within the original.
|
|
53
|
+
*
|
|
54
|
+
* @note Written with the help of OpenAI
|
|
55
|
+
*
|
|
56
|
+
* @param sitemapUrl
|
|
57
|
+
* @returns
|
|
58
|
+
*/
|
|
59
|
+
function getSitemap(sitemapUrl) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
const xmlData = yield fetchSitemap(sitemapUrl);
|
|
62
|
+
if (!xmlData) {
|
|
63
|
+
console.error('No sitemap data retrieved.');
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const $ = cheerio.load(xmlData, {
|
|
67
|
+
xmlMode: true
|
|
68
|
+
});
|
|
69
|
+
let allUrls = [];
|
|
70
|
+
// Extract all urls
|
|
71
|
+
$('url').each((i, url) => {
|
|
72
|
+
const loc = $(url).find('loc').text();
|
|
73
|
+
allUrls.push(loc);
|
|
74
|
+
});
|
|
75
|
+
// Extract sitemaps and crawl recursively
|
|
76
|
+
const sitemapNodes = $('sitemap').toArray();
|
|
77
|
+
for (const sitemapNode of sitemapNodes) {
|
|
78
|
+
const loc = $(sitemapNode).find('loc').text();
|
|
79
|
+
const urls = yield getSitemap(loc);
|
|
80
|
+
allUrls = allUrls.concat(urls);
|
|
81
|
+
}
|
|
82
|
+
return allUrls;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
exports.getSitemap = getSitemap;
|
|
86
|
+
//# sourceMappingURL=getSiteMap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getSiteMap.js","sourceRoot":"","sources":["../../src/utils/getSiteMap.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AAErE,iDAAmC;AAEnC,MAAM,YAAY,GAAG,CAAO,GAAW,EAAmB,EAAE;IACxD,IAAI;QACA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;KAC1B;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,CAAC;KACb;AACL,CAAC,CAAA,CAAA;AAED;;;;;;;;;GASG;AACH,SAAsB,UAAU,CAAC,UAAkB;;QAC/C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;SACb;QAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YAC5B,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,OAAO,GAAa,EAAE,CAAC;QAE3B,mBAAmB;QACnB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5C,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;YACnC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SAClC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CAAA;AA7BD,gCA6BC"}
|
package/lib/utils/image.d.ts
CHANGED
|
@@ -19,4 +19,5 @@ export declare function imageIsSquare(url: string, minimumSize?: number): Promis
|
|
|
19
19
|
* @returns
|
|
20
20
|
*/
|
|
21
21
|
export declare function complementHexColor(hex: string | undefined): string | undefined;
|
|
22
|
+
export declare function hexToRGB(hex: string | undefined): string | undefined;
|
|
22
23
|
export declare function generateImage(height: number, width: number, hex: string): Promise<Buffer>;
|
package/lib/utils/image.js
CHANGED
|
@@ -13,7 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
13
13
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
14
14
|
};
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.generateImage = exports.complementHexColor = exports.imageIsSquare = void 0;
|
|
16
|
+
exports.generateImage = exports.hexToRGB = exports.complementHexColor = exports.imageIsSquare = void 0;
|
|
17
17
|
const sharp_1 = __importDefault(require("sharp"));
|
|
18
18
|
/**
|
|
19
19
|
* Determines if the image at the URL provided is square.
|
|
@@ -72,6 +72,16 @@ function complementHexColor(hex) {
|
|
|
72
72
|
return compColor;
|
|
73
73
|
}
|
|
74
74
|
exports.complementHexColor = complementHexColor;
|
|
75
|
+
function hexToRGB(hex) {
|
|
76
|
+
if (!hex) {
|
|
77
|
+
return hex;
|
|
78
|
+
}
|
|
79
|
+
const r = parseInt(hex.slice(1, 3), 16);
|
|
80
|
+
const g = parseInt(hex.slice(3, 5), 16);
|
|
81
|
+
const b = parseInt(hex.slice(5, 7), 16);
|
|
82
|
+
return `${r};${g};${b}`;
|
|
83
|
+
}
|
|
84
|
+
exports.hexToRGB = hexToRGB;
|
|
75
85
|
// not tested yet
|
|
76
86
|
function generateImage(height, width, hex) {
|
|
77
87
|
return __awaiter(this, void 0, void 0, function* () {
|
package/lib/utils/image.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;AAElC,kDAA0B;AAE1B;;;;;;;;GAQG;AACH,SAAsB,aAAa,CAAC,GAAW,EAAE,WAAoB;;QAEjE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAG,MAAM,IAAA,eAAK,EAAC,SAAS,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,KAAK,MAAM,CAAC;QAElC,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC9D,OAAO,QAAQ,IAAI,KAAK,IAAI,WAAW,CAAC;SAC3C;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CAAA;AAnBD,sCAmBC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAAC,GAAuB;IACtD,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,SAAS,CAAC;KACpB;IACD,kDAAkD;IAClD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;KACnD;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;QAClB,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KACnE;IAED,oCAAoC;IACpC,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE;QAC3B,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5E,SAAS,IAAI,aAAa,CAAC;KAC9B;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAxBD,gDAwBC;AAGD,iBAAiB;AACjB,SAAsB,aAAa,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW;;QAC1E,OAAO,IAAA,eAAK,EAAC;YACT,MAAM,EAAE;gBACJ,KAAK;gBACL,MAAM;gBACN,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,GAAG;aAClB;SACJ,CAAC;aACG,GAAG,EAAE;aACL,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAXD,sCAWC"}
|
|
1
|
+
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;AAElC,kDAA0B;AAE1B;;;;;;;;GAQG;AACH,SAAsB,aAAa,CAAC,GAAW,EAAE,WAAoB;;QAEjE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAG,MAAM,IAAA,eAAK,EAAC,SAAS,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,KAAK,MAAM,CAAC;QAElC,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC9D,OAAO,QAAQ,IAAI,KAAK,IAAI,WAAW,CAAC;SAC3C;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CAAA;AAnBD,sCAmBC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAAC,GAAuB;IACtD,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,SAAS,CAAC;KACpB;IACD,kDAAkD;IAClD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;KACnD;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;QAClB,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KACnE;IAED,oCAAoC;IACpC,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE;QAC3B,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5E,SAAS,IAAI,aAAa,CAAC;KAC9B;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAxBD,gDAwBC;AAED,SAAgB,QAAQ,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,GAAG,CAAC;KACd;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5B,CAAC;AARD,4BAQC;AAGD,iBAAiB;AACjB,SAAsB,aAAa,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW;;QAC1E,OAAO,IAAA,eAAK,EAAC;YACT,MAAM,EAAE;gBACJ,KAAK;gBACL,MAAM;gBACN,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,GAAG;aAClB;SACJ,CAAC;aACG,GAAG,EAAE;aACL,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAXD,sCAWC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xapp/arachne-web-service",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"types": "lib/index",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"files": [
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"node": "^14 || ^16 || ^18"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@microsoft/api-extractor": "7.
|
|
17
|
+
"@microsoft/api-extractor": "7.35.0",
|
|
18
18
|
"@types/chai": "4.3.4",
|
|
19
19
|
"@types/mocha": "10.0.1",
|
|
20
20
|
"@types/node": "18.16.7",
|
|
@@ -24,23 +24,23 @@
|
|
|
24
24
|
"chai": "4.3.7",
|
|
25
25
|
"dotenv": "16.0.3",
|
|
26
26
|
"mocha": "10.2.0",
|
|
27
|
-
"sinon": "15.0
|
|
27
|
+
"sinon": "15.1.0",
|
|
28
28
|
"sinon-chai": "3.7.0",
|
|
29
29
|
"ts-node": "10.9.1",
|
|
30
|
-
"tslib": "2.5.
|
|
31
|
-
"typescript": "
|
|
30
|
+
"tslib": "2.5.2",
|
|
31
|
+
"typescript": "5.0.4"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@xapp/arachne": "0.6.
|
|
35
|
-
"@xapp/arachne-parsers": "0.6.
|
|
34
|
+
"@xapp/arachne": "0.6.1",
|
|
35
|
+
"@xapp/arachne-parsers": "0.6.1",
|
|
36
36
|
"gpt-3-encoder": "1.1.4",
|
|
37
37
|
"isomorphic-fetch": "3.0.0",
|
|
38
38
|
"node-vibrant": "3.2.1-alpha.1",
|
|
39
39
|
"schema-dts": "1.1.2",
|
|
40
40
|
"sharp": "0.32.1",
|
|
41
|
-
"stentor-logger": "1.57.
|
|
42
|
-
"stentor-models": "1.57.
|
|
43
|
-
"stentor-utils": "1.57.
|
|
41
|
+
"stentor-logger": "1.57.143",
|
|
42
|
+
"stentor-models": "1.57.143",
|
|
43
|
+
"stentor-utils": "1.57.143"
|
|
44
44
|
},
|
|
45
45
|
"scripts": {
|
|
46
46
|
"api": "api-extractor run --local",
|
|
@@ -48,5 +48,5 @@
|
|
|
48
48
|
"clean": "rm -rf ./lib/*",
|
|
49
49
|
"test:f": "mocha --recursive -r ts-node/register \"./src/**/*.ftest.ts\""
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "f46a5f6a7f4c64b97555f393cfd26df11b85e0f7"
|
|
52
52
|
}
|