@xapp/arachne-web-service 0.5.7 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ArachneWebService.d.ts +20 -30
- package/lib/ArachneWebService.js +167 -26
- package/lib/ArachneWebService.js.map +1 -1
- package/lib/index.d.ts +1 -0
- package/lib/index.js +1 -0
- package/lib/index.js.map +1 -1
- package/lib/models.d.ts +106 -0
- package/lib/models.js +3 -0
- package/lib/models.js.map +1 -0
- package/lib/utils/getBusinessData.d.ts +4 -0
- package/lib/utils/getBusinessData.js +16 -0
- package/lib/utils/getBusinessData.js.map +1 -0
- package/lib/utils/getColors.d.ts +0 -0
- package/lib/utils/getColors.js +4 -0
- package/lib/utils/getColors.js.map +1 -0
- package/lib/utils/getPrompt.d.ts +11 -0
- package/lib/utils/getPrompt.js +25 -0
- package/lib/utils/getPrompt.js.map +1 -0
- package/lib/utils/getSiteMap.d.ts +13 -0
- package/lib/utils/getSiteMap.js +86 -0
- package/lib/utils/getSiteMap.js.map +1 -0
- package/lib/utils/image.d.ts +1 -0
- package/lib/utils/image.js +11 -1
- package/lib/utils/image.js.map +1 -1
- package/package.json +11 -11
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/*! Copyright (c) 2023, XAPP AI */
|
|
2
2
|
import "isomorphic-fetch";
|
|
3
3
|
import { DetectorResult } from "@xapp/arachne-parsers";
|
|
4
|
-
import {
|
|
4
|
+
import { WebsiteSchema, WebsiteDescriptiveData, ScreenshotLocation, BusinessData } from "./models";
|
|
5
5
|
export interface ArachneAppGetInfoOptions {
|
|
6
6
|
/**
|
|
7
7
|
* Provide an openAI key to use OpenAI to determine the call to action of the website.
|
|
@@ -15,10 +15,17 @@ export interface ArachneAppGetInfoOptions {
|
|
|
15
15
|
* If not provided, you will not get primaryColor, secondaryColor, or images.
|
|
16
16
|
*/
|
|
17
17
|
screenshotsPath?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Provide a path for saving the markdown of the website
|
|
20
|
+
*/
|
|
21
|
+
markdownPath?: string;
|
|
18
22
|
/**
|
|
19
23
|
* Run headless, true by default.
|
|
20
24
|
*/
|
|
21
25
|
headless?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Override the viewport size for the browser window.
|
|
28
|
+
*/
|
|
22
29
|
viewportSize?: {
|
|
23
30
|
width: number;
|
|
24
31
|
height: number;
|
|
@@ -32,49 +39,32 @@ export interface ArachneAppGetInfoOptions {
|
|
|
32
39
|
*/
|
|
33
40
|
pageHandlerTimeout?: number;
|
|
34
41
|
}
|
|
35
|
-
export interface
|
|
36
|
-
localBusiness?: LocalBusiness;
|
|
37
|
-
}
|
|
38
|
-
export interface WebsiteDescriptiveData {
|
|
39
|
-
/**
|
|
40
|
-
* A square image that most likely is the logo.
|
|
41
|
-
*/
|
|
42
|
-
logo?: string;
|
|
42
|
+
export interface WebsiteData {
|
|
43
43
|
/**
|
|
44
|
-
*
|
|
44
|
+
* Descriptive metadata for the website.
|
|
45
45
|
*/
|
|
46
|
-
|
|
46
|
+
site: WebsiteDescriptiveData;
|
|
47
47
|
/**
|
|
48
|
-
*
|
|
48
|
+
* A list of detected technologies
|
|
49
49
|
*/
|
|
50
|
-
|
|
50
|
+
detectedTechnologies: DetectorResult[];
|
|
51
51
|
/**
|
|
52
|
-
*
|
|
52
|
+
* Information related to the organization
|
|
53
53
|
*/
|
|
54
|
-
|
|
55
|
-
callToAction: string;
|
|
56
|
-
url?: string;
|
|
57
|
-
}[];
|
|
54
|
+
business: BusinessData;
|
|
58
55
|
/**
|
|
59
|
-
*
|
|
56
|
+
* JSON+LD schema detected on the website
|
|
60
57
|
*/
|
|
61
|
-
|
|
58
|
+
schema: WebsiteSchema;
|
|
62
59
|
/**
|
|
63
|
-
*
|
|
60
|
+
* The screenshots that were taken from the website.
|
|
64
61
|
*/
|
|
65
|
-
|
|
66
|
-
number: string;
|
|
67
|
-
purpose?: string;
|
|
68
|
-
}[];
|
|
69
|
-
}
|
|
70
|
-
export interface WebsiteData {
|
|
71
|
-
site: WebsiteDescriptiveData;
|
|
72
|
-
detectedTechnologies: DetectorResult[];
|
|
73
|
-
schema: WebsiteSchema;
|
|
62
|
+
screenshots: ScreenshotLocation[];
|
|
74
63
|
}
|
|
75
64
|
export declare class ArachneWebService {
|
|
76
65
|
/**
|
|
77
66
|
* Get data on the provided website.
|
|
67
|
+
*
|
|
78
68
|
* @param url
|
|
79
69
|
* @param options
|
|
80
70
|
* @returns
|
package/lib/ArachneWebService.js
CHANGED
|
@@ -1,4 +1,27 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
2
25
|
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
26
|
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
27
|
return new (P || (P = Promise))(function (resolve, reject) {
|
|
@@ -16,19 +39,24 @@ exports.ArachneWebService = void 0;
|
|
|
16
39
|
/*! Copyright (c) 2023, XAPP AI */
|
|
17
40
|
require("isomorphic-fetch"); // must be at top, making available to all
|
|
18
41
|
const path_1 = require("path");
|
|
42
|
+
const fs = __importStar(require("fs"));
|
|
43
|
+
const cheerio = __importStar(require("cheerio"));
|
|
19
44
|
const arachne_1 = require("@xapp/arachne");
|
|
20
45
|
const arachne_parsers_1 = require("@xapp/arachne-parsers");
|
|
21
46
|
const stentor_logger_1 = require("stentor-logger");
|
|
22
47
|
const stentor_utils_1 = require("stentor-utils");
|
|
23
48
|
const node_vibrant_1 = __importDefault(require("node-vibrant"));
|
|
24
|
-
const { encode } = require('gpt-3-encoder');
|
|
25
49
|
const autoScroll_1 = require("./utils/autoScroll");
|
|
26
50
|
const findBestFaviconURL_1 = require("./utils/findBestFaviconURL");
|
|
27
51
|
const image_1 = require("./utils/image");
|
|
28
52
|
const isLambda_1 = require("./utils/isLambda");
|
|
53
|
+
const getSiteMap_1 = require("./utils/getSiteMap");
|
|
54
|
+
const getPrompt_1 = require("./utils/getPrompt");
|
|
55
|
+
const getBusinessData_1 = require("./utils/getBusinessData");
|
|
29
56
|
class ArachneWebService {
|
|
30
57
|
/**
|
|
31
58
|
* Get data on the provided website.
|
|
59
|
+
*
|
|
32
60
|
* @param url
|
|
33
61
|
* @param options
|
|
34
62
|
* @returns
|
|
@@ -36,8 +64,33 @@ class ArachneWebService {
|
|
|
36
64
|
getWebsiteData(url, options = {}) {
|
|
37
65
|
return __awaiter(this, void 0, void 0, function* () {
|
|
38
66
|
const site = {};
|
|
39
|
-
|
|
67
|
+
const detectedTechnologies = [];
|
|
40
68
|
const schema = {};
|
|
69
|
+
const screenshots = [];
|
|
70
|
+
const business = {};
|
|
71
|
+
// first get the sitemap
|
|
72
|
+
const siteUrl = new URL(url);
|
|
73
|
+
const originalPath = siteUrl.pathname;
|
|
74
|
+
const sitemapUrl = new URL(url);
|
|
75
|
+
sitemapUrl.pathname = `${originalPath}sitemap.xml`;
|
|
76
|
+
(0, stentor_logger_1.log)().debug(`Loading sitemap at ${sitemapUrl.toString()}`);
|
|
77
|
+
const sitemap = yield (0, getSiteMap_1.getSitemap)(sitemapUrl.toString());
|
|
78
|
+
// we could have been passed a URL that already has a pathname, we filter by those first
|
|
79
|
+
const filtered = sitemap.filter((value) => {
|
|
80
|
+
const siteUrl = new URL(value);
|
|
81
|
+
const sitePath = siteUrl.pathname;
|
|
82
|
+
return sitePath.startsWith(originalPath);
|
|
83
|
+
});
|
|
84
|
+
(0, stentor_logger_1.log)().debug(`Sitemap has ${sitemap.length} pages, filtered to ${filtered.length}`);
|
|
85
|
+
// find about us page
|
|
86
|
+
const aboutUs = filtered.find((value) => {
|
|
87
|
+
const url = new URL(value);
|
|
88
|
+
const path = url.pathname;
|
|
89
|
+
// provided by OpenAI
|
|
90
|
+
const aboutUsRegex = /\/(about(-us)?|company\/about|who-we-are|our-story|info\/about|en\/about|about\/our-company)(\/)?$/i;
|
|
91
|
+
return aboutUsRegex.test(path);
|
|
92
|
+
});
|
|
93
|
+
(0, stentor_logger_1.log)().debug(`Found an about us page at ${aboutUs}`);
|
|
41
94
|
const queue = new arachne_1.MemoryRequestQueue();
|
|
42
95
|
queue.addRequest({ url });
|
|
43
96
|
const launchOptions = {
|
|
@@ -56,7 +109,7 @@ class ArachneWebService {
|
|
|
56
109
|
}
|
|
57
110
|
let browser;
|
|
58
111
|
try {
|
|
59
|
-
(0, stentor_logger_1.log)().debug('Looking for chrome-aws-lambda');
|
|
112
|
+
(0, stentor_logger_1.log)().debug('Looking for chrome-aws-lambda...');
|
|
60
113
|
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
|
61
114
|
const chromium = require('@sparticuz/chrome-aws-lambda');
|
|
62
115
|
browser = yield chromium.puppeteer.launch({
|
|
@@ -79,6 +132,8 @@ class ArachneWebService {
|
|
|
79
132
|
pageHandlerTimeout,
|
|
80
133
|
pageHandler: (page, request) => __awaiter(this, void 0, void 0, function* () {
|
|
81
134
|
const title = yield page.title();
|
|
135
|
+
const u = new URL(url);
|
|
136
|
+
const fileName = `${u.hostname}${u.pathname}`.replace(/[./]/g, "_");
|
|
82
137
|
(0, stentor_logger_1.log)().info(`Loaded ${request.url} : ${title}`);
|
|
83
138
|
if (pageWait) {
|
|
84
139
|
(0, stentor_logger_1.log)().info(`Waiting for ${pageWaitTime} ms for page to completely load.`);
|
|
@@ -90,19 +145,31 @@ class ArachneWebService {
|
|
|
90
145
|
// page.content() gets you the HTML after it has been modified by the JavaScript
|
|
91
146
|
// on the page while response.text() only gives you the HTML from the original GET
|
|
92
147
|
const html = yield page.content();
|
|
148
|
+
const $ = cheerio.load(html);
|
|
149
|
+
const description = $('meta[name="description"]').attr('content');
|
|
93
150
|
const detectors = [
|
|
94
151
|
new arachne_parsers_1.XAPPDetector(html),
|
|
95
152
|
new arachne_parsers_1.SurefireDetector(html)
|
|
96
153
|
];
|
|
97
|
-
detectedTechnologies
|
|
154
|
+
detectedTechnologies.push(...new arachne_parsers_1.MultiDetector(html, detectors).detect());
|
|
98
155
|
const things = new arachne_parsers_1.ArachneJSONLD(html).getThings();
|
|
99
156
|
let localBusiness;
|
|
157
|
+
let homeConstructionBusiness;
|
|
158
|
+
(0, stentor_logger_1.log)().info('Looking through JSON Schema...');
|
|
100
159
|
for (const thing of things) {
|
|
101
160
|
if ((0, arachne_parsers_1.isLocalBusiness)(thing)) {
|
|
102
161
|
(0, stentor_logger_1.log)().debug(`Found local business information`);
|
|
103
162
|
localBusiness = thing;
|
|
104
163
|
schema.localBusiness = thing;
|
|
105
164
|
}
|
|
165
|
+
else if ((0, arachne_parsers_1.isHomeConstructionBusiness)(thing)) {
|
|
166
|
+
(0, stentor_logger_1.log)().debug(`Found home construction business`);
|
|
167
|
+
homeConstructionBusiness = thing;
|
|
168
|
+
schema.homeConstructionBusiness = thing;
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
(0, stentor_logger_1.log)().debug(`Found JSON Schema ${thing["@id"]} ${thing["@type"]}`);
|
|
172
|
+
}
|
|
106
173
|
}
|
|
107
174
|
// We are going to try to figure out the icon!
|
|
108
175
|
if (localBusiness && typeof localBusiness !== "string") {
|
|
@@ -126,9 +193,7 @@ class ArachneWebService {
|
|
|
126
193
|
}
|
|
127
194
|
// Try to figure out primary color, yup
|
|
128
195
|
if (options.screenshotsPath) {
|
|
129
|
-
const
|
|
130
|
-
const imageName = `${u.hostname}${u.pathname}`.replace(/[./]/g, "_");
|
|
131
|
-
const path = (0, path_1.resolve)(options.screenshotsPath, `${imageName}.png`);
|
|
196
|
+
const path = (0, path_1.resolve)(options.screenshotsPath, `${fileName}.png`);
|
|
132
197
|
(0, stentor_logger_1.log)().debug(`Saving screenshot to ${path}`);
|
|
133
198
|
const captureBeyondViewport = options.viewportSize ? false : true;
|
|
134
199
|
const fullPage = options.viewportSize ? false : true;
|
|
@@ -137,7 +202,11 @@ class ArachneWebService {
|
|
|
137
202
|
fullPage,
|
|
138
203
|
captureBeyondViewport
|
|
139
204
|
});
|
|
140
|
-
|
|
205
|
+
screenshots.push({
|
|
206
|
+
filePath: path
|
|
207
|
+
});
|
|
208
|
+
(0, stentor_logger_1.log)().debug(`Screenshot saved.`);
|
|
209
|
+
(0, stentor_logger_1.log)().debug(`Analyzing colors...`);
|
|
141
210
|
const v = new node_vibrant_1.default(path);
|
|
142
211
|
const palette = yield v.getPalette();
|
|
143
212
|
let swatchName = undefined;
|
|
@@ -154,7 +223,18 @@ class ArachneWebService {
|
|
|
154
223
|
swatchName = swatch;
|
|
155
224
|
mostPopular = current;
|
|
156
225
|
}
|
|
157
|
-
|
|
226
|
+
switch (swatch) {
|
|
227
|
+
case "LightVibrant":
|
|
228
|
+
case "Vibrant":
|
|
229
|
+
site.vibrantColor = current === null || current === void 0 ? void 0 : current.hex;
|
|
230
|
+
continue;
|
|
231
|
+
case "DarkVibrant":
|
|
232
|
+
site.darkVibrantColor = current === null || current === void 0 ? void 0 : current.hex;
|
|
233
|
+
continue;
|
|
234
|
+
default:
|
|
235
|
+
}
|
|
236
|
+
const debugPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(current === null || current === void 0 ? void 0 : current.hex) + 'm \x1b[0m';
|
|
237
|
+
debugSwatches = `${debugSwatches} ${swatch} (${current === null || current === void 0 ? void 0 : current.population}):${current === null || current === void 0 ? void 0 : current.hex} ${debugPreview} `;
|
|
158
238
|
}
|
|
159
239
|
}
|
|
160
240
|
if (swatchName) {
|
|
@@ -165,24 +245,64 @@ class ArachneWebService {
|
|
|
165
245
|
}
|
|
166
246
|
site.primaryColor = mostPopular === null || mostPopular === void 0 ? void 0 : mostPopular.hex;
|
|
167
247
|
site.secondaryColor = (0, image_1.complementHexColor)(site.primaryColor);
|
|
168
|
-
(0,
|
|
169
|
-
(0,
|
|
248
|
+
const primaryPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.primaryColor) + 'm \x1b[0m';
|
|
249
|
+
const secondaryPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.secondaryColor) + 'm \x1b[0m';
|
|
250
|
+
const vibrantPreview = '\x1b[48;2;' + (0, image_1.hexToRGB)(site.vibrantColor) + 'm \x1b[0m';
|
|
251
|
+
(0, stentor_logger_1.log)().debug(`Primary color: ${site.primaryColor} ${primaryPreview}`);
|
|
252
|
+
(0, stentor_logger_1.log)().debug(`Secondary color: ${site.secondaryColor} ${secondaryPreview}`);
|
|
253
|
+
(0, stentor_logger_1.log)().debug(`Vibrant color: ${site.vibrantColor} ${vibrantPreview}`);
|
|
254
|
+
(0, stentor_logger_1.log)().debug(`All swatches are: ${debugSwatches}`);
|
|
170
255
|
}
|
|
171
256
|
else {
|
|
172
257
|
(0, stentor_logger_1.log)().info(`screenshotsPath not provided, cannot determine primaryColor and secondaryColor`);
|
|
173
258
|
}
|
|
259
|
+
// Convert to markdown
|
|
260
|
+
const markdown = new arachne_parsers_1.ArachneHTMLtoText({ stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"] }).convert(html, true);
|
|
261
|
+
if (options.markdownPath) {
|
|
262
|
+
const path = (0, path_1.resolve)(options.markdownPath, `${fileName}.md`);
|
|
263
|
+
(0, stentor_logger_1.log)().debug(`Writing markdown to ${path}...`);
|
|
264
|
+
yield fs.promises.writeFile(path, markdown);
|
|
265
|
+
(0, stentor_logger_1.log)().debug(`Written.`);
|
|
266
|
+
}
|
|
267
|
+
let aiData = undefined;
|
|
174
268
|
const { openAIKey = process.env.OPENAI_API_KEY } = options;
|
|
175
269
|
// Use OpenAI to determine Call To Action
|
|
176
270
|
if (openAIKey) {
|
|
177
271
|
(0, stentor_logger_1.log)().info(`Calling OpenAI....`);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
272
|
+
const webPrompt = (0, getPrompt_1.getWebMarkdownPrompt)(markdown, url);
|
|
273
|
+
let prompt = webPrompt.prompt;
|
|
274
|
+
let maxTokens = webPrompt.maxTokens;
|
|
275
|
+
let tokens = webPrompt.tokens;
|
|
276
|
+
// first try if over max tokens
|
|
277
|
+
if (maxTokens < 0 && aboutUs) {
|
|
278
|
+
// load about us
|
|
279
|
+
(0, stentor_logger_1.log)().info(`Too many tokens, attempting to load about us page at ${aboutUs}...`);
|
|
280
|
+
yield page.goto(aboutUs);
|
|
281
|
+
(0, stentor_logger_1.log)().info(`Loaded...`);
|
|
282
|
+
const aboutUsHTML = yield page.content();
|
|
283
|
+
const aboutUsMarkdown = new arachne_parsers_1.ArachneHTMLtoText({ stripTags: [...arachne_parsers_1.DEFAULT_STRIP_TAGS, "head", "img", "iframe", "style"] }).convert(aboutUsHTML, true);
|
|
284
|
+
const aboutUsPrompt = (0, getPrompt_1.getWebMarkdownPrompt)(aboutUsMarkdown, url);
|
|
285
|
+
(0, stentor_logger_1.log)().info(`About Us tokens: ${aboutUsPrompt.tokens} max: ${aboutUsPrompt.maxTokens}`);
|
|
286
|
+
if (aboutUsPrompt.maxTokens > 0) {
|
|
287
|
+
(0, stentor_logger_1.log)().info(`Using about us page.`);
|
|
288
|
+
prompt = aboutUsPrompt.prompt;
|
|
289
|
+
maxTokens = aboutUsPrompt.maxTokens;
|
|
290
|
+
tokens = aboutUsPrompt.tokens;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
if (maxTokens < 0 && localBusiness || homeConstructionBusiness) {
|
|
294
|
+
(0, stentor_logger_1.log)().info(`Too many tokens, attempting to use LocalBusiness schema to generate description.`);
|
|
295
|
+
// try second prompt where we send the JSON.
|
|
296
|
+
const jsonSchemaPrompt = (0, getPrompt_1.getJSONSchemaPrompt)(localBusiness || homeConstructionBusiness, url);
|
|
297
|
+
(0, stentor_logger_1.log)().info(`JSON Schema tokens: ${jsonSchemaPrompt.tokens} max: ${jsonSchemaPrompt.maxTokens}`);
|
|
298
|
+
if (jsonSchemaPrompt.maxTokens > 0) {
|
|
299
|
+
(0, stentor_logger_1.log)().info(`LocalBusiness JSON fits within token limit, using it.`);
|
|
300
|
+
prompt = jsonSchemaPrompt.prompt;
|
|
301
|
+
maxTokens = jsonSchemaPrompt.maxTokens;
|
|
302
|
+
tokens = jsonSchemaPrompt.tokens;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
(0, stentor_logger_1.log)().info(`Final prompt is ${tokens} tokens, setting max_tokens to ${maxTokens}.`);
|
|
186
306
|
const options = {
|
|
187
307
|
method: 'POST',
|
|
188
308
|
headers: {
|
|
@@ -193,7 +313,7 @@ class ArachneWebService {
|
|
|
193
313
|
model: "text-davinci-003",
|
|
194
314
|
prompt,
|
|
195
315
|
temperature: 0.25,
|
|
196
|
-
max_tokens,
|
|
316
|
+
max_tokens: maxTokens,
|
|
197
317
|
top_p: 1,
|
|
198
318
|
frequency_penalty: 0,
|
|
199
319
|
presence_penalty: 0
|
|
@@ -207,7 +327,7 @@ class ArachneWebService {
|
|
|
207
327
|
if (status === 200) {
|
|
208
328
|
return response.json();
|
|
209
329
|
}
|
|
210
|
-
return;
|
|
330
|
+
return response;
|
|
211
331
|
}).finally(() => {
|
|
212
332
|
// eslint-disable-next-line no-console
|
|
213
333
|
console.timeEnd("open_ai_web");
|
|
@@ -217,17 +337,36 @@ class ArachneWebService {
|
|
|
217
337
|
(0, stentor_logger_1.log)().info(`OpenAI token usage: prompt ${usage.prompt_tokens} completion ${usage.completion_tokens} total ${usage.total_tokens}`);
|
|
218
338
|
// Add the leading { as it is included in the query
|
|
219
339
|
const completion = `{\n${openAIResponse.choices[0].text}`;
|
|
220
|
-
|
|
340
|
+
aiData = JSON.parse(completion);
|
|
341
|
+
(0, stentor_logger_1.log)().debug(JSON.stringify(aiData, undefined, 2));
|
|
221
342
|
// eslint-disable-next-line no-console
|
|
222
|
-
site.callsToAction = siteData.calls_to_action;
|
|
223
|
-
site.description = siteData.description;
|
|
224
|
-
site.phoneNumbers = siteData.phone_numbers;
|
|
343
|
+
// site.callsToAction = siteData.calls_to_action;
|
|
344
|
+
// site.description = siteData.description;
|
|
345
|
+
// site.phoneNumbers = siteData.phone_numbers;
|
|
225
346
|
}
|
|
226
347
|
else {
|
|
227
348
|
(0, stentor_logger_1.log)().warn(`Unabled to get website info from OpenAI status:${status}`);
|
|
228
349
|
(0, stentor_logger_1.log)().debug(openAIResponse);
|
|
229
350
|
}
|
|
230
351
|
}
|
|
352
|
+
const businessData = (0, getBusinessData_1.getBusinessData)(schema.localBusiness || schema.homeConstructionBusiness);
|
|
353
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.organization_name) {
|
|
354
|
+
business.name = aiData.organization_name;
|
|
355
|
+
}
|
|
356
|
+
else if (businessData.name) {
|
|
357
|
+
business.name = businessData.name;
|
|
358
|
+
}
|
|
359
|
+
// Set some items
|
|
360
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.description) {
|
|
361
|
+
site.description = aiData.description;
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
// what is our alternative here?
|
|
365
|
+
site.description = description;
|
|
366
|
+
}
|
|
367
|
+
if (aiData === null || aiData === void 0 ? void 0 : aiData.calls_to_action) {
|
|
368
|
+
site.callsToAction = aiData.calls_to_action;
|
|
369
|
+
}
|
|
231
370
|
else {
|
|
232
371
|
// Try to figure out the call to action
|
|
233
372
|
// TODO: Can we use our model export and a simple NLU to better find possible matches?
|
|
@@ -249,7 +388,9 @@ class ArachneWebService {
|
|
|
249
388
|
return {
|
|
250
389
|
site,
|
|
251
390
|
detectedTechnologies,
|
|
252
|
-
schema
|
|
391
|
+
schema,
|
|
392
|
+
screenshots,
|
|
393
|
+
business
|
|
253
394
|
};
|
|
254
395
|
});
|
|
255
396
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ArachneWebService.js","sourceRoot":"","sources":["../src/ArachneWebService.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AACrE,+BAA+B;AAE/B,2CAAyF;AACzF,2DAU+B;AAE/B,mDAAqC;AACrC,iDAA8D;AAG9D,gEAAmC;AAGnC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAG5C,mDAAgD;AAChD,mEAAgE;AAChE,yCAAkE;AAClE,+CAA4C;AAwE5C,MAAa,iBAAiB;IAC1B;;;;;OAKG;IACU,cAAc,CAAC,GAAW,EAAE,UAAoC,EAAE;;YAE3E,MAAM,IAAI,GAA2B,EAAE,CAAC;YACxC,IAAI,oBAAoB,GAAqB,EAAE,CAAC;YAChD,MAAM,MAAM,GAAkB,EAAE,CAAC;YAEjC,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;YACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1B,MAAM,aAAa,GAAyE;gBACxF,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,OAAO,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC;YAEF,MAAM,QAAQ,GAAY,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAC9E,MAAM,YAAY,GAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAA,mBAAQ,GAAE,EAAE;gBACb,8BAA8B;gBAC9B,aAAa,CAAC,cAAc,GAAG,wBAAwB,CAAC;aAC3D;YAED,IAAI,OAAO,CAAC,YAAY,EAAE;gBACtB,aAAa,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC;gBACrD,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;aACvG;YAED,IAAI,OAAuD,CAAC;YAE5D,IAAI;gBACA,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;gBAE7C,8DAA8D;gBAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;gBAEzD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;oBACtC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,eAAe,EAAE,QAAQ,CAAC,eAAe;oBACzC,cAAc,EAAE,MAAM,QAAQ,CAAC,cAAc;oBAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,iBAAiB,EAAE,IAAI;iBAC1B,CAAC,CAAC;aACN;YAAC,OAAO,CAAC,EAAE;gBACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;aAChG;YAED,MAAM,kBAAkB,GAAW,OAAO,OAAO,CAAC,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvH,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;gBAC5B,OAAO,EAAE,IAAI;gBACb,aAAa;gBACb,KAAK;gBACL,OAAO;gBACP,kBAAkB;gBAClB,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,EAAE;oBAE9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;oBAE/C,IAAI,QAAQ,EAAE;wBACV,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,eAAe,YAAY,kCAAkC,CAAC,CAAC;wBAC1E,MAAM,IAAA,uBAAU,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;wBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;qBACtC;oBAED,mBAAmB;oBACnB,qEAAqE;oBACrE,gFAAgF;oBAChF,kFAAkF;oBAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAElC,MAAM,SAAS,GAAuB;wBAClC,IAAI,8BAAY,CAAC,IAAI,CAAC;wBACtB,IAAI,kCAAgB,CAAC,IAAI,CAAC;qBAC7B,CAAC;oBAEF,oBAAoB,GAAG,IAAI,+BAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC;oBAEnE,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;oBAEnD,IAAI,aAAwC,CAAC;oBAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;wBACxB,IAAI,IAAA,iCAAe,EAAC,KAAK,CAAC,EAAE;4BACxB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC;yBAChC;qBACJ;oBAED,8CAA8C;oBAC9C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;wBACpD,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC;wBAC7C,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;4BACvC,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;4BAC1D,IAAI,MAAM,EAAE;gCACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,iBAAiB,EAAE,CAAC,CAAC;gCACrE,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;6BACjC;yBACJ;qBACJ;oBAED,0BAA0B;oBAC1B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBACZ,kBAAkB;wBAClB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;wBAE7C,MAAM,OAAO,GAAG,MAAM,IAAA,uCAAkB,EAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,OAAO,aAAa,CAAC,CAAC;wBACrD,8CAA8C;wBAC9C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;qBACvB;oBAED,uCAAuC;oBACvC,IAAI,OAAO,CAAC,eAAe,EAAE;wBACzB,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;wBACvB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;wBACrE,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,eAAe,EAAE,GAAG,SAAS,MAAM,CAAC,CAAC;wBAClE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;wBAE5C,MAAM,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAErD,MAAM,IAAI,CAAC,UAAU,CAAC;4BAClB,IAAI;4BACJ,QAAQ;4BACR,qBAAqB;yBACxB,CAAC,CAAC;wBACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;wBAElD,MAAM,CAAC,GAAG,IAAI,sBAAO,CAAC,IAAI,CAAC,CAAC;wBAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;wBACrC,IAAI,UAAU,GAAuB,SAAS,CAAC;wBAC/C,IAAI,WAAW,CAAC;wBAChB,IAAI,aAAa,GAAG,EAAE,CAAC;wBAEvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;4BAC1B,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE;gCACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gCAChC,IAAI,CAAC,WAAW,EAAE;oCACd,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;qCAAM,IAAI,OAAO,IAAI,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE;oCAC/D,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;gCACD,aAAa,GAAG,GAAG,aAAa,IAAI,MAAM,IAAI,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,EAAE,CAAA;6BAC/D;yBACJ;wBACD,IAAI,UAAU,EAAE;4BACZ,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,UAAU,IAAI,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,EAAE,CAAC,CAAC;yBACxE;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;yBACxD;wBAED,IAAI,CAAC,YAAY,GAAG,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,CAAC;wBACrC,IAAI,CAAC,cAAc,GAAG,IAAA,0BAAkB,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;wBAC9D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,aAAa,EAAE,CAAC,CAAC;qBAC7D;yBAAM;wBACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;qBAChG;oBAED,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC;oBAC3D,yCAAyC;oBACzC,IAAI,SAAS,EAAE;wBAEX,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;wBACjC,sBAAsB;wBACtB,MAAM,QAAQ,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;wBAC1G,uBAAuB;wBACvB,MAAM,MAAM,GAAG,GAAG,QAAQ,8EAA8E,GAAG,+yCAA+yC,CAAC;wBAE35C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;wBACrC,MAAM,iBAAiB,GAAG,GAAG,CAAC;wBAC9B,MAAM,UAAU,GAAW,IAAI,GAAG,iBAAiB,GAAG,MAAM,CAAC;wBAE7D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,aAAa,MAAM,kCAAkC,UAAU,EAAE,CAAC,CAAC;wBAE9E,MAAM,OAAO,GAAgB;4BACzB,MAAM,EAAE,MAAM;4BACd,OAAO,EAAE;gCACL,cAAc,EAAE,kBAAkB;gCAClC,eAAe,EAAE,UAAU,SAAS,EAAE;6BACzC;4BACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACjB,KAAK,EAAE,kBAAkB;gCACzB,MAAM;gCACN,WAAW,EAAE,IAAI;gCACjB,UAAU;gCACV,KAAK,EAAE,CAAC;gCACR,iBAAiB,EAAE,CAAC;gCACpB,gBAAgB,EAAE,CAAC;6BACtB,CAAC;yBACL,CAAA;wBAED,sCAAsC;wBACtC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;wBAC5B,IAAI,MAAM,GAAW,GAAG,CAAC;wBACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;4BACnG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;4BACzB,IAAI,MAAM,KAAK,GAAG,EAAE;gCAChB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;6BAC1B;4BACD,OAAO;wBACX,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;4BACZ,sCAAsC;4BACtC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;wBACnC,CAAC,CAAC,CAAC;wBAEH,IAAI,MAAM,KAAK,GAAG,EAAE;4BAUhB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;4BACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;4BAEjI,mDAAmD;4BACnD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC1D,MAAM,QAAQ,GAAc,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BACnD,sCAAsC;4BAEtC,IAAI,CAAC,aAAa,GAAG,QAAQ,CAAC,eAAe,CAAC;4BAC9C,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC;4BACxC,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,aAAa,CAAC;yBAC9C;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;4BACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;yBAC/B;qBACJ;yBAAM;wBACH,uCAAuC;wBACvC,sFAAsF;wBACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mHAAmH,CAAC,CAAC;wBAE9I,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAElC,sBAAsB;wBACtB,IAAI,OAAO,IAAI,IAAA,iCAAiB,EAAC,OAAO,CAAC,EAAE;4BACvC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAC;4BACtD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;4BACvC,MAAM,OAAO,GAAW,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAU,EAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAC7E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,OAAO,GAAG,CAAC,CAAC;4BACtC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;yBACpD;qBACJ;gBACL,CAAC,CAAA;gBACD,0BAA0B;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;YAEpB,OAAO;gBACH,IAAI;gBACJ,oBAAoB;gBACpB,MAAM;aACT,CAAC;QACN,CAAC;KAAA;CACJ;AA9QD,8CA8QC"}
|
|
1
|
+
{"version":3,"file":"ArachneWebService.js","sourceRoot":"","sources":["../src/ArachneWebService.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AACrE,+BAA+B;AAC/B,uCAAyB;AACzB,iDAAmC;AAEnC,2CAAyF;AACzF,2DAW+B;AAE/B,mDAAqC;AACrC,iDAA8D;AAG9D,gEAAmC;AAInC,mDAAgD;AAChD,mEAAgE;AAChE,yCAA4E;AAC5E,+CAA4C;AAC5C,mDAAgD;AAChD,iDAA8E;AAC9E,6DAA0D;AA+D1D,MAAa,iBAAiB;IAC1B;;;;;;OAMG;IACU,cAAc,CAAC,GAAW,EAAE,UAAoC,EAAE;;YAE3E,MAAM,IAAI,GAA2B,EAAE,CAAC;YACxC,MAAM,oBAAoB,GAAqB,EAAE,CAAC;YAClD,MAAM,MAAM,GAAkB,EAAE,CAAC;YACjC,MAAM,WAAW,GAAyB,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAiB,EAAE,CAAC;YAElC,wBAAwB;YAExB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,CAAC;YAEtC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,UAAU,CAAC,QAAQ,GAAG,GAAG,YAAY,aAAa,CAAC;YACnD,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,sBAAsB,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YAE3D,MAAM,OAAO,GAAG,MAAM,IAAA,uBAAU,EAAC,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;YAExD,wFAAwF;YACxF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;gBACtC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAClC,OAAO,QAAQ,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;YAC7C,CAAC,CAAC,CAAC;YAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,eAAe,OAAO,CAAC,MAAM,uBAAuB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACnF,qBAAqB;YACrB,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;gBACpC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC3B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC;gBAC1B,qBAAqB;gBACrB,MAAM,YAAY,GAAG,qGAAqG,CAAC;gBAE3H,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC,CAAC,CAAC;YAEH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;YAEpD,MAAM,KAAK,GAAG,IAAI,4BAAkB,EAAE,CAAC;YACvC,KAAK,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YAE1B,MAAM,aAAa,GAAyE;gBACxF,OAAO,EAAE,IAAI;gBACb,QAAQ,EAAE,OAAO,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;aAC5E,CAAC;YAEF,MAAM,QAAQ,GAAY,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC;YAC9E,MAAM,YAAY,GAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YAEnD,IAAI,CAAC,IAAA,mBAAQ,GAAE,EAAE;gBACb,8BAA8B;gBAC9B,aAAa,CAAC,cAAc,GAAG,wBAAwB,CAAC;aAC3D;YAED,IAAI,OAAO,CAAC,YAAY,EAAE;gBACtB,aAAa,CAAC,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC;gBACrD,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,OAAO,CAAC,YAAY,CAAC,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC;aACvG;YAED,IAAI,OAAuD,CAAC;YAE5D,IAAI;gBACA,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;gBAEhD,8DAA8D;gBAC9D,MAAM,QAAQ,GAAG,OAAO,CAAC,8BAA8B,CAAC,CAAC;gBAEzD,OAAO,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC;oBACtC,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,eAAe,EAAE,QAAQ,CAAC,eAAe;oBACzC,cAAc,EAAE,MAAM,QAAQ,CAAC,cAAc;oBAC7C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;oBAC3B,iBAAiB,EAAE,IAAI;iBAC1B,CAAC,CAAC;aACN;YAAC,OAAO,CAAC,EAAE;gBACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;aAChG;YAED,MAAM,kBAAkB,GAAW,OAAO,OAAO,CAAC,kBAAkB,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,KAAK,CAAC;YAEvH,MAAM,OAAO,GAAG,iBAAO,CAAC,OAAO,CAAC;gBAC5B,OAAO,EAAE,IAAI;gBACb,aAAa;gBACb,KAAK;gBACL,OAAO;gBACP,kBAAkB;gBAClB,WAAW,EAAE,CAAO,IAAiB,EAAE,OAAuB,EAAE,EAAE;oBAE9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;oBACjC,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;oBACvB,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;oBACpE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,CAAC,CAAC;oBAE/C,IAAI,QAAQ,EAAE;wBACV,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,eAAe,YAAY,kCAAkC,CAAC,CAAC;wBAC1E,MAAM,IAAA,uBAAU,EAAC,IAAI,EAAE,YAAY,CAAC,CAAC;wBACrC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;qBACtC;oBAED,mBAAmB;oBACnB,qEAAqE;oBACrE,gFAAgF;oBAChF,kFAAkF;oBAClF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBAClC,MAAM,CAAC,GAAiB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAE3C,MAAM,WAAW,GAAG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAElE,MAAM,SAAS,GAAuB;wBAClC,IAAI,8BAAY,CAAC,IAAI,CAAC;wBACtB,IAAI,kCAAgB,CAAC,IAAI,CAAC;qBAC7B,CAAC;oBAEF,oBAAoB,CAAC,IAAI,CAAC,GAAG,IAAI,+BAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;oBAE1E,MAAM,MAAM,GAAG,IAAI,+BAAa,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,CAAC;oBAEnD,IAAI,aAAwC,CAAC;oBAC7C,IAAI,wBAAiE,CAAC;oBAEtE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;oBAC7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;wBACxB,IAAI,IAAA,iCAAe,EAAC,KAAK,CAAC,EAAE;4BACxB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,aAAa,GAAG,KAAK,CAAC;4BACtB,MAAM,CAAC,aAAa,GAAG,KAAK,CAAC;yBAChC;6BAAM,IAAI,IAAA,4CAA0B,EAAC,KAAK,CAAC,EAAE;4BAC1C,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;4BAChD,wBAAwB,GAAG,KAAK,CAAC;4BACjC,MAAM,CAAC,wBAAwB,GAAG,KAAK,CAAC;yBAC3C;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,KAAK,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;yBACtE;qBACJ;oBAED,8CAA8C;oBAC9C,IAAI,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;wBACpD,MAAM,iBAAiB,GAAG,aAAa,CAAC,IAAI,CAAC;wBAC7C,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;4BACvC,MAAM,MAAM,GAAG,MAAM,IAAA,qBAAa,EAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;4BAC1D,IAAI,MAAM,EAAE;gCACR,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oCAAoC,iBAAiB,EAAE,CAAC,CAAC;gCACrE,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;6BACjC;yBACJ;qBACJ;oBAED,0BAA0B;oBAC1B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;wBACZ,kBAAkB;wBAClB,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;wBAE7C,MAAM,OAAO,GAAG,MAAM,IAAA,uCAAkB,EAAC,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;wBAC5D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,OAAO,aAAa,CAAC,CAAC;wBACrD,8CAA8C;wBAC9C,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC;qBACvB;oBAED,uCAAuC;oBACvC,IAAI,OAAO,CAAC,eAAe,EAAE;wBAEzB,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,eAAe,EAAE,GAAG,QAAQ,MAAM,CAAC,CAAC;wBAEjE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;wBAE5C,MAAM,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;wBAErD,MAAM,IAAI,CAAC,UAAU,CAAC;4BAClB,IAAI;4BACJ,QAAQ;4BACR,qBAAqB;yBACxB,CAAC,CAAC;wBAEH,WAAW,CAAC,IAAI,CAAC;4BACb,QAAQ,EAAE,IAAI;yBACjB,CAAC,CAAC;wBACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;wBAEjC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAC;wBACnC,MAAM,CAAC,GAAG,IAAI,sBAAO,CAAC,IAAI,CAAC,CAAC;wBAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC;wBACrC,IAAI,UAAU,GAAuB,SAAS,CAAC;wBAC/C,IAAI,WAAW,CAAC;wBAChB,IAAI,aAAa,GAAG,EAAE,CAAC;wBAEvB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE;4BAC1B,IAAI,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE;gCACnD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gCAChC,IAAI,CAAC,WAAW,EAAE;oCACd,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;qCAAM,IAAI,OAAO,IAAI,WAAW,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE;oCAC/D,UAAU,GAAG,MAAM,CAAC;oCACpB,WAAW,GAAG,OAAO,CAAC;iCACzB;gCAED,QAAQ,MAAM,EAAE;oCACZ,KAAK,cAAc,CAAC;oCACpB,KAAK,SAAS;wCACV,IAAI,CAAC,YAAY,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC;wCACjC,SAAS;oCACb,KAAK,aAAa;wCACd,IAAI,CAAC,gBAAgB,GAAG,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC;wCACrC,SAAS;oCACb,QAAQ;iCACX;gCAED,MAAM,YAAY,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,CAAC,GAAG,gBAAgB,CAAC;gCAC9E,aAAa,GAAG,GAAG,aAAa,IAAI,MAAM,KAAK,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,UAAU,KAAK,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,IAAI,YAAY,GAAG,CAAA;6BACzG;yBACJ;wBACD,IAAI,UAAU,EAAE;4BACZ,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,UAAU,IAAI,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,EAAE,CAAC,CAAC;yBACxE;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;yBACxD;wBAED,IAAI,CAAC,YAAY,GAAG,WAAW,aAAX,WAAW,uBAAX,WAAW,CAAE,GAAG,CAAC;wBACrC,IAAI,CAAC,cAAc,GAAG,IAAA,0BAAkB,EAAC,IAAI,CAAC,YAAY,CAAC,CAAC;wBAE5D,MAAM,cAAc,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAC;wBACrF,MAAM,gBAAgB,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,cAAc,CAAC,GAAG,gBAAgB,CAAC;wBACzF,MAAM,cAAc,GAAG,YAAY,GAAG,IAAA,gBAAQ,EAAC,IAAI,CAAC,YAAY,CAAC,GAAG,gBAAgB,CAAA;wBAEpF,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,YAAY,IAAI,cAAc,EAAE,CAAC,CAAC;wBACrE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,oBAAoB,IAAI,CAAC,cAAc,IAAI,gBAAgB,EAAE,CAAC,CAAC;wBAC3E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,kBAAkB,IAAI,CAAC,YAAY,IAAI,cAAc,EAAE,CAAC,CAAC;wBACrE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,qBAAqB,aAAa,EAAE,CAAC,CAAC;qBACrD;yBAAM;wBACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,gFAAgF,CAAC,CAAC;qBAChG;oBAED,sBAAsB;oBACtB,MAAM,QAAQ,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAErI,IAAI,OAAO,CAAC,YAAY,EAAE;wBACtB,MAAM,IAAI,GAAG,IAAA,cAAO,EAAC,OAAO,CAAC,YAAY,EAAE,GAAG,QAAQ,KAAK,CAAC,CAAC;wBAC7D,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,uBAAuB,IAAI,KAAK,CAAC,CAAC;wBAC9C,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;wBAC5C,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;qBAC1B;oBAUD,IAAI,MAAM,GAA0B,SAAS,CAAC;oBAE9C,MAAM,EAAE,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,GAAG,OAAO,CAAC;oBAC3D,yCAAyC;oBACzC,IAAI,SAAS,EAAE;wBAEX,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;wBAEjC,MAAM,SAAS,GAAG,IAAA,gCAAoB,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBAEtD,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;wBACtC,IAAI,SAAS,GAAW,SAAS,CAAC,SAAS,CAAC;wBAC5C,IAAI,MAAM,GAAW,SAAS,CAAC,MAAM,CAAC;wBAEtC,+BAA+B;wBAC/B,IAAI,SAAS,GAAG,CAAC,IAAI,OAAO,EAAE;4BAC1B,iBAAiB;4BACjB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,wDAAwD,OAAO,KAAK,CAAC,CAAC;4BACjF,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;4BACzB,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;4BACxB,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;4BACzC,MAAM,eAAe,GAAG,IAAI,mCAAiB,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,oCAAkB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;4BAEnJ,MAAM,aAAa,GAAG,IAAA,gCAAoB,EAAC,eAAe,EAAE,GAAG,CAAC,CAAC;4BAEjE,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC;4BAEvF,IAAI,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE;gCAC7B,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;gCACnC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;gCAC9B,SAAS,GAAG,aAAa,CAAC,SAAS,CAAC;gCACpC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;6BACjC;yBACJ;wBAED,IAAI,SAAS,GAAG,CAAC,IAAI,aAAa,IAAI,wBAAwB,EAAE;4BAC5D,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;4BAC/F,4CAA4C;4BAE5C,MAAM,gBAAgB,GAAG,IAAA,+BAAmB,EAAC,aAAa,IAAI,wBAAwB,EAAE,GAAG,CAAC,CAAC;4BAE7F,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uBAAuB,gBAAgB,CAAC,MAAM,SAAS,gBAAgB,CAAC,SAAS,EAAE,CAAC,CAAC;4BAEhG,IAAI,gBAAgB,CAAC,SAAS,GAAG,CAAC,EAAE;gCAChC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAA;gCACnE,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;gCACjC,SAAS,GAAG,gBAAgB,CAAC,SAAS,CAAC;gCACvC,MAAM,GAAG,gBAAgB,CAAC,MAAM,CAAC;6BACpC;yBACJ;wBAED,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,mBAAmB,MAAM,kCAAkC,SAAS,GAAG,CAAC,CAAC;wBAEpF,MAAM,OAAO,GAAgB;4BACzB,MAAM,EAAE,MAAM;4BACd,OAAO,EAAE;gCACL,cAAc,EAAE,kBAAkB;gCAClC,eAAe,EAAE,UAAU,SAAS,EAAE;6BACzC;4BACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gCACjB,KAAK,EAAE,kBAAkB;gCACzB,MAAM;gCACN,WAAW,EAAE,IAAI;gCACjB,UAAU,EAAE,SAAS;gCACrB,KAAK,EAAE,CAAC;gCACR,iBAAiB,EAAE,CAAC;gCACpB,gBAAgB,EAAE,CAAC;6BACtB,CAAC;yBACL,CAAA;wBAED,sCAAsC;wBACtC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;wBAC5B,IAAI,MAAM,GAAW,GAAG,CAAC;wBACzB,MAAM,cAAc,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;4BACnG,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;4BACzB,IAAI,MAAM,KAAK,GAAG,EAAE;gCAChB,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;6BAC1B;4BACD,OAAO,QAAQ,CAAC;wBACpB,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;4BACZ,sCAAsC;4BACtC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;wBACnC,CAAC,CAAC,CAAC;wBAEH,IAAI,MAAM,KAAK,GAAG,EAAE;4BAEhB,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC;4BACnC,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,8BAA8B,KAAK,CAAC,aAAa,eAAe,KAAK,CAAC,iBAAiB,UAAU,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;4BAEjI,mDAAmD;4BACnD,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC1D,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;4BAEhC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;4BAClD,sCAAsC;4BAEtC,iDAAiD;4BACjD,4CAA4C;4BAC5C,8CAA8C;yBACjD;6BAAM;4BACH,IAAA,oBAAG,GAAE,CAAC,IAAI,CAAC,kDAAkD,MAAM,EAAE,CAAC,CAAC;4BACvE,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;yBAC/B;qBACJ;oBAED,MAAM,YAAY,GAAG,IAAA,iCAAe,EAAC,MAAM,CAAC,aAAa,IAAI,MAAM,CAAC,wBAAwB,CAAC,CAAC;oBAE9F,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,iBAAiB,EAAE;wBAC3B,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,iBAAiB,CAAC;qBAC5C;yBAAM,IAAI,YAAY,CAAC,IAAI,EAAE;wBAC1B,QAAQ,CAAC,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC;qBACrC;oBAED,iBAAiB;oBACjB,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,WAAW,EAAE;wBACrB,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;qBACzC;yBAAM;wBACH,gCAAgC;wBAChC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;qBAClC;oBAED,IAAI,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,eAAe,EAAE;wBACzB,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,eAAe,CAAC;qBAC/C;yBAAM;wBACH,uCAAuC;wBACvC,sFAAsF;wBACtF,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,mHAAmH,CAAC,CAAC;wBAE9I,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;wBAElC,sBAAsB;wBACtB,IAAI,OAAO,IAAI,IAAA,iCAAiB,EAAC,OAAO,CAAC,EAAE;4BACvC,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,MAAM,iBAAiB,CAAC,CAAC;4BACtD,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;4BACvC,MAAM,OAAO,GAAW,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,0BAAU,EAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAC7E,IAAA,oBAAG,GAAE,CAAC,KAAK,CAAC,cAAc,OAAO,GAAG,CAAC,CAAC;4BACtC,IAAI,CAAC,aAAa,GAAG,CAAC,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC,CAAC;yBACpD;qBACJ;gBACL,CAAC,CAAA;gBACD,0BAA0B;aAC7B,CAAC,CAAC;YAEH,MAAM,OAAO,CAAC,GAAG,EAAE,CAAC;YAEpB,OAAO;gBACH,IAAI;gBACJ,oBAAoB;gBACpB,MAAM;gBACN,WAAW;gBACX,QAAQ;aACX,CAAC;QACN,CAAC;KAAA;CACJ;AA7ZD,8CA6ZC"}
|
package/lib/index.d.ts
CHANGED
package/lib/index.js
CHANGED
|
@@ -16,4 +16,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
/*! Copyright (c) 2023, XAPP AI */
|
|
18
18
|
__exportStar(require("./ArachneWebService"), exports);
|
|
19
|
+
__exportStar(require("./models"), exports);
|
|
19
20
|
//# sourceMappingURL=index.js.map
|
package/lib/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,sDAAoC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,sDAAoC;AACpC,2CAAyB"}
|
package/lib/models.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
3
|
+
export interface WebsiteSchema {
|
|
4
|
+
localBusiness?: LocalBusiness;
|
|
5
|
+
homeConstructionBusiness?: HomeAndConstructionBusiness;
|
|
6
|
+
}
|
|
7
|
+
export interface WebsiteDescriptiveData {
|
|
8
|
+
/**
|
|
9
|
+
* A square image that most likely is the logo.
|
|
10
|
+
*/
|
|
11
|
+
logo?: string;
|
|
12
|
+
/**
|
|
13
|
+
* Calls to action detected on the website such as "Free Quote" or "Schedule Consultation"
|
|
14
|
+
*/
|
|
15
|
+
callsToAction?: {
|
|
16
|
+
callToAction: string;
|
|
17
|
+
url?: string;
|
|
18
|
+
}[];
|
|
19
|
+
/**
|
|
20
|
+
* A description of the organization found on the website
|
|
21
|
+
*/
|
|
22
|
+
description?: string;
|
|
23
|
+
/**
|
|
24
|
+
* Phone numbers
|
|
25
|
+
*/
|
|
26
|
+
phoneNumbers?: {
|
|
27
|
+
number: string;
|
|
28
|
+
purpose?: string;
|
|
29
|
+
}[];
|
|
30
|
+
/**
|
|
31
|
+
* Primary (most popular) color detected on the website
|
|
32
|
+
*/
|
|
33
|
+
primaryColor?: string;
|
|
34
|
+
/**
|
|
35
|
+
* Secondary color that complements the primary color on the website.
|
|
36
|
+
*/
|
|
37
|
+
secondaryColor?: string;
|
|
38
|
+
/**
|
|
39
|
+
* Vibrant color, excellent for highlight
|
|
40
|
+
*/
|
|
41
|
+
vibrantColor?: string;
|
|
42
|
+
/**
|
|
43
|
+
* Dark vibrant color
|
|
44
|
+
*/
|
|
45
|
+
darkVibrantColor?: string;
|
|
46
|
+
}
|
|
47
|
+
export interface ScreenshotLocation {
|
|
48
|
+
/**
|
|
49
|
+
* The local file path that the screenshot was saved to.
|
|
50
|
+
*/
|
|
51
|
+
filePath: string;
|
|
52
|
+
}
|
|
53
|
+
export interface BusinessHourPeriod {
|
|
54
|
+
open: {
|
|
55
|
+
day: "SUNDAY" | "MONDAY" | "TUESDAY" | "WEDNESDAY" | "THURSDAY" | "FRIDAY" | "SATURDAY";
|
|
56
|
+
time: string;
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
type HOME_CONSTRUCTION_SERVICES = "plumbing" | "hvac" | "roofing" | "remodelling" | "";
|
|
60
|
+
type HOME_SERVICES = "moving" | "cleaning";
|
|
61
|
+
type PROFESSIONAL_SERVICES = "legal";
|
|
62
|
+
export interface BusinessAddress {
|
|
63
|
+
formated?: string;
|
|
64
|
+
streetAddress?: string;
|
|
65
|
+
postalCode?: string;
|
|
66
|
+
/**
|
|
67
|
+
* City, Town or Locality. This is typically from the "city" of "city", "state"
|
|
68
|
+
*/
|
|
69
|
+
locality?: string;
|
|
70
|
+
region?: string;
|
|
71
|
+
country?: string;
|
|
72
|
+
}
|
|
73
|
+
export interface BusinessAreasServed {
|
|
74
|
+
locality?: string;
|
|
75
|
+
}
|
|
76
|
+
export interface BusinessData {
|
|
77
|
+
/**
|
|
78
|
+
* Name of the business
|
|
79
|
+
*/
|
|
80
|
+
name?: string;
|
|
81
|
+
services?: (HOME_SERVICES | HOME_CONSTRUCTION_SERVICES | PROFESSIONAL_SERVICES)[];
|
|
82
|
+
/**
|
|
83
|
+
* How the business describes themselves
|
|
84
|
+
*/
|
|
85
|
+
typeRaw?: string;
|
|
86
|
+
/**
|
|
87
|
+
* Phone numbers
|
|
88
|
+
*/
|
|
89
|
+
phoneNumbers?: {
|
|
90
|
+
number: string;
|
|
91
|
+
purpose?: string;
|
|
92
|
+
}[];
|
|
93
|
+
/**
|
|
94
|
+
* Primary business address
|
|
95
|
+
*/
|
|
96
|
+
address?: BusinessAddress;
|
|
97
|
+
serviceArea?: BusinessAreasServed;
|
|
98
|
+
geo?: {
|
|
99
|
+
lat: string;
|
|
100
|
+
lon: string;
|
|
101
|
+
};
|
|
102
|
+
hours?: {
|
|
103
|
+
periods: [];
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
export {};
|
package/lib/models.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import { HomeAndConstructionBusiness, LocalBusiness } from "schema-dts";
|
|
3
|
+
import { BusinessData } from "../models";
|
|
4
|
+
export declare function getBusinessData(schema: LocalBusiness | HomeAndConstructionBusiness | undefined): Partial<BusinessData>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getBusinessData = void 0;
|
|
4
|
+
function getBusinessData(schema) {
|
|
5
|
+
if (!schema) {
|
|
6
|
+
return {};
|
|
7
|
+
}
|
|
8
|
+
const partial = {};
|
|
9
|
+
// try to get it from JSON?
|
|
10
|
+
if (schema && typeof schema === "object" && typeof (schema === null || schema === void 0 ? void 0 : schema.name) === "string") {
|
|
11
|
+
partial.name = schema === null || schema === void 0 ? void 0 : schema.name;
|
|
12
|
+
}
|
|
13
|
+
return partial;
|
|
14
|
+
}
|
|
15
|
+
exports.getBusinessData = getBusinessData;
|
|
16
|
+
//# sourceMappingURL=getBusinessData.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getBusinessData.js","sourceRoot":"","sources":["../../src/utils/getBusinessData.ts"],"names":[],"mappings":";;;AAIA,SAAgB,eAAe,CAAC,MAA+D;IAE3F,IAAI,CAAC,MAAM,EAAE;QACT,OAAO,EAAE,CAAC;KACb;IAED,MAAM,OAAO,GAA0B,EAAE,CAAC;IAE1C,2BAA2B;IAC3B,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,CAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CAAA,KAAK,QAAQ,EAAE;QAC1E,OAAO,CAAC,IAAI,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,IAAI,CAAC;KAC/B;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAdD,0CAcC"}
|
|
File without changes
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getColors.js","sourceRoot":"","sources":["../../src/utils/getColors.ts"],"names":[],"mappings":";AACA,SAAS;AAER,kCAAkC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface Prompt {
|
|
2
|
+
prompt: string;
|
|
3
|
+
tokens: number;
|
|
4
|
+
maxTokens: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function getTokens(prompt: string, response?: number): {
|
|
7
|
+
tokens: number;
|
|
8
|
+
maxTokens: number;
|
|
9
|
+
};
|
|
10
|
+
export declare function getWebMarkdownPrompt(markdown: string, url: string): Prompt;
|
|
11
|
+
export declare function getJSONSchemaPrompt(json: any, url: string): Prompt;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.getJSONSchemaPrompt = exports.getWebMarkdownPrompt = exports.getTokens = void 0;
|
|
4
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
5
|
+
const { encode } = require('gpt-3-encoder');
|
|
6
|
+
function getTokens(prompt, response = 500) {
|
|
7
|
+
const tokens = encode(prompt).length;
|
|
8
|
+
const maxTokens = 4000 - response - tokens;
|
|
9
|
+
return { tokens, maxTokens };
|
|
10
|
+
}
|
|
11
|
+
exports.getTokens = getTokens;
|
|
12
|
+
function getWebMarkdownPrompt(markdown, url) {
|
|
13
|
+
const prompt = `Your task is to analyze the contents of a website (provided in markdown below) found at ${url} and provide information about the organization from what is on the website. You will only provide information available through the website content about the services they provide, contact information and a description of the business.\n\n===========\n${markdown}\n===========\nPlease print out in JSON format information about the website above with the following keys:\n\n\"organization_name\" - The name of the organization for the website.\n\"phone_numbers\" - An array of phone numbers for the organization. If available, include what they are used for such as \"customer support\" or \"sales\", if unknown then leave it blank. Use an object with keys: number and purpose.\n\"addresses\" - An array of addresses for the organization.\n\"services\" - An array of the top five unique most common services the organization provides, they should be concinse and no more than a few words such as \"Roofing\", \"Conversational AI Services\", \"Bathroom Remodelling\"\n\"calls_to_action\" - An array of the top five unique calls to action found on the site and their fully formed URL if they have one. Good examples of calls to actions are \"Request Demo\", \"Get Started\", \"Free Quote\", \"Schedule Consultation\", \"Sign Up\". Bad examples are \"Learn More\" or \"Follow us on Facebook\". Use an object with keys: url & callToAction in the array. For the url please combine it properly with the website's URL.\n\"description\" - A description of what the organization does, include the name of the organization, their services, their location, phone number, email. Keep it concise and to a few sentences and around 480 characters.\n \n{`;
|
|
14
|
+
const tokens = getTokens(prompt);
|
|
15
|
+
return Object.assign({ prompt }, tokens);
|
|
16
|
+
}
|
|
17
|
+
exports.getWebMarkdownPrompt = getWebMarkdownPrompt;
|
|
18
|
+
function getJSONSchemaPrompt(json, url) {
|
|
19
|
+
const jsonString = JSON.stringify(json);
|
|
20
|
+
const prompt = `${jsonString}\n====\nAbove is the JSON schema for a website found at ${url}. Please print out in JSON format information about the website above with the following keys:\n\n\"organization_name\" - The name of the organization for the website.\n\"description\" - A description of what the organization does, include the name of the organization, their services, their location, phone number, email and hours if available. Keep it concise and to a few sentences and around 480 characters.\n \n{`;
|
|
21
|
+
const tokens = getTokens(prompt);
|
|
22
|
+
return Object.assign({ prompt }, tokens);
|
|
23
|
+
}
|
|
24
|
+
exports.getJSONSchemaPrompt = getJSONSchemaPrompt;
|
|
25
|
+
//# sourceMappingURL=getPrompt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getPrompt.js","sourceRoot":"","sources":["../../src/utils/getPrompt.ts"],"names":[],"mappings":";;;AAAA,kCAAkC;AAClC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;AAS5C,SAAgB,SAAS,CAAC,MAAc,EAAE,WAAmB,GAAG;IAC5D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrC,MAAM,SAAS,GAAW,IAAI,GAAG,QAAQ,GAAG,MAAM,CAAC;IAEnD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AACjC,CAAC;AALD,8BAKC;AAGD,SAAgB,oBAAoB,CAAC,QAAgB,EAAE,GAAW;IAE9D,MAAM,MAAM,GAAG,2FAA2F,GAAG,iQAAiQ,QAAQ,w2CAAw2C,CAAC;IAC/tD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IACjC,uBACI,MAAM,IACH,MAAM,EACZ;AACL,CAAC;AARD,oDAQC;AAED,SAAgB,mBAAmB,CAAC,IAAS,EAAE,GAAW;IAEtD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,GAAG,UAAU,2DAA2D,GAAG,saAAsa,CAAC;IACjgB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAEjC,uBACI,MAAM,IACH,MAAM,EACZ;AACL,CAAC;AAVD,kDAUC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
2
|
+
import "isomorphic-fetch";
|
|
3
|
+
/**
|
|
4
|
+
* Crawls a webpages sitemap and returns a list of all the URLs found within.
|
|
5
|
+
*
|
|
6
|
+
* It will additionally crawl and sitemaps listed within the original.
|
|
7
|
+
*
|
|
8
|
+
* @note Written with the help of OpenAI
|
|
9
|
+
*
|
|
10
|
+
* @param sitemapUrl
|
|
11
|
+
* @returns
|
|
12
|
+
*/
|
|
13
|
+
export declare function getSitemap(sitemapUrl: string): Promise<string[]>;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
28
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
29
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
30
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
31
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
+
exports.getSitemap = void 0;
|
|
36
|
+
/*! Copyright (c) 2023, XAPP AI */
|
|
37
|
+
require("isomorphic-fetch"); // must be at top, making available to all
|
|
38
|
+
const cheerio = __importStar(require("cheerio"));
|
|
39
|
+
const fetchSitemap = (url) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
+
try {
|
|
41
|
+
const response = yield fetch(url);
|
|
42
|
+
return response.text();
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
console.error(`Failed to fetch sitemap: ${error}`);
|
|
46
|
+
return '';
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
/**
|
|
50
|
+
* Crawls a webpages sitemap and returns a list of all the URLs found within.
|
|
51
|
+
*
|
|
52
|
+
* It will additionally crawl and sitemaps listed within the original.
|
|
53
|
+
*
|
|
54
|
+
* @note Written with the help of OpenAI
|
|
55
|
+
*
|
|
56
|
+
* @param sitemapUrl
|
|
57
|
+
* @returns
|
|
58
|
+
*/
|
|
59
|
+
function getSitemap(sitemapUrl) {
|
|
60
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
61
|
+
const xmlData = yield fetchSitemap(sitemapUrl);
|
|
62
|
+
if (!xmlData) {
|
|
63
|
+
console.error('No sitemap data retrieved.');
|
|
64
|
+
return [];
|
|
65
|
+
}
|
|
66
|
+
const $ = cheerio.load(xmlData, {
|
|
67
|
+
xmlMode: true
|
|
68
|
+
});
|
|
69
|
+
let allUrls = [];
|
|
70
|
+
// Extract all urls
|
|
71
|
+
$('url').each((i, url) => {
|
|
72
|
+
const loc = $(url).find('loc').text();
|
|
73
|
+
allUrls.push(loc);
|
|
74
|
+
});
|
|
75
|
+
// Extract sitemaps and crawl recursively
|
|
76
|
+
const sitemapNodes = $('sitemap').toArray();
|
|
77
|
+
for (const sitemapNode of sitemapNodes) {
|
|
78
|
+
const loc = $(sitemapNode).find('loc').text();
|
|
79
|
+
const urls = yield getSitemap(loc);
|
|
80
|
+
allUrls = allUrls.concat(urls);
|
|
81
|
+
}
|
|
82
|
+
return allUrls;
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
exports.getSitemap = getSitemap;
|
|
86
|
+
//# sourceMappingURL=getSiteMap.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"getSiteMap.js","sourceRoot":"","sources":["../../src/utils/getSiteMap.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,kCAAkC;AAClC,4BAA0B,CAAC,0CAA0C;AAErE,iDAAmC;AAEnC,MAAM,YAAY,GAAG,CAAO,GAAW,EAAmB,EAAE;IACxD,IAAI;QACA,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;KAC1B;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACnD,OAAO,EAAE,CAAC;KACb;AACL,CAAC,CAAA,CAAA;AAED;;;;;;;;;GASG;AACH,SAAsB,UAAU,CAAC,UAAkB;;QAC/C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE;YACV,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC5C,OAAO,EAAE,CAAC;SACb;QAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE;YAC5B,OAAO,EAAE,IAAI;SAChB,CAAC,CAAC;QAEH,IAAI,OAAO,GAAa,EAAE,CAAC;QAE3B,mBAAmB;QACnB,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE;YACrB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5C,KAAK,MAAM,WAAW,IAAI,YAAY,EAAE;YACpC,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9C,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAC;YACnC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;SAClC;QAED,OAAO,OAAO,CAAC;IACnB,CAAC;CAAA;AA7BD,gCA6BC"}
|
package/lib/utils/image.d.ts
CHANGED
|
@@ -19,4 +19,5 @@ export declare function imageIsSquare(url: string, minimumSize?: number): Promis
|
|
|
19
19
|
* @returns
|
|
20
20
|
*/
|
|
21
21
|
export declare function complementHexColor(hex: string | undefined): string | undefined;
|
|
22
|
+
export declare function hexToRGB(hex: string | undefined): string | undefined;
|
|
22
23
|
export declare function generateImage(height: number, width: number, hex: string): Promise<Buffer>;
|
package/lib/utils/image.js
CHANGED
|
@@ -13,7 +13,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
13
13
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
14
14
|
};
|
|
15
15
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
-
exports.generateImage = exports.complementHexColor = exports.imageIsSquare = void 0;
|
|
16
|
+
exports.generateImage = exports.hexToRGB = exports.complementHexColor = exports.imageIsSquare = void 0;
|
|
17
17
|
const sharp_1 = __importDefault(require("sharp"));
|
|
18
18
|
/**
|
|
19
19
|
* Determines if the image at the URL provided is square.
|
|
@@ -72,6 +72,16 @@ function complementHexColor(hex) {
|
|
|
72
72
|
return compColor;
|
|
73
73
|
}
|
|
74
74
|
exports.complementHexColor = complementHexColor;
|
|
75
|
+
function hexToRGB(hex) {
|
|
76
|
+
if (!hex) {
|
|
77
|
+
return hex;
|
|
78
|
+
}
|
|
79
|
+
const r = parseInt(hex.slice(1, 3), 16);
|
|
80
|
+
const g = parseInt(hex.slice(3, 5), 16);
|
|
81
|
+
const b = parseInt(hex.slice(5, 7), 16);
|
|
82
|
+
return `${r};${g};${b}`;
|
|
83
|
+
}
|
|
84
|
+
exports.hexToRGB = hexToRGB;
|
|
75
85
|
// not tested yet
|
|
76
86
|
function generateImage(height, width, hex) {
|
|
77
87
|
return __awaiter(this, void 0, void 0, function* () {
|
package/lib/utils/image.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;AAElC,kDAA0B;AAE1B;;;;;;;;GAQG;AACH,SAAsB,aAAa,CAAC,GAAW,EAAE,WAAoB;;QAEjE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAG,MAAM,IAAA,eAAK,EAAC,SAAS,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,KAAK,MAAM,CAAC;QAElC,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC9D,OAAO,QAAQ,IAAI,KAAK,IAAI,WAAW,CAAC;SAC3C;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CAAA;AAnBD,sCAmBC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAAC,GAAuB;IACtD,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,SAAS,CAAC;KACpB;IACD,kDAAkD;IAClD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;KACnD;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;QAClB,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KACnE;IAED,oCAAoC;IACpC,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE;QAC3B,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5E,SAAS,IAAI,aAAa,CAAC;KAC9B;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAxBD,gDAwBC;AAGD,iBAAiB;AACjB,SAAsB,aAAa,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW;;QAC1E,OAAO,IAAA,eAAK,EAAC;YACT,MAAM,EAAE;gBACJ,KAAK;gBACL,MAAM;gBACN,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,GAAG;aAClB;SACJ,CAAC;aACG,GAAG,EAAE;aACL,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAXD,sCAWC"}
|
|
1
|
+
{"version":3,"file":"image.js","sourceRoot":"","sources":["../../src/utils/image.ts"],"names":[],"mappings":";AAAA,kCAAkC;;;;;;;;;;;;;;;AAElC,kDAA0B;AAE1B;;;;;;;;GAQG;AACH,SAAsB,aAAa,CAAC,GAAW,EAAE,WAAoB;;QAEjE,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5C,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAE3C,MAAM,KAAK,GAAG,MAAM,IAAA,eAAK,EAAC,SAAS,CAAC,CAAC;QACrC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QAEpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAE3B,MAAM,QAAQ,GAAG,KAAK,KAAK,MAAM,CAAC;QAElC,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YAC9D,OAAO,QAAQ,IAAI,KAAK,IAAI,WAAW,CAAC;SAC3C;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;CAAA;AAnBD,sCAmBC;AAED;;;;;;;GAOG;AACH,SAAgB,kBAAkB,CAAC,GAAuB;IACtD,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,SAAS,CAAC;KACpB;IACD,kDAAkD;IAClD,MAAM,QAAQ,GAAG,oCAAoC,CAAC;IACtD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;QACrB,MAAM,IAAI,SAAS,CAAC,0BAA0B,CAAC,CAAC;KACnD;IAED,gDAAgD;IAChD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;QAClB,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;KACnE;IAED,oCAAoC;IACpC,IAAI,SAAS,GAAG,GAAG,CAAC;IACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE;QAC3B,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,CAAC,IAAI,GAAG,cAAc,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC5E,SAAS,IAAI,aAAa,CAAC;KAC9B;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAxBD,gDAwBC;AAED,SAAgB,QAAQ,CAAC,GAAuB;IAC5C,IAAI,CAAC,GAAG,EAAE;QACN,OAAO,GAAG,CAAC;KACd;IACD,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxC,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AAC5B,CAAC;AARD,4BAQC;AAGD,iBAAiB;AACjB,SAAsB,aAAa,CAAC,MAAc,EAAE,KAAa,EAAE,GAAW;;QAC1E,OAAO,IAAA,eAAK,EAAC;YACT,MAAM,EAAE;gBACJ,KAAK;gBACL,MAAM;gBACN,QAAQ,EAAE,CAAC;gBACX,UAAU,EAAE,GAAG;aAClB;SACJ,CAAC;aACG,GAAG,EAAE;aACL,QAAQ,EAAE,CAAC;IACpB,CAAC;CAAA;AAXD,sCAWC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@xapp/arachne-web-service",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.1",
|
|
4
4
|
"types": "lib/index",
|
|
5
5
|
"main": "lib/index",
|
|
6
6
|
"files": [
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"node": "^14 || ^16 || ^18"
|
|
15
15
|
},
|
|
16
16
|
"devDependencies": {
|
|
17
|
-
"@microsoft/api-extractor": "7.
|
|
17
|
+
"@microsoft/api-extractor": "7.35.0",
|
|
18
18
|
"@types/chai": "4.3.4",
|
|
19
19
|
"@types/mocha": "10.0.1",
|
|
20
20
|
"@types/node": "18.16.7",
|
|
@@ -24,23 +24,23 @@
|
|
|
24
24
|
"chai": "4.3.7",
|
|
25
25
|
"dotenv": "16.0.3",
|
|
26
26
|
"mocha": "10.2.0",
|
|
27
|
-
"sinon": "15.0
|
|
27
|
+
"sinon": "15.1.0",
|
|
28
28
|
"sinon-chai": "3.7.0",
|
|
29
29
|
"ts-node": "10.9.1",
|
|
30
|
-
"tslib": "2.5.
|
|
31
|
-
"typescript": "
|
|
30
|
+
"tslib": "2.5.2",
|
|
31
|
+
"typescript": "5.0.4"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@xapp/arachne": "0.
|
|
35
|
-
"@xapp/arachne-parsers": "0.
|
|
34
|
+
"@xapp/arachne": "0.6.1",
|
|
35
|
+
"@xapp/arachne-parsers": "0.6.1",
|
|
36
36
|
"gpt-3-encoder": "1.1.4",
|
|
37
37
|
"isomorphic-fetch": "3.0.0",
|
|
38
38
|
"node-vibrant": "3.2.1-alpha.1",
|
|
39
39
|
"schema-dts": "1.1.2",
|
|
40
40
|
"sharp": "0.32.1",
|
|
41
|
-
"stentor-logger": "1.57.
|
|
42
|
-
"stentor-models": "1.57.
|
|
43
|
-
"stentor-utils": "1.57.
|
|
41
|
+
"stentor-logger": "1.57.143",
|
|
42
|
+
"stentor-models": "1.57.143",
|
|
43
|
+
"stentor-utils": "1.57.143"
|
|
44
44
|
},
|
|
45
45
|
"scripts": {
|
|
46
46
|
"api": "api-extractor run --local",
|
|
@@ -48,5 +48,5 @@
|
|
|
48
48
|
"clean": "rm -rf ./lib/*",
|
|
49
49
|
"test:f": "mocha --recursive -r ts-node/register \"./src/**/*.ftest.ts\""
|
|
50
50
|
},
|
|
51
|
-
"gitHead": "
|
|
51
|
+
"gitHead": "f46a5f6a7f4c64b97555f393cfd26df11b85e0f7"
|
|
52
52
|
}
|