@dbx-tools/appkit-web-search 0.3.44 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/index.d.ts +32 -0
- package/lib/index.js +28 -0
- package/lib/src/allowlist.d.ts +66 -0
- package/lib/src/allowlist.js +113 -0
- package/lib/src/config.d.ts +233 -0
- package/lib/src/config.js +269 -0
- package/lib/src/defaults.d.ts +98 -0
- package/lib/src/defaults.js +96 -0
- package/lib/src/fetch.d.ts +27 -0
- package/lib/src/fetch.js +86 -0
- package/lib/src/html-text.d.ts +30 -0
- package/lib/src/html-text.js +66 -0
- package/lib/src/plugin.d.ts +162 -0
- package/lib/src/plugin.js +225 -0
- package/lib/src/provider.d.ts +62 -0
- package/lib/src/provider.js +89 -0
- package/lib/src/runtime.d.ts +61 -0
- package/lib/src/runtime.js +95 -0
- package/lib/src/schema.d.ts +76 -0
- package/lib/src/schema.js +102 -0
- package/lib/src/scrape.d.ts +29 -0
- package/lib/src/scrape.js +113 -0
- package/lib/src/search.d.ts +49 -0
- package/lib/src/search.js +252 -0
- package/lib/src/tool.d.ts +85 -0
- package/lib/src/tool.js +113 -0
- package/lib/tsconfig.tsbuildinfo +1 -0
- package/package.json +13 -9
package/lib/src/fetch.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Page fetching over {@link https://www.npmjs.com/package/got-scraping | got-scraping}
|
|
3
|
+
* - a `got` wrapper that generates browser-like TLS + header fingerprints, so
|
|
4
|
+
* fetches survive the bot walls a plain `fetch` trips. {@link runWebFetch}
|
|
5
|
+
* enforces the URL allow-list (an explicit fetch of a disallowed URL is
|
|
6
|
+
* refused, not silently emptied), fetches with the plugin's timeout, and
|
|
7
|
+
* returns either the raw HTML or a readable plain-text reduction, capped at
|
|
8
|
+
* the configured length.
|
|
9
|
+
*
|
|
10
|
+
* The HTML-to-text reduction is deliberately dependency-free (strip
|
|
11
|
+
* script/style, unwrap tags, decode a handful of entities, collapse
|
|
12
|
+
* whitespace): good enough to feed a model, and it keeps the add-on's
|
|
13
|
+
* dependency surface to the two libraries the task called for.
|
|
14
|
+
*
|
|
15
|
+
* @module
|
|
16
|
+
*/
|
|
17
|
+
import { log } from "@dbx-tools/shared-core";
|
|
18
|
+
import { gotScraping } from "got-scraping";
|
|
19
|
+
import { assertUrlAllowed } from "./allowlist.js";
|
|
20
|
+
import { toCallSettings, webFetchExecuteDefaults } from "./defaults.js";
|
|
21
|
+
import { decodeHtmlEntities, htmlToText } from "./html-text.js";
|
|
22
|
+
import { executeRead } from "./runtime.js";
|
|
23
|
+
const logger = log.logger("web-search/fetch");
|
|
24
|
+
/** Pull the <title> text out of an HTML document, when present. */
|
|
25
|
+
function extractTitle(html) {
|
|
26
|
+
const match = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
|
|
27
|
+
const title = match?.[1] ? decodeHtmlEntities(match[1]).trim() : "";
|
|
28
|
+
return title.length > 0 ? title : undefined;
|
|
29
|
+
}
|
|
30
|
+
/** Truncate `text` to `max` chars, reporting whether it was cut. */
|
|
31
|
+
function truncate(text, max) {
|
|
32
|
+
if (text.length <= max)
|
|
33
|
+
return { content: text, truncated: false };
|
|
34
|
+
return { content: text.slice(0, max), truncated: true };
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Fetch a single URL and return its content in the requested format.
|
|
38
|
+
*
|
|
39
|
+
* Throws when the URL is not permitted by the allow-list (the visible,
|
|
40
|
+
* correctable failure the design calls for on the fetch path). The request's
|
|
41
|
+
* `maxLength` narrows (never widens) the plugin's `fetchMaxLength` cap.
|
|
42
|
+
* `signal` cancels the in-flight fetch.
|
|
43
|
+
*/
|
|
44
|
+
export async function runWebFetch(request, config, signal) {
|
|
45
|
+
assertUrlAllowed(request.url, config.allowList);
|
|
46
|
+
const cap = Math.min(request.maxLength ?? config.fetchMaxLength, config.fetchMaxLength);
|
|
47
|
+
// The response is cached before the format reduction, so the same page read
|
|
48
|
+
// as text and as html shares one network round trip.
|
|
49
|
+
const page = await executeRead("page-fetch", toCallSettings(webFetchExecuteDefaults, config.timeoutMs, ["web-search", "fetch", request.url]), async (executeSignal) => {
|
|
50
|
+
const response = await gotScraping({
|
|
51
|
+
url: request.url,
|
|
52
|
+
// got's own timeout aborts the socket and reports which phase timed
|
|
53
|
+
// out; the interceptor timeout bounds the whole attempt around it.
|
|
54
|
+
timeout: { request: config.timeoutMs },
|
|
55
|
+
throwHttpErrors: false,
|
|
56
|
+
followRedirect: true,
|
|
57
|
+
...(executeSignal ? { signal: executeSignal } : {}),
|
|
58
|
+
});
|
|
59
|
+
return {
|
|
60
|
+
url: response.url ?? request.url,
|
|
61
|
+
statusCode: response.statusCode,
|
|
62
|
+
contentType: response.headers["content-type"],
|
|
63
|
+
body: typeof response.body === "string" ? response.body : String(response.body ?? ""),
|
|
64
|
+
};
|
|
65
|
+
}, signal);
|
|
66
|
+
const isHtml = !page.contentType || /html|xml/i.test(page.contentType);
|
|
67
|
+
const rawContent = request.format === "html" || !isHtml ? page.body : htmlToText(page.body);
|
|
68
|
+
const { content, truncated } = truncate(rawContent, cap);
|
|
69
|
+
const title = isHtml ? extractTitle(page.body) : undefined;
|
|
70
|
+
logger.debug("fetched", {
|
|
71
|
+
url: page.url,
|
|
72
|
+
status: page.statusCode,
|
|
73
|
+
bytes: page.body.length,
|
|
74
|
+
returned: content.length,
|
|
75
|
+
...(truncated ? { truncated: true } : {}),
|
|
76
|
+
});
|
|
77
|
+
return {
|
|
78
|
+
url: page.url,
|
|
79
|
+
status: page.statusCode,
|
|
80
|
+
...(page.contentType ? { contentType: page.contentType } : {}),
|
|
81
|
+
...(title ? { title } : {}),
|
|
82
|
+
content,
|
|
83
|
+
truncated,
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmV0Y2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvZmV0Y2gudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7Ozs7Ozs7Ozs7Ozs7OztHQWVHO0FBRUgsT0FBTyxFQUFFLEdBQUcsRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBQzdDLE9BQU8sRUFBRSxXQUFXLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFDM0MsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sYUFBYSxDQUFDO0FBRS9DLE9BQU8sRUFBRSxjQUFjLEVBQUUsdUJBQXVCLEVBQUUsTUFBTSxZQUFZLENBQUM7QUFDckUsT0FBTyxFQUFFLGtCQUFrQixFQUFFLFVBQVUsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUM3RCxPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0sV0FBVyxDQUFDO0FBR3hDLE1BQU0sTUFBTSxHQUFHLEdBQUcsQ0FBQyxNQUFNLENBQUMsa0JBQWtCLENBQUMsQ0FBQztBQVU5QyxtRUFBbUU7QUFDbkUsU0FBUyxZQUFZLENBQUMsSUFBWTtJQUNoQyxNQUFNLEtBQUssR0FBRyxrQ0FBa0MsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7SUFDNUQsTUFBTSxLQUFLLEdBQUcsS0FBSyxFQUFFLENBQUMsQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLGtCQUFrQixDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7SUFDcEUsT0FBTyxLQUFLLENBQUMsTUFBTSxHQUFHLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7QUFDOUMsQ0FBQztBQUVELG9FQUFvRTtBQUNwRSxTQUFTLFFBQVEsQ0FBQyxJQUFZLEVBQUUsR0FBVztJQUN6QyxJQUFJLElBQUksQ0FBQyxNQUFNLElBQUksR0FBRztRQUFFLE9BQU8sRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLFNBQVMsRUFBRSxLQUFLLEVBQUUsQ0FBQztJQUNuRSxPQUFPLEVBQUUsT0FBTyxFQUFFLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxFQUFFLEdBQUcsQ0FBQyxFQUFFLFNBQVMsRUFBRSxJQUFJLEVBQUUsQ0FBQztBQUMxRCxDQUFDO0FBRUQ7Ozs7Ozs7R0FPRztBQUNILE1BQU0sQ0FBQyxLQUFLLFVBQVUsV0FBVyxDQUMvQixPQUF3QixFQUN4QixNQUErQixFQUMvQixNQUFvQjtJQUVwQixnQkFBZ0IsQ0FBQyxPQUFPLENBQUMsR0FBRyxFQUFFLE1BQU0sQ0FBQyxTQUFTLENBQUMsQ0FBQztJQUNoRCxNQUFNLEdBQUcsR0FBRyxJQUFJLENBQUMsR0FBRyxDQUFDLE9BQU8sQ0FBQyxTQUFTLElBQUksTUFBTSxDQUFDLGNBQWMsRUFBRSxNQUFNLENBQUMsY0FBYyxDQUFDLENBQUM7SUFFeEYsNEVBQTRFO0lBQzVFLHFEQUFxRDtJQUNyRCxNQUFNLElBQUksR0FBRyxNQUFNLFdBQVcsQ0FDNUIsWUFBWSxFQUNaLGNBQWMsQ0FBQyx1QkFBdUIsRUFBRSxNQUFNLENBQUMsU0FBUyxFQUFFLENBQUMsWUFBWSxFQUFFLE9BQU8sRUFBRSxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUMsRUFDL0YsS0FBSyxFQUFFLGFBQWEsRUFBd0IsRUFBRTtRQUM1QyxNQUFNLFFBQVEsR0FBRyxNQUFNLFdBQVcsQ0FBQztZQUNqQyxHQUFHLEVBQUUsT0FBTyxDQUFDLEdBQUc7WUFDaEIsb0VBQW9FO1lBQ3BFLG1FQUFtRTtZQUNuRSxPQUFPLEVBQUUsRUFBRSxPQUFPLEVBQUUsTUFBTSxDQUFDLFNBQVMsRUFBRTtZQUN0QyxlQUFlLEVBQUUsS0FBSztZQUN0QixjQUFjLEVBQUUsSUFBSTtZQUNwQixHQUFHLENBQUMsYUFBYSxDQUFDLENBQUMsQ0FBQyxFQUFFLE1BQU0sRUFBRSxhQUFhLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1NBQ3BELENBQUMsQ0FBQztRQUNILE9BQU87WUFDTCxHQUFHLEVBQUUsUUFBUSxDQUFDLEdBQUcsSUFBSSxPQUFPLENBQUMsR0FBRztZQUNoQyxVQUFVLEVBQUUsUUFBUSxDQUFDLFVBQVU7WUFDL0IsV0FBVyxFQUFFLFFBQVEsQ0FBQyxPQUFPLENBQUMsY0FBYyxDQUFDO1lBQzdDLElBQUksRUFBRSxPQUFPLFFBQVEsQ0FBQyxJQUFJLEtBQUssUUFBUSxDQUFDLENBQUMsQ0FBQyxRQUFRLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsUUFBUSxDQUFDLElBQUksSUFBSSxFQUFFLENBQUM7U0FDdEYsQ0FBQztJQUNKLENBQUMsRUFDRCxNQUFNLENBQ1AsQ0FBQztJQUVGLE1BQU0sTUFBTSxHQUFHLENBQUMsSUFBSSxDQUFDLFdBQVcsSUFBSSxXQUFXLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxXQUFXLENBQUMsQ0FBQztJQUN2RSxNQUFNLFVBQVUsR0FBRyxPQUFPLENBQUMsTUFBTSxLQUFLLE1BQU0sSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsVUFBVSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUM1RixNQUFNLEVBQUUsT0FBTyxFQUFFLFNBQVMsRUFBRSxHQUFHLFFBQVEsQ0FBQyxVQUFVLEVBQUUsR0FBRyxDQUFDLENBQUM7SUFDekQsTUFBTSxLQUFLLEdBQUcsTUFBTSxDQUFDLENBQUMsQ0FBQyxZQUFZLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsQ0FBQyxTQUFTLENBQUM7SUFFM0QsTUFBTSxDQUFDLEtBQUssQ0FBQyxTQUFTLEVBQUU7UUFDdEIsR0FBRyxFQUFFLElBQUksQ0FBQyxHQUFHO1FBQ2IsTUFBTSxFQUFFLElBQUksQ0FBQyxVQUFVO1FBQ3ZCLEtBQUssRUFBRSxJQUFJLENBQUMsSUFBSSxDQUFDLE1BQU07UUFDdkIsUUFBUSxFQUFFLE9BQU8sQ0FBQyxNQUFNO1FBQ3hCLEdBQUcsQ0FBQyxTQUFTLENBQUMsQ0FBQyxDQUFDLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7S0FDMUMsQ0FBQyxDQUFDO0lBRUgsT0FBTztRQUNMLEdBQUcsRUFBRSxJQUFJLENBQUMsR0FBRztRQUNiLE1BQU0sRUFBRSxJQUFJLENBQUMsVUFBVTtRQUN2QixHQUFHLENBQUMsSUFBSSxDQUFDLFdBQVcsQ0FBQyxDQUFDLENBQUMsRUFBRSxXQUFXLEVBQUUsSUFBSSxDQUFDLFdBQVcsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7UUFDOUQsR0FBRyxDQUFDLEtBQUssQ0FBQyxDQUFDLENBQUMsRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDO1FBQzNCLE9BQU87UUFDUCxTQUFTO0tBQ1YsQ0FBQztBQUNKLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFBhZ2UgZmV0Y2hpbmcgb3ZlciB7QGxpbmsgaHR0cHM6Ly93d3cubnBtanMuY29tL3BhY2thZ2UvZ290LXNjcmFwaW5nIHwgZ290LXNjcmFwaW5nfVxuICogLSBhIGBnb3RgIHdyYXBwZXIgdGhhdCBnZW5lcmF0ZXMgYnJvd3Nlci1saWtlIFRMUyArIGhlYWRlciBmaW5nZXJwcmludHMsIHNvXG4gKiBmZXRjaGVzIHN1cnZpdmUgdGhlIGJvdCB3YWxscyBhIHBsYWluIGBmZXRjaGAgdHJpcHMuIHtAbGluayBydW5XZWJGZXRjaH1cbiAqIGVuZm9yY2VzIHRoZSBVUkwgYWxsb3ctbGlzdCAoYW4gZXhwbGljaXQgZmV0Y2ggb2YgYSBkaXNhbGxvd2VkIFVSTCBpc1xuICogcmVmdXNlZCwgbm90IHNpbGVudGx5IGVtcHRpZWQpLCBmZXRjaGVzIHdpdGggdGhlIHBsdWdpbidzIHRpbWVvdXQsIGFuZFxuICogcmV0dXJucyBlaXRoZXIgdGhlIHJhdyBIVE1MIG9yIGEgcmVhZGFibGUgcGxhaW4tdGV4dCByZWR1Y3Rpb24sIGNhcHBlZCBhdFxuICogdGhlIGNvbmZpZ3VyZWQgbGVuZ3RoLlxuICpcbiAqIFRoZSBIVE1MLXRvLXRleHQgcmVkdWN0aW9uIGlzIGRlbGliZXJhdGVseSBkZXBlbmRlbmN5LWZyZWUgKHN0cmlwXG4gKiBzY3JpcHQvc3R5bGUsIHVud3JhcCB0YWdzLCBkZWNvZGUgYSBoYW5kZnVsIG9mIGVudGl0aWVzLCBjb2xsYXBzZVxuICogd2hpdGVzcGFjZSk6IGdvb2QgZW5vdWdoIHRvIGZlZWQgYSBtb2RlbCwgYW5kIGl0IGtlZXBzIHRoZSBhZGQtb24nc1xuICogZGVwZW5kZW5jeSBzdXJmYWNlIHRvIHRoZSB0d28gbGlicmFyaWVzIHRoZSB0YXNrIGNhbGxlZCBmb3IuXG4gKlxuICogQG1vZHVsZVxuICovXG5cbmltcG9ydCB7IGxvZyB9IGZyb20gXCJAZGJ4LXRvb2xzL3NoYXJlZC1jb3JlXCI7XG5pbXBvcnQgeyBnb3RTY3JhcGluZyB9IGZyb20gXCJnb3Qtc2NyYXBpbmdcIjtcbmltcG9ydCB7IGFzc2VydFVybEFsbG93ZWQgfSBmcm9tIFwiLi9hbGxvd2xpc3RcIjtcbmltcG9ydCB0eXBlIHsgUmVzb2x2ZWRXZWJTZWFyY2hDb25maWcgfSBmcm9tIFwiLi9jb25maWdcIjtcbmltcG9ydCB7IHRvQ2FsbFNldHRpbmdzLCB3ZWJGZXRjaEV4ZWN1dGVEZWZhdWx0cyB9IGZyb20gXCIuL2RlZmF1bHRzXCI7XG5pbXBvcnQgeyBkZWNvZGVIdG1sRW50aXRpZXMsIGh0bWxUb1RleHQgfSBmcm9tIFwiLi9odG1sLXRleHRcIjtcbmltcG9ydCB7IGV4ZWN1dGVSZWFkIH0gZnJvbSBcIi4vcnVudGltZVwiO1xuaW1wb3J0IHR5cGUgeyBXZWJGZXRjaFJlcXVlc3QsIFdlYkZldGNoUmVzdWx0IH0gZnJvbSBcIi4vc2NoZW1hXCI7XG5cbmNvbnN0IGxvZ2dlciA9IGxvZy5sb2dnZXIoXCJ3ZWItc2VhcmNoL2ZldGNoXCIpO1xuXG4vKiogVGhlIHNsaWNlIG9mIGEgZ290LXNjcmFwaW5nIHJlc3BvbnNlIHRoaXMgbW9kdWxlIHJlYWRzLiAqL1xuaW50ZXJmYWNlIEZldGNoZWRQYWdlIHtcbiAgdXJsOiBzdHJpbmc7XG4gIHN0YXR1c0NvZGU6IG51bWJlcjtcbiAgY29udGVudFR5cGU6IHN0cmluZyB8IHVuZGVmaW5lZDtcbiAgYm9keTogc3RyaW5nO1xufVxuXG4vKiogUHVsbCB0aGUgPHRpdGxlPiB0ZXh0IG91dCBvZiBhbiBIVE1MIGRvY3VtZW50LCB3aGVuIHByZXNlbnQuICovXG5mdW5jdGlvbiBleHRyYWN0VGl0bGUoaHRtbDogc3RyaW5nKTogc3RyaW5nIHwgdW5kZWZpbmVkIHtcbiAgY29uc3QgbWF0Y2ggPSAvPHRpdGxlW14+XSo+KFtcXHNcXFNdKj8pPFxcL3RpdGxlPi9pLmV4ZWMoaHRtbCk7XG4gIGNvbnN0IHRpdGxlID0gbWF0Y2g/LlsxXSA/IGRlY29kZUh0bWxFbnRpdGllcyhtYXRjaFsxXSkudHJpbSgpIDogXCJcIjtcbiAgcmV0dXJuIHRpdGxlLmxlbmd0aCA+IDAgPyB0aXRsZSA6IHVuZGVmaW5lZDtcbn1cblxuLyoqIFRydW5jYXRlIGB0ZXh0YCB0byBgbWF4YCBjaGFycywgcmVwb3J0aW5nIHdoZXRoZXIgaXQgd2FzIGN1dC4gKi9cbmZ1bmN0aW9uIHRydW5jYXRlKHRleHQ6IHN0cmluZywgbWF4OiBudW1iZXIpOiB7IGNvbnRlbnQ6IHN0cmluZzsgdHJ1bmNhdGVkOiBib29sZWFuIH0ge1xuICBpZiAodGV4dC5sZW5ndGggPD0gbWF4KSByZXR1cm4geyBjb250ZW50OiB0ZXh0LCB0cnVuY2F0ZWQ6IGZhbHNlIH07XG4gIHJldHVybiB7IGNvbnRlbnQ6IHRleHQuc2xpY2UoMCwgbWF4KSwgdHJ1bmNhdGVkOiB0cnVlIH07XG59XG5cbi8qKlxuICogRmV0Y2ggYSBzaW5nbGUgVVJMIGFuZCByZXR1cm4gaXRzIGNvbnRlbnQgaW4gdGhlIHJlcXVlc3RlZCBmb3JtYXQuXG4gKlxuICogVGhyb3dzIHdoZW4gdGhlIFVSTCBpcyBub3QgcGVybWl0dGVkIGJ5IHRoZSBhbGxvdy1saXN0ICh0aGUgdmlzaWJsZSxcbiAqIGNvcnJlY3RhYmxlIGZhaWx1cmUgdGhlIGRlc2lnbiBjYWxscyBmb3Igb24gdGhlIGZldGNoIHBhdGgpLiBUaGUgcmVxdWVzdCdzXG4gKiBgbWF4TGVuZ3RoYCBuYXJyb3dzIChuZXZlciB3aWRlbnMpIHRoZSBwbHVnaW4ncyBgZmV0Y2hNYXhMZW5ndGhgIGNhcC5cbiAqIGBzaWduYWxgIGNhbmNlbHMgdGhlIGluLWZsaWdodCBmZXRjaC5cbiAqL1xuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIHJ1bldlYkZldGNoKFxuICByZXF1ZXN0OiBXZWJGZXRjaFJlcXVlc3QsXG4gIGNvbmZpZzogUmVzb2x2ZWRXZWJTZWFyY2hDb25maWcsXG4gIHNpZ25hbD86IEFib3J0U2lnbmFsLFxuKTogUHJvbWlzZTxXZWJGZXRjaFJlc3VsdD4ge1xuICBhc3NlcnRVcmxBbGxvd2VkKHJlcXVlc3QudXJsLCBjb25maWcuYWxsb3dMaXN0KTtcbiAgY29uc3QgY2FwID0gTWF0aC5taW4ocmVxdWVzdC5tYXhMZW5ndGggPz8gY29uZmlnLmZldGNoTWF4TGVuZ3RoLCBjb25maWcuZmV0Y2hNYXhMZW5ndGgpO1xuXG4gIC8vIFRoZSByZXNwb25zZSBpcyBjYWNoZWQgYmVmb3JlIHRoZSBmb3JtYXQgcmVkdWN0aW9uLCBzbyB0aGUgc2FtZSBwYWdlIHJlYWRcbiAgLy8gYXMgdGV4dCBhbmQgYXMgaHRtbCBzaGFyZXMgb25lIG5ldHdvcmsgcm91bmQgdHJpcC5cbiAgY29uc3QgcGFnZSA9IGF3YWl0IGV4ZWN1dGVSZWFkKFxuICAgIFwicGFnZS1mZXRjaFwiLFxuICAgIHRvQ2FsbFNldHRpbmdzKHdlYkZldGNoRXhlY3V0ZURlZmF1bHRzLCBjb25maWcudGltZW91dE1zLCBbXCJ3ZWItc2VhcmNoXCIsIFwiZmV0Y2hcIiwgcmVxdWVzdC51cmxdKSxcbiAgICBhc3luYyAoZXhlY3V0ZVNpZ25hbCk6IFByb21pc2U8RmV0Y2hlZFBhZ2U+ID0+IHtcbiAgICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgZ290U2NyYXBpbmcoe1xuICAgICAgICB1cmw6IHJlcXVlc3QudXJsLFxuICAgICAgICAvLyBnb3QncyBvd24gdGltZW91dCBhYm9ydHMgdGhlIHNvY2tldCBhbmQgcmVwb3J0cyB3aGljaCBwaGFzZSB0aW1lZFxuICAgICAgICAvLyBvdXQ7IHRoZSBpbnRlcmNlcHRvciB0aW1lb3V0IGJvdW5kcyB0aGUgd2hvbGUgYXR0ZW1wdCBhcm91bmQgaXQuXG4gICAgICAgIHRpbWVvdXQ6IHsgcmVxdWVzdDogY29uZmlnLnRpbWVvdXRNcyB9LFxuICAgICAgICB0aHJvd0h0dHBFcnJvcnM6IGZhbHNlLFxuICAgICAgICBmb2xsb3dSZWRpcmVjdDogdHJ1ZSxcbiAgICAgICAgLi4uKGV4ZWN1dGVTaWduYWwgPyB7IHNpZ25hbDogZXhlY3V0ZVNpZ25hbCB9IDoge30pLFxuICAgICAgfSk7XG4gICAgICByZXR1cm4ge1xuICAgICAgICB1cmw6IHJlc3BvbnNlLnVybCA/PyByZXF1ZXN0LnVybCxcbiAgICAgICAgc3RhdHVzQ29kZTogcmVzcG9uc2Uuc3RhdHVzQ29kZSxcbiAgICAgICAgY29udGVudFR5cGU6IHJlc3BvbnNlLmhlYWRlcnNbXCJjb250ZW50LXR5cGVcIl0sXG4gICAgICAgIGJvZHk6IHR5cGVvZiByZXNwb25zZS5ib2R5ID09PSBcInN0cmluZ1wiID8gcmVzcG9uc2UuYm9keSA6IFN0cmluZyhyZXNwb25zZS5ib2R5ID8/IFwiXCIpLFxuICAgICAgfTtcbiAgICB9LFxuICAgIHNpZ25hbCxcbiAgKTtcblxuICBjb25zdCBpc0h0bWwgPSAhcGFnZS5jb250ZW50VHlwZSB8fCAvaHRtbHx4bWwvaS50ZXN0KHBhZ2UuY29udGVudFR5cGUpO1xuICBjb25zdCByYXdDb250ZW50ID0gcmVxdWVzdC5mb3JtYXQgPT09IFwiaHRtbFwiIHx8ICFpc0h0bWwgPyBwYWdlLmJvZHkgOiBodG1sVG9UZXh0KHBhZ2UuYm9keSk7XG4gIGNvbnN0IHsgY29udGVudCwgdHJ1bmNhdGVkIH0gPSB0cnVuY2F0ZShyYXdDb250ZW50LCBjYXApO1xuICBjb25zdCB0aXRsZSA9IGlzSHRtbCA/IGV4dHJhY3RUaXRsZShwYWdlLmJvZHkpIDogdW5kZWZpbmVkO1xuXG4gIGxvZ2dlci5kZWJ1ZyhcImZldGNoZWRcIiwge1xuICAgIHVybDogcGFnZS51cmwsXG4gICAgc3RhdHVzOiBwYWdlLnN0YXR1c0NvZGUsXG4gICAgYnl0ZXM6IHBhZ2UuYm9keS5sZW5ndGgsXG4gICAgcmV0dXJuZWQ6IGNvbnRlbnQubGVuZ3RoLFxuICAgIC4uLih0cnVuY2F0ZWQgPyB7IHRydW5jYXRlZDogdHJ1ZSB9IDoge30pLFxuICB9KTtcblxuICByZXR1cm4ge1xuICAgIHVybDogcGFnZS51cmwsXG4gICAgc3RhdHVzOiBwYWdlLnN0YXR1c0NvZGUsXG4gICAgLi4uKHBhZ2UuY29udGVudFR5cGUgPyB7IGNvbnRlbnRUeXBlOiBwYWdlLmNvbnRlbnRUeXBlIH0gOiB7fSksXG4gICAgLi4uKHRpdGxlID8geyB0aXRsZSB9IDoge30pLFxuICAgIGNvbnRlbnQsXG4gICAgdHJ1bmNhdGVkLFxuICB9O1xufVxuIl19
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning fetched HTML into the plain text a model can read.
|
|
3
|
+
*
|
|
4
|
+
* Both scraping paths need this: `runWebFetch`'s full-page read and
|
|
5
|
+
* `runScrapeSearch`'s title/snippet extraction. The entity table is shared so
|
|
6
|
+
* the two cannot drift apart.
|
|
7
|
+
*
|
|
8
|
+
* This is deliberately regex-based rather than a DOM parse. The inputs are
|
|
9
|
+
* whole pages fetched for summarization, not documents to be queried, so the
|
|
10
|
+
* cost of a real parser buys nothing - and a malformed page still degrades to
|
|
11
|
+
* readable text instead of throwing.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
/** Decode the HTML entities that survive tag-stripping. */
|
|
16
|
+
export declare function decodeHtmlEntities(text: string): string;
|
|
17
|
+
/**
|
|
18
|
+
* Strip tags and decode entities from a short HTML fragment, collapsing all
|
|
19
|
+
* whitespace to single spaces. For inline snippets (a search result title or
|
|
20
|
+
* summary), where layout carries no meaning.
|
|
21
|
+
*/
|
|
22
|
+
export declare function htmlFragmentToText(html: string): string;
|
|
23
|
+
/**
|
|
24
|
+
* Reduce a full HTML document to readable plain text: drop `<script>` /
|
|
25
|
+
* `<style>` / `<noscript>` blocks and comments, turn block-level tags into
|
|
26
|
+
* newlines, strip the remaining tags, decode entities, and collapse runs of
|
|
27
|
+
* blank lines / trailing spaces. Unlike {@link htmlFragmentToText}, this
|
|
28
|
+
* preserves line structure because paragraph breaks carry meaning in a page.
|
|
29
|
+
*/
|
|
30
|
+
export declare function htmlToText(html: string): string;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Turning fetched HTML into the plain text a model can read.
|
|
3
|
+
*
|
|
4
|
+
* Both scraping paths need this: `runWebFetch`'s full-page read and
|
|
5
|
+
* `runScrapeSearch`'s title/snippet extraction. The entity table is shared so
|
|
6
|
+
* the two cannot drift apart.
|
|
7
|
+
*
|
|
8
|
+
* This is deliberately regex-based rather than a DOM parse. The inputs are
|
|
9
|
+
* whole pages fetched for summarization, not documents to be queried, so the
|
|
10
|
+
* cost of a real parser buys nothing - and a malformed page still degrades to
|
|
11
|
+
* readable text instead of throwing.
|
|
12
|
+
*
|
|
13
|
+
* @module
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* The named entities that survive tag-stripping in practice, plus numeric
|
|
17
|
+
* escapes. Not a complete HTML entity table - anything rarer passes through
|
|
18
|
+
* as-is, which reads acceptably in model input.
|
|
19
|
+
*/
|
|
20
|
+
const NAMED_ENTITIES = {
|
|
21
|
+
" ": " ",
|
|
22
|
+
"&": "&",
|
|
23
|
+
"<": "<",
|
|
24
|
+
">": ">",
|
|
25
|
+
""": '"',
|
|
26
|
+
"'": "'",
|
|
27
|
+
"'": "'",
|
|
28
|
+
"'": "'",
|
|
29
|
+
};
|
|
30
|
+
const NAMED_ENTITY_REGEXP = new RegExp(Object.keys(NAMED_ENTITIES).join("|"), "g");
|
|
31
|
+
const NUMERIC_ENTITY_REGEXP = /&#(\d+);/g;
|
|
32
|
+
const TAG_REGEXP = /<[^>]+>/g;
|
|
33
|
+
/** Decode the HTML entities that survive tag-stripping. */
|
|
34
|
+
export function decodeHtmlEntities(text) {
|
|
35
|
+
return text
|
|
36
|
+
.replace(NAMED_ENTITY_REGEXP, (entity) => NAMED_ENTITIES[entity] ?? entity)
|
|
37
|
+
.replace(NUMERIC_ENTITY_REGEXP, (_, code) => String.fromCodePoint(Number(code)));
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Strip tags and decode entities from a short HTML fragment, collapsing all
|
|
41
|
+
* whitespace to single spaces. For inline snippets (a search result title or
|
|
42
|
+
* summary), where layout carries no meaning.
|
|
43
|
+
*/
|
|
44
|
+
export function htmlFragmentToText(html) {
|
|
45
|
+
return decodeHtmlEntities(html.replace(TAG_REGEXP, "")).replace(/\s+/g, " ").trim();
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Reduce a full HTML document to readable plain text: drop `<script>` /
|
|
49
|
+
* `<style>` / `<noscript>` blocks and comments, turn block-level tags into
|
|
50
|
+
* newlines, strip the remaining tags, decode entities, and collapse runs of
|
|
51
|
+
* blank lines / trailing spaces. Unlike {@link htmlFragmentToText}, this
|
|
52
|
+
* preserves line structure because paragraph breaks carry meaning in a page.
|
|
53
|
+
*/
|
|
54
|
+
export function htmlToText(html) {
|
|
55
|
+
return decodeHtmlEntities(html
|
|
56
|
+
.replace(/<!--[\s\S]*?-->/g, "")
|
|
57
|
+
.replace(/<(script|style|noscript)[^>]*>[\s\S]*?<\/\1>/gi, "")
|
|
58
|
+
.replace(/<\/(p|div|section|article|li|tr|h[1-6]|header|footer|br)>/gi, "\n")
|
|
59
|
+
.replace(/<br\s*\/?>/gi, "\n")
|
|
60
|
+
.replace(TAG_REGEXP, ""))
|
|
61
|
+
.replace(/[ \t]+\n/g, "\n")
|
|
62
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
63
|
+
.replace(/[ \t]{2,}/g, " ")
|
|
64
|
+
.trim();
|
|
65
|
+
}
|
|
66
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaHRtbC10ZXh0LmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL2h0bWwtdGV4dC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7OztHQWFHO0FBRUg7Ozs7R0FJRztBQUNILE1BQU0sY0FBYyxHQUEyQjtJQUM3QyxRQUFRLEVBQUUsR0FBRztJQUNiLE9BQU8sRUFBRSxHQUFHO0lBQ1osTUFBTSxFQUFFLEdBQUc7SUFDWCxNQUFNLEVBQUUsR0FBRztJQUNYLFFBQVEsRUFBRSxHQUFHO0lBQ2IsT0FBTyxFQUFFLEdBQUc7SUFDWixRQUFRLEVBQUUsR0FBRztJQUNiLFFBQVEsRUFBRSxHQUFHO0NBQ2QsQ0FBQztBQUVGLE1BQU0sbUJBQW1CLEdBQUcsSUFBSSxNQUFNLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxjQUFjLENBQUMsQ0FBQyxJQUFJLENBQUMsR0FBRyxDQUFDLEVBQUUsR0FBRyxDQUFDLENBQUM7QUFDbkYsTUFBTSxxQkFBcUIsR0FBRyxXQUFXLENBQUM7QUFDMUMsTUFBTSxVQUFVLEdBQUcsVUFBVSxDQUFDO0FBRTlCLDJEQUEyRDtBQUMzRCxNQUFNLFVBQVUsa0JBQWtCLENBQUMsSUFBWTtJQUM3QyxPQUFPLElBQUk7U0FDUixPQUFPLENBQUMsbUJBQW1CLEVBQUUsQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLGNBQWMsQ0FBQyxNQUFNLENBQUMsSUFBSSxNQUFNLENBQUM7U0FDMUUsT0FBTyxDQUFDLHFCQUFxQixFQUFFLENBQUMsQ0FBQyxFQUFFLElBQUksRUFBRSxFQUFFLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFDO0FBQ3JGLENBQUM7QUFFRDs7OztHQUlHO0FBQ0gsTUFBTSxVQUFVLGtCQUFrQixDQUFDLElBQVk7SUFDN0MsT0FBTyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLFVBQVUsRUFBRSxFQUFFLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxNQUFNLEVBQUUsR0FBRyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUM7QUFDdEYsQ0FBQztBQUVEOzs7Ozs7R0FNRztBQUNILE1BQU0sVUFBVSxVQUFVLENBQUMsSUFBWTtJQUNyQyxPQUFPLGtCQUFrQixDQUN2QixJQUFJO1NBQ0QsT0FBTyxDQUFDLGtCQUFrQixFQUFFLEVBQUUsQ0FBQztTQUMvQixPQUFPLENBQUMsZ0RBQWdELEVBQUUsRUFBRSxDQUFDO1NBQzdELE9BQU8sQ0FBQyw2REFBNkQsRUFBRSxJQUFJLENBQUM7U0FDNUUsT0FBTyxDQUFDLGNBQWMsRUFBRSxJQUFJLENBQUM7U0FDN0IsT0FBTyxDQUFDLFVBQVUsRUFBRSxFQUFFLENBQUMsQ0FDM0I7U0FDRSxPQUFPLENBQUMsV0FBVyxFQUFFLElBQUksQ0FBQztTQUMxQixPQUFPLENBQUMsU0FBUyxFQUFFLE1BQU0sQ0FBQztTQUMxQixPQUFPLENBQUMsWUFBWSxFQUFFLEdBQUcsQ0FBQztTQUMxQixJQUFJLEVBQUUsQ0FBQztBQUNaLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIFR1cm5pbmcgZmV0Y2hlZCBIVE1MIGludG8gdGhlIHBsYWluIHRleHQgYSBtb2RlbCBjYW4gcmVhZC5cbiAqXG4gKiBCb3RoIHNjcmFwaW5nIHBhdGhzIG5lZWQgdGhpczogYHJ1bldlYkZldGNoYCdzIGZ1bGwtcGFnZSByZWFkIGFuZFxuICogYHJ1blNjcmFwZVNlYXJjaGAncyB0aXRsZS9zbmlwcGV0IGV4dHJhY3Rpb24uIFRoZSBlbnRpdHkgdGFibGUgaXMgc2hhcmVkIHNvXG4gKiB0aGUgdHdvIGNhbm5vdCBkcmlmdCBhcGFydC5cbiAqXG4gKiBUaGlzIGlzIGRlbGliZXJhdGVseSByZWdleC1iYXNlZCByYXRoZXIgdGhhbiBhIERPTSBwYXJzZS4gVGhlIGlucHV0cyBhcmVcbiAqIHdob2xlIHBhZ2VzIGZldGNoZWQgZm9yIHN1bW1hcml6YXRpb24sIG5vdCBkb2N1bWVudHMgdG8gYmUgcXVlcmllZCwgc28gdGhlXG4gKiBjb3N0IG9mIGEgcmVhbCBwYXJzZXIgYnV5cyBub3RoaW5nIC0gYW5kIGEgbWFsZm9ybWVkIHBhZ2Ugc3RpbGwgZGVncmFkZXMgdG9cbiAqIHJlYWRhYmxlIHRleHQgaW5zdGVhZCBvZiB0aHJvd2luZy5cbiAqXG4gKiBAbW9kdWxlXG4gKi9cblxuLyoqXG4gKiBUaGUgbmFtZWQgZW50aXRpZXMgdGhhdCBzdXJ2aXZlIHRhZy1zdHJpcHBpbmcgaW4gcHJhY3RpY2UsIHBsdXMgbnVtZXJpY1xuICogZXNjYXBlcy4gTm90IGEgY29tcGxldGUgSFRNTCBlbnRpdHkgdGFibGUgLSBhbnl0aGluZyByYXJlciBwYXNzZXMgdGhyb3VnaFxuICogYXMtaXMsIHdoaWNoIHJlYWRzIGFjY2VwdGFibHkgaW4gbW9kZWwgaW5wdXQuXG4gKi9cbmNvbnN0IE5BTUVEX0VOVElUSUVTOiBSZWNvcmQ8c3RyaW5nLCBzdHJpbmc+ID0ge1xuICBcIiZuYnNwO1wiOiBcIiBcIixcbiAgXCImYW1wO1wiOiBcIiZcIixcbiAgXCImbHQ7XCI6IFwiPFwiLFxuICBcIiZndDtcIjogXCI+XCIsXG4gIFwiJnF1b3Q7XCI6ICdcIicsXG4gIFwiJiMzOTtcIjogXCInXCIsXG4gIFwiJiN4Mjc7XCI6IFwiJ1wiLFxuICBcIiZhcG9zO1wiOiBcIidcIixcbn07XG5cbmNvbnN0IE5BTUVEX0VOVElUWV9SRUdFWFAgPSBuZXcgUmVnRXhwKE9iamVjdC5rZXlzKE5BTUVEX0VOVElUSUVTKS5qb2luKFwifFwiKSwgXCJnXCIpO1xuY29uc3QgTlVNRVJJQ19FTlRJVFlfUkVHRVhQID0gLyYjKFxcZCspOy9nO1xuY29uc3QgVEFHX1JFR0VYUCA9IC88W14+XSs+L2c7XG5cbi8qKiBEZWNvZGUgdGhlIEhUTUwgZW50aXRpZXMgdGhhdCBzdXJ2aXZlIHRhZy1zdHJpcHBpbmcuICovXG5leHBvcnQgZnVuY3Rpb24gZGVjb2RlSHRtbEVudGl0aWVzKHRleHQ6IHN0cmluZyk6IHN0cmluZyB7XG4gIHJldHVybiB0ZXh0XG4gICAgLnJlcGxhY2UoTkFNRURfRU5USVRZX1JFR0VYUCwgKGVudGl0eSkgPT4gTkFNRURfRU5USVRJRVNbZW50aXR5XSA/PyBlbnRpdHkpXG4gICAgLnJlcGxhY2UoTlVNRVJJQ19FTlRJVFlfUkVHRVhQLCAoXywgY29kZSkgPT4gU3RyaW5nLmZyb21Db2RlUG9pbnQoTnVtYmVyKGNvZGUpKSk7XG59XG5cbi8qKlxuICogU3RyaXAgdGFncyBhbmQgZGVjb2RlIGVudGl0aWVzIGZyb20gYSBzaG9ydCBIVE1MIGZyYWdtZW50LCBjb2xsYXBzaW5nIGFsbFxuICogd2hpdGVzcGFjZSB0byBzaW5nbGUgc3BhY2VzLiBGb3IgaW5saW5lIHNuaXBwZXRzIChhIHNlYXJjaCByZXN1bHQgdGl0bGUgb3JcbiAqIHN1bW1hcnkpLCB3aGVyZSBsYXlvdXQgY2FycmllcyBubyBtZWFuaW5nLlxuICovXG5leHBvcnQgZnVuY3Rpb24gaHRtbEZyYWdtZW50VG9UZXh0KGh0bWw6IHN0cmluZyk6IHN0cmluZyB7XG4gIHJldHVybiBkZWNvZGVIdG1sRW50aXRpZXMoaHRtbC5yZXBsYWNlKFRBR19SRUdFWFAsIFwiXCIpKS5yZXBsYWNlKC9cXHMrL2csIFwiIFwiKS50cmltKCk7XG59XG5cbi8qKlxuICogUmVkdWNlIGEgZnVsbCBIVE1MIGRvY3VtZW50IHRvIHJlYWRhYmxlIHBsYWluIHRleHQ6IGRyb3AgYDxzY3JpcHQ+YCAvXG4gKiBgPHN0eWxlPmAgLyBgPG5vc2NyaXB0PmAgYmxvY2tzIGFuZCBjb21tZW50cywgdHVybiBibG9jay1sZXZlbCB0YWdzIGludG9cbiAqIG5ld2xpbmVzLCBzdHJpcCB0aGUgcmVtYWluaW5nIHRhZ3MsIGRlY29kZSBlbnRpdGllcywgYW5kIGNvbGxhcHNlIHJ1bnMgb2ZcbiAqIGJsYW5rIGxpbmVzIC8gdHJhaWxpbmcgc3BhY2VzLiBVbmxpa2Uge0BsaW5rIGh0bWxGcmFnbWVudFRvVGV4dH0sIHRoaXNcbiAqIHByZXNlcnZlcyBsaW5lIHN0cnVjdHVyZSBiZWNhdXNlIHBhcmFncmFwaCBicmVha3MgY2FycnkgbWVhbmluZyBpbiBhIHBhZ2UuXG4gKi9cbmV4cG9ydCBmdW5jdGlvbiBodG1sVG9UZXh0KGh0bWw6IHN0cmluZyk6IHN0cmluZyB7XG4gIHJldHVybiBkZWNvZGVIdG1sRW50aXRpZXMoXG4gICAgaHRtbFxuICAgICAgLnJlcGxhY2UoLzwhLS1bXFxzXFxTXSo/LS0+L2csIFwiXCIpXG4gICAgICAucmVwbGFjZSgvPChzY3JpcHR8c3R5bGV8bm9zY3JpcHQpW14+XSo+W1xcc1xcU10qPzxcXC9cXDE+L2dpLCBcIlwiKVxuICAgICAgLnJlcGxhY2UoLzxcXC8ocHxkaXZ8c2VjdGlvbnxhcnRpY2xlfGxpfHRyfGhbMS02XXxoZWFkZXJ8Zm9vdGVyfGJyKT4vZ2ksIFwiXFxuXCIpXG4gICAgICAucmVwbGFjZSgvPGJyXFxzKlxcLz8+L2dpLCBcIlxcblwiKVxuICAgICAgLnJlcGxhY2UoVEFHX1JFR0VYUCwgXCJcIiksXG4gIClcbiAgICAucmVwbGFjZSgvWyBcXHRdK1xcbi9nLCBcIlxcblwiKVxuICAgIC5yZXBsYWNlKC9cXG57Myx9L2csIFwiXFxuXFxuXCIpXG4gICAgLnJlcGxhY2UoL1sgXFx0XXsyLH0vZywgXCIgXCIpXG4gICAgLnRyaW0oKTtcbn1cbiJdfQ==
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AppKit plugin (registered name: `web-search`) that owns the resolved
|
|
3
|
+
* web-search runtime - the URL policy, result / length caps, timeout, and
|
|
4
|
+
* default approval gate the {@link webSearchTool} / {@link webFetchTool}
|
|
5
|
+
* read. Registering it resolves and logs the effective config (which URL
|
|
6
|
+
* policy is in force, the caps) so a misconfiguration is visible in the boot
|
|
7
|
+
* logs rather than on the first search, and installs the plugin's
|
|
8
|
+
* `execute()` as the runtime's executor so every outbound call picks up
|
|
9
|
+
* AppKit's cache / retry / timeout / telemetry chain.
|
|
10
|
+
*
|
|
11
|
+
* The plugin also implements AppKit's `ToolProvider`, so an AppKit agent gets
|
|
12
|
+
* `web_search` / `web_fetch` without going through Mastra; the Mastra tools in
|
|
13
|
+
* `tool.ts` are the other half and share the same runtime.
|
|
14
|
+
*
|
|
15
|
+
* @module
|
|
16
|
+
*/
|
|
17
|
+
import { Plugin, ResourceType, type ResourceRequirement } from "@databricks/appkit";
|
|
18
|
+
import { type AgentToolDefinition, type ToolProvider } from "@databricks/appkit/beta";
|
|
19
|
+
import { type WebSearchPluginConfig } from "./config.js";
|
|
20
|
+
import type { WebFetchRequest, WebFetchResult, WebSearchRequest, WebSearchResult } from "./schema.js";
|
|
21
|
+
/**
|
|
22
|
+
* AppKit plugin that resolves and holds the web-search runtime config used
|
|
23
|
+
* by the `web_search` / `web_fetch` tools.
|
|
24
|
+
*
|
|
25
|
+
* @example
|
|
26
|
+
* ```ts
|
|
27
|
+
* import { createApp, server } from "@databricks/appkit";
|
|
28
|
+
* import { plugin as webSearchPlugin } from "@dbx-tools/appkit-web-search";
|
|
29
|
+
*
|
|
30
|
+
* await createApp({
|
|
31
|
+
* plugins: [
|
|
32
|
+
* server(),
|
|
33
|
+
* webSearchPlugin.webSearch({
|
|
34
|
+
* model: "gemini",
|
|
35
|
+
* urlPolicy: "allowlist",
|
|
36
|
+
* allowedUrls: ["*.databricks.com"],
|
|
37
|
+
* }),
|
|
38
|
+
* ],
|
|
39
|
+
* });
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
export declare class WebSearchPlugin extends Plugin<WebSearchPluginConfig> implements ToolProvider {
|
|
43
|
+
static manifest: {
|
|
44
|
+
name: "web-search";
|
|
45
|
+
displayName: string;
|
|
46
|
+
description: string;
|
|
47
|
+
stability: "beta";
|
|
48
|
+
resources: {
|
|
49
|
+
required: never[];
|
|
50
|
+
optional: {
|
|
51
|
+
type: ResourceType.SERVING_ENDPOINT;
|
|
52
|
+
alias: string;
|
|
53
|
+
resourceKey: string;
|
|
54
|
+
description: string;
|
|
55
|
+
permission: "CAN_QUERY";
|
|
56
|
+
fields: {
|
|
57
|
+
name: {
|
|
58
|
+
env: string;
|
|
59
|
+
description: string;
|
|
60
|
+
discovery: {
|
|
61
|
+
type: "cli";
|
|
62
|
+
cliCommand: string;
|
|
63
|
+
selectField: string;
|
|
64
|
+
};
|
|
65
|
+
};
|
|
66
|
+
};
|
|
67
|
+
}[];
|
|
68
|
+
};
|
|
69
|
+
config: {
|
|
70
|
+
schema: import("json-schema").JSONSchema7;
|
|
71
|
+
};
|
|
72
|
+
};
|
|
73
|
+
/**
|
|
74
|
+
* Promote the serving endpoint to a required resource once a deployment
|
|
75
|
+
* pins one, through plugin config or either environment name. Left optional
|
|
76
|
+
* otherwise, because the plugin picks a web-search-capable endpoint out of
|
|
77
|
+
* the live catalogue on its own.
|
|
78
|
+
*/
|
|
79
|
+
static getResourceRequirements(config: WebSearchPluginConfig): ResourceRequirement[];
|
|
80
|
+
/**
|
|
81
|
+
* The tools this plugin offers to an AppKit agent. Both are reads and both
|
|
82
|
+
* run under the caller's identity.
|
|
83
|
+
*
|
|
84
|
+
* Neither is `autoInheritable`. `web_fetch` reaches a URL the model chose,
|
|
85
|
+
* which is reachable from inside the workspace network, so it must only
|
|
86
|
+
* appear in an agent that asked for it and accepted the URL policy that
|
|
87
|
+
* comes with it. `web_search` is safer but still spends serving tokens on
|
|
88
|
+
* every call, so it is opt-in for cost rather than for safety.
|
|
89
|
+
*
|
|
90
|
+
* `execute` re-parses its arguments with the local schema: AppKit validates
|
|
91
|
+
* against the same schema first, but re-parsing is what gives the body typed
|
|
92
|
+
* arguments instead of `unknown`.
|
|
93
|
+
*/
|
|
94
|
+
private readonly tools;
|
|
95
|
+
/**
|
|
96
|
+
* Prime the shared runtime from this plugin's config (over env), route the
|
|
97
|
+
* tools' outbound calls through this plugin's interceptor chain, and log the
|
|
98
|
+
* effective policy so an active allow-list / caps are obvious at boot.
|
|
99
|
+
*/
|
|
100
|
+
setup(): Promise<void>;
|
|
101
|
+
/**
|
|
102
|
+
* Drop the shared runtime so a restarted app re-resolves config and does not
|
|
103
|
+
* keep calling through a torn-down plugin's `execute()`. Bounded and
|
|
104
|
+
* idempotent: there is no connection to drain, only the memo to clear.
|
|
105
|
+
*/
|
|
106
|
+
shutdown(): Promise<void>;
|
|
107
|
+
/**
|
|
108
|
+
* Abort in-flight work. AppKit's graceful shutdown only invokes this hook -
|
|
109
|
+
* it never calls {@link shutdown} - so the runtime memo is dropped from here
|
|
110
|
+
* to keep a restarted app from calling through a torn-down `execute()`. The
|
|
111
|
+
* teardown is synchronous and idempotent, so the un-awaited call costs
|
|
112
|
+
* nothing.
|
|
113
|
+
*/
|
|
114
|
+
abortActiveOperations(): void;
|
|
115
|
+
/** AppKit `ToolProvider`: the tool definitions offered to an agent. */
|
|
116
|
+
getAgentTools(): AgentToolDefinition[];
|
|
117
|
+
/**
|
|
118
|
+
* AppKit `ToolProvider`: run one tool call. Arguments are validated against
|
|
119
|
+
* the tool's schema first, and a validation failure comes back as an
|
|
120
|
+
* LLM-friendly string so the model can correct itself on the next turn.
|
|
121
|
+
*/
|
|
122
|
+
executeAgentTool(name: string, args: unknown, signal?: AbortSignal): Promise<unknown>;
|
|
123
|
+
exports(): {
|
|
124
|
+
/**
|
|
125
|
+
* Run a web search directly (bypassing the agent tool). Resolves the
|
|
126
|
+
* OBO client from the active execution context and reads the shared
|
|
127
|
+
* runtime config primed at setup.
|
|
128
|
+
*/
|
|
129
|
+
search: (request: WebSearchRequest, signal?: AbortSignal) => Promise<WebSearchResult>;
|
|
130
|
+
/**
|
|
131
|
+
* Fetch one URL directly (bypassing the agent tool). Enforces the
|
|
132
|
+
* configured URL policy. Reads the shared runtime config.
|
|
133
|
+
*/
|
|
134
|
+
fetch: (request: WebFetchRequest, signal?: AbortSignal) => Promise<WebFetchResult>;
|
|
135
|
+
};
|
|
136
|
+
private search;
|
|
137
|
+
private fetch;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Register the web-search runtime with AppKit.
|
|
141
|
+
*
|
|
142
|
+
* @example
|
|
143
|
+
* ```ts
|
|
144
|
+
* import { createApp, server } from "@databricks/appkit";
|
|
145
|
+
* import { plugin as webSearchPlugin, tool as webTool } from "@dbx-tools/appkit-web-search";
|
|
146
|
+
* import { agents, plugin as mastraPlugin } from "@dbx-tools/appkit-mastra";
|
|
147
|
+
*
|
|
148
|
+
* const researcher = agents.createAgent({
|
|
149
|
+
* instructions: "Research questions with web_search, then read sources with web_fetch.",
|
|
150
|
+
* tools: () => ({ web_search: webTool.webSearchTool(), web_fetch: webTool.webFetchTool() }),
|
|
151
|
+
* });
|
|
152
|
+
*
|
|
153
|
+
* await createApp({
|
|
154
|
+
* plugins: [
|
|
155
|
+
* server(),
|
|
156
|
+
* webSearchPlugin.webSearch({ model: "gemini" }),
|
|
157
|
+
* mastraPlugin.mastra({ agents: researcher }),
|
|
158
|
+
* ],
|
|
159
|
+
* });
|
|
160
|
+
* ```
|
|
161
|
+
*/
|
|
162
|
+
export declare const webSearch: import("@databricks/appkit").ToPlugin<typeof WebSearchPlugin, WebSearchPluginConfig, "web-search">;
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AppKit plugin (registered name: `web-search`) that owns the resolved
|
|
3
|
+
* web-search runtime - the URL policy, result / length caps, timeout, and
|
|
4
|
+
* default approval gate the {@link webSearchTool} / {@link webFetchTool}
|
|
5
|
+
* read. Registering it resolves and logs the effective config (which URL
|
|
6
|
+
* policy is in force, the caps) so a misconfiguration is visible in the boot
|
|
7
|
+
* logs rather than on the first search, and installs the plugin's
|
|
8
|
+
* `execute()` as the runtime's executor so every outbound call picks up
|
|
9
|
+
* AppKit's cache / retry / timeout / telemetry chain.
|
|
10
|
+
*
|
|
11
|
+
* The plugin also implements AppKit's `ToolProvider`, so an AppKit agent gets
|
|
12
|
+
* `web_search` / `web_fetch` without going through Mastra; the Mastra tools in
|
|
13
|
+
* `tool.ts` are the other half and share the same runtime.
|
|
14
|
+
*
|
|
15
|
+
* @module
|
|
16
|
+
*/
|
|
17
|
+
import { Plugin, ResourceType, toPlugin, } from "@databricks/appkit";
|
|
18
|
+
import { defineTool, executeFromRegistry, toolsFromRegistry, } from "@databricks/appkit/beta";
|
|
19
|
+
import { log, string } from "@dbx-tools/shared-core";
|
|
20
|
+
import { MODEL_ENV, SERVING_ENDPOINT_ENV, WEB_SEARCH_CONFIG_SCHEMA, } from "./config.js";
|
|
21
|
+
import { runWebFetch } from "./fetch.js";
|
|
22
|
+
import { getWebSearchRuntime, resetWebSearchRuntime, setWebSearchExecutor } from "./runtime.js";
|
|
23
|
+
import { webFetchRequestSchema, webSearchRequestSchema, WEB_FETCH_TOOL_DESCRIPTION, WEB_SEARCH_TOOL_DESCRIPTION, } from "./schema.js";
|
|
24
|
+
import { resolveWebSearchContext, runWebSearch } from "./search.js";
|
|
25
|
+
const logger = log.logger("web-search");
|
|
26
|
+
/**
|
|
27
|
+
* The Model Serving endpoint the native web-search tool runs on. Declared
|
|
28
|
+
* optional because the plugin resolves a web-search-capable endpoint from the
|
|
29
|
+
* live catalogue when nothing is pinned; {@link WebSearchPlugin.getResourceRequirements}
|
|
30
|
+
* promotes it to required once a deployment names one. `CAN_QUERY` is the
|
|
31
|
+
* weakest permission that can invoke an endpoint.
|
|
32
|
+
*/
|
|
33
|
+
const SERVING_ENDPOINT_RESOURCE = {
|
|
34
|
+
type: ResourceType.SERVING_ENDPOINT,
|
|
35
|
+
alias: "Web Search Endpoint",
|
|
36
|
+
resourceKey: "web-search-endpoint",
|
|
37
|
+
description: "Model Serving endpoint running the native web-search tool. Optional: resolved from the " +
|
|
38
|
+
"workspace catalogue (Gemini, then GPT) when no endpoint is pinned.",
|
|
39
|
+
permission: "CAN_QUERY",
|
|
40
|
+
fields: {
|
|
41
|
+
name: {
|
|
42
|
+
env: SERVING_ENDPOINT_ENV,
|
|
43
|
+
description: `Serving endpoint name for web search. ${MODEL_ENV} overrides it.`,
|
|
44
|
+
discovery: {
|
|
45
|
+
type: "cli",
|
|
46
|
+
cliCommand: "databricks serving-endpoints list --profile <PROFILE> --output json",
|
|
47
|
+
selectField: ".name",
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* AppKit plugin that resolves and holds the web-search runtime config used
|
|
54
|
+
* by the `web_search` / `web_fetch` tools.
|
|
55
|
+
*
|
|
56
|
+
* @example
|
|
57
|
+
* ```ts
|
|
58
|
+
* import { createApp, server } from "@databricks/appkit";
|
|
59
|
+
* import { plugin as webSearchPlugin } from "@dbx-tools/appkit-web-search";
|
|
60
|
+
*
|
|
61
|
+
* await createApp({
|
|
62
|
+
* plugins: [
|
|
63
|
+
* server(),
|
|
64
|
+
* webSearchPlugin.webSearch({
|
|
65
|
+
* model: "gemini",
|
|
66
|
+
* urlPolicy: "allowlist",
|
|
67
|
+
* allowedUrls: ["*.databricks.com"],
|
|
68
|
+
* }),
|
|
69
|
+
* ],
|
|
70
|
+
* });
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
export class WebSearchPlugin extends Plugin {
|
|
74
|
+
static manifest = {
|
|
75
|
+
name: "web-search",
|
|
76
|
+
displayName: "Web Search",
|
|
77
|
+
description: "Searches the web through the Databricks native web-search tool on its own " +
|
|
78
|
+
"web-capable serving endpoint, and fetches pages via got-scraping, behind an " +
|
|
79
|
+
"optional URL allow-list and per-tool approval gating.",
|
|
80
|
+
stability: "beta",
|
|
81
|
+
resources: {
|
|
82
|
+
required: [],
|
|
83
|
+
optional: [SERVING_ENDPOINT_RESOURCE],
|
|
84
|
+
},
|
|
85
|
+
config: { schema: WEB_SEARCH_CONFIG_SCHEMA },
|
|
86
|
+
};
|
|
87
|
+
/**
|
|
88
|
+
* Promote the serving endpoint to a required resource once a deployment
|
|
89
|
+
* pins one, through plugin config or either environment name. Left optional
|
|
90
|
+
* otherwise, because the plugin picks a web-search-capable endpoint out of
|
|
91
|
+
* the live catalogue on its own.
|
|
92
|
+
*/
|
|
93
|
+
static getResourceRequirements(config) {
|
|
94
|
+
const pinned = string.trimToNull(config.model) ??
|
|
95
|
+
string.trimToNull(process.env[MODEL_ENV]) ??
|
|
96
|
+
string.trimToNull(process.env[SERVING_ENDPOINT_ENV]);
|
|
97
|
+
return pinned === null ? [] : [{ ...SERVING_ENDPOINT_RESOURCE, required: true }];
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The tools this plugin offers to an AppKit agent. Both are reads and both
|
|
101
|
+
* run under the caller's identity.
|
|
102
|
+
*
|
|
103
|
+
* Neither is `autoInheritable`. `web_fetch` reaches a URL the model chose,
|
|
104
|
+
* which is reachable from inside the workspace network, so it must only
|
|
105
|
+
* appear in an agent that asked for it and accepted the URL policy that
|
|
106
|
+
* comes with it. `web_search` is safer but still spends serving tokens on
|
|
107
|
+
* every call, so it is opt-in for cost rather than for safety.
|
|
108
|
+
*
|
|
109
|
+
* `execute` re-parses its arguments with the local schema: AppKit validates
|
|
110
|
+
* against the same schema first, but re-parsing is what gives the body typed
|
|
111
|
+
* arguments instead of `unknown`.
|
|
112
|
+
*/
|
|
113
|
+
tools = {
|
|
114
|
+
web_search: defineTool({
|
|
115
|
+
description: WEB_SEARCH_TOOL_DESCRIPTION,
|
|
116
|
+
schema: webSearchRequestSchema,
|
|
117
|
+
annotations: { effect: "read", requiresUserContext: true },
|
|
118
|
+
autoInheritable: false,
|
|
119
|
+
execute: async (args, signal) => this.search(webSearchRequestSchema.parse(args), signal),
|
|
120
|
+
}),
|
|
121
|
+
web_fetch: defineTool({
|
|
122
|
+
description: WEB_FETCH_TOOL_DESCRIPTION,
|
|
123
|
+
schema: webFetchRequestSchema,
|
|
124
|
+
annotations: { effect: "read", requiresUserContext: true },
|
|
125
|
+
autoInheritable: false,
|
|
126
|
+
execute: async (args, signal) => this.fetch(webFetchRequestSchema.parse(args), signal),
|
|
127
|
+
}),
|
|
128
|
+
};
|
|
129
|
+
/**
|
|
130
|
+
* Prime the shared runtime from this plugin's config (over env), route the
|
|
131
|
+
* tools' outbound calls through this plugin's interceptor chain, and log the
|
|
132
|
+
* effective policy so an active allow-list / caps are obvious at boot.
|
|
133
|
+
*/
|
|
134
|
+
async setup() {
|
|
135
|
+
const { config } = getWebSearchRuntime(this.config);
|
|
136
|
+
setWebSearchExecutor((fn, settings) => this.execute(fn, settings));
|
|
137
|
+
logger.info("ready", {
|
|
138
|
+
model: config.model ?? `fallbacks:[${config.modelFallbacks.join(", ")}]`,
|
|
139
|
+
modelSource: config.modelSource,
|
|
140
|
+
urlPolicy: config.urlPolicy,
|
|
141
|
+
...(config.allowList.restricted ? { allowedUrls: config.allowList.patterns } : {}),
|
|
142
|
+
maxCitations: config.maxCitations,
|
|
143
|
+
fetchMaxLength: config.fetchMaxLength,
|
|
144
|
+
approval: config.approval.mode,
|
|
145
|
+
scrapeFallback: config.scrapeFallback,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Drop the shared runtime so a restarted app re-resolves config and does not
|
|
150
|
+
* keep calling through a torn-down plugin's `execute()`. Bounded and
|
|
151
|
+
* idempotent: there is no connection to drain, only the memo to clear.
|
|
152
|
+
*/
|
|
153
|
+
async shutdown() {
|
|
154
|
+
resetWebSearchRuntime();
|
|
155
|
+
}
|
|
156
|
+
/**
|
|
157
|
+
* Abort in-flight work. AppKit's graceful shutdown only invokes this hook -
|
|
158
|
+
* it never calls {@link shutdown} - so the runtime memo is dropped from here
|
|
159
|
+
* to keep a restarted app from calling through a torn-down `execute()`. The
|
|
160
|
+
* teardown is synchronous and idempotent, so the un-awaited call costs
|
|
161
|
+
* nothing.
|
|
162
|
+
*/
|
|
163
|
+
abortActiveOperations() {
|
|
164
|
+
super.abortActiveOperations();
|
|
165
|
+
void this.shutdown();
|
|
166
|
+
}
|
|
167
|
+
/** AppKit `ToolProvider`: the tool definitions offered to an agent. */
|
|
168
|
+
getAgentTools() {
|
|
169
|
+
return toolsFromRegistry(this.tools);
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* AppKit `ToolProvider`: run one tool call. Arguments are validated against
|
|
173
|
+
* the tool's schema first, and a validation failure comes back as an
|
|
174
|
+
* LLM-friendly string so the model can correct itself on the next turn.
|
|
175
|
+
*/
|
|
176
|
+
async executeAgentTool(name, args, signal) {
|
|
177
|
+
return executeFromRegistry(this.tools, name, args, signal);
|
|
178
|
+
}
|
|
179
|
+
exports() {
|
|
180
|
+
return {
|
|
181
|
+
/**
|
|
182
|
+
* Run a web search directly (bypassing the agent tool). Resolves the
|
|
183
|
+
* OBO client from the active execution context and reads the shared
|
|
184
|
+
* runtime config primed at setup.
|
|
185
|
+
*/
|
|
186
|
+
search: (request, signal) => this.search(request, signal),
|
|
187
|
+
/**
|
|
188
|
+
* Fetch one URL directly (bypassing the agent tool). Enforces the
|
|
189
|
+
* configured URL policy. Reads the shared runtime config.
|
|
190
|
+
*/
|
|
191
|
+
fetch: (request, signal) => this.fetch(request, signal),
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
async search(request, signal) {
|
|
195
|
+
return runWebSearch(request, getWebSearchRuntime().config, await resolveWebSearchContext(), signal);
|
|
196
|
+
}
|
|
197
|
+
async fetch(request, signal) {
|
|
198
|
+
return runWebFetch(request, getWebSearchRuntime().config, signal);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Register the web-search runtime with AppKit.
|
|
203
|
+
*
|
|
204
|
+
* @example
|
|
205
|
+
* ```ts
|
|
206
|
+
* import { createApp, server } from "@databricks/appkit";
|
|
207
|
+
* import { plugin as webSearchPlugin, tool as webTool } from "@dbx-tools/appkit-web-search";
|
|
208
|
+
* import { agents, plugin as mastraPlugin } from "@dbx-tools/appkit-mastra";
|
|
209
|
+
*
|
|
210
|
+
* const researcher = agents.createAgent({
|
|
211
|
+
* instructions: "Research questions with web_search, then read sources with web_fetch.",
|
|
212
|
+
* tools: () => ({ web_search: webTool.webSearchTool(), web_fetch: webTool.webFetchTool() }),
|
|
213
|
+
* });
|
|
214
|
+
*
|
|
215
|
+
* await createApp({
|
|
216
|
+
* plugins: [
|
|
217
|
+
* server(),
|
|
218
|
+
* webSearchPlugin.webSearch({ model: "gemini" }),
|
|
219
|
+
* mastraPlugin.mastra({ agents: researcher }),
|
|
220
|
+
* ],
|
|
221
|
+
* });
|
|
222
|
+
* ```
|
|
223
|
+
*/
|
|
224
|
+
export const webSearch = toPlugin(WebSearchPlugin);
|
|
225
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicGx1Z2luLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3BsdWdpbi50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQTs7Ozs7Ozs7Ozs7Ozs7O0dBZUc7QUFFSCxPQUFPLEVBQ0wsTUFBTSxFQUNOLFlBQVksRUFDWixRQUFRLEdBR1QsTUFBTSxvQkFBb0IsQ0FBQztBQUM1QixPQUFPLEVBQ0wsVUFBVSxFQUNWLG1CQUFtQixFQUNuQixpQkFBaUIsR0FJbEIsTUFBTSx5QkFBeUIsQ0FBQztBQUNqQyxPQUFPLEVBQUUsR0FBRyxFQUFFLE1BQU0sRUFBRSxNQUFNLHdCQUF3QixDQUFDO0FBQ3JELE9BQU8sRUFDTCxTQUFTLEVBQ1Qsb0JBQW9CLEVBQ3BCLHdCQUF3QixHQUV6QixNQUFNLFVBQVUsQ0FBQztBQUNsQixPQUFPLEVBQUUsV0FBVyxFQUFFLE1BQU0sU0FBUyxDQUFDO0FBQ3RDLE9BQU8sRUFBRSxtQkFBbUIsRUFBRSxxQkFBcUIsRUFBRSxvQkFBb0IsRUFBRSxNQUFNLFdBQVcsQ0FBQztBQUM3RixPQUFPLEVBQ0wscUJBQXFCLEVBQ3JCLHNCQUFzQixFQUN0QiwwQkFBMEIsRUFDMUIsMkJBQTJCLEdBQzVCLE1BQU0sVUFBVSxDQUFDO0FBRWxCLE9BQU8sRUFBRSx1QkFBdUIsRUFBRSxZQUFZLEVBQUUsTUFBTSxVQUFVLENBQUM7QUFFakUsTUFBTSxNQUFNLEdBQUcsR0FBRyxDQUFDLE1BQU0sQ0FBQyxZQUFZLENBQUMsQ0FBQztBQUV4Qzs7Ozs7O0dBTUc7QUFDSCxNQUFNLHlCQUF5QixHQUFHO0lBQ2hDLElBQUksRUFBRSxZQUFZLENBQUMsZ0JBQWdCO0lBQ25DLEtBQUssRUFBRSxxQkFBcUI7SUFDNUIsV0FBVyxFQUFFLHFCQUFxQjtJQUNsQyxXQUFXLEVBQ1QseUZBQXlGO1FBQ3pGLG9FQUFvRTtJQUN0RSxVQUFVLEVBQUUsV0FBVztJQUN2QixNQUFNLEVBQUU7UUFDTixJQUFJLEVBQUU7WUFDSixHQUFHLEVBQUUsb0JBQW9CO1lBQ3pCLFdBQVcsRUFBRSx5Q0FBeUMsU0FBUyxnQkFBZ0I7WUFDL0UsU0FBUyxFQUFFO2dCQUNULElBQUksRUFBRSxLQUFLO2dCQUNYLFVBQVUsRUFBRSxxRUFBcUU7Z0JBQ2pGLFdBQVcsRUFBRSxPQUFPO2FBQ3JCO1NBQ0Y7S0FDRjtDQUM4QyxDQUFDO0FBRWxEOzs7Ozs7Ozs7Ozs7Ozs7Ozs7OztHQW9CRztBQUNILE1BQU0sT0FBTyxlQUFnQixTQUFRLE1BQTZCO0lBQ2hFLE1BQU0sQ0FBQyxRQUFRLEdBQUc7UUFDaEIsSUFBSSxFQUFFLFlBQVk7UUFDbEIsV0FBVyxFQUFFLFlBQVk7UUFDekIsV0FBVyxFQUNULDRFQUE0RTtZQUM1RSw4RUFBOEU7WUFDOUUsdURBQXVEO1FBQ3pELFNBQVMsRUFBRSxNQUFNO1FBQ2pCLFNBQVMsRUFBRTtZQUNULFFBQVEsRUFBRSxFQUFFO1lBQ1osUUFBUSxFQUFFLENBQUMseUJBQXlCLENBQUM7U0FDdEM7UUFDRCxNQUFNLEVBQUUsRUFBRSxNQUFNLEVBQUUsd0JBQXdCLEVBQUU7S0FDTixDQUFDO0lBRXpDOzs7OztPQUtHO0lBQ0gsTUFBTSxDQUFDLHVCQUF1QixDQUFDLE1BQTZCO1FBQzFELE1BQU0sTUFBTSxHQUNWLE1BQU0sQ0FBQyxVQUFVLENBQUMsTUFBTSxDQUFDLEtBQUssQ0FBQztZQUMvQixNQUFNLENBQUMsVUFBVSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxDQUFDLENBQUM7WUFDekMsTUFBTSxDQUFDLFVBQVUsQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLG9CQUFvQixDQUFDLENBQUMsQ0FBQztRQUN2RCxPQUFPLE1BQU0sS0FBSyxJQUFJLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxFQUFFLEdBQUcseUJBQXlCLEVBQUUsUUFBUSxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7SUFDbkYsQ0FBQztJQUVEOzs7Ozs7Ozs7Ozs7O09BYUc7SUFDYyxLQUFLLEdBQWlCO1FBQ3JDLFVBQVUsRUFBRSxVQUFVLENBQUM7WUFDckIsV0FBVyxFQUFFLDJCQUEyQjtZQUN4QyxNQUFNLEVBQUUsc0JBQXNCO1lBQzlCLFdBQVcsRUFBRSxFQUFFLE1BQU0sRUFBRSxNQUFNLEVBQUUsbUJBQW1CLEVBQUUsSUFBSSxFQUFFO1lBQzFELGVBQWUsRUFBRSxLQUFLO1lBQ3RCLE9BQU8sRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFFLE1BQU0sRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxzQkFBc0IsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLEVBQUUsTUFBTSxDQUFDO1NBQ3pGLENBQUM7UUFDRixTQUFTLEVBQUUsVUFBVSxDQUFDO1lBQ3BCLFdBQVcsRUFBRSwwQkFBMEI7WUFDdkMsTUFBTSxFQUFFLHFCQUFxQjtZQUM3QixXQUFXLEVBQUUsRUFBRSxNQUFNLEVBQUUsTUFBTSxFQUFFLG1CQUFtQixFQUFFLElBQUksRUFBRTtZQUMxRCxlQUFlLEVBQUUsS0FBSztZQUN0QixPQUFPLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxNQUFNLEVBQUUsRUFBRSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMscUJBQXFCLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxFQUFFLE1BQU0sQ0FBQztTQUN2RixDQUFDO0tBQ0gsQ0FBQztJQUVGOzs7O09BSUc7SUFDTSxLQUFLLENBQUMsS0FBSztRQUNsQixNQUFNLEVBQUUsTUFBTSxFQUFFLEdBQUcsbUJBQW1CLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQ3BELG9CQUFvQixDQUFDLENBQUMsRUFBRSxFQUFFLFFBQVEsRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxFQUFFLEVBQUUsUUFBUSxDQUFDLENBQUMsQ0FBQztRQUNuRSxNQUFNLENBQUMsSUFBSSxDQUFDLE9BQU8sRUFBRTtZQUNuQixLQUFLLEVBQUUsTUFBTSxDQUFDLEtBQUssSUFBSSxjQUFjLE1BQU0sQ0FBQyxjQUFjLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHO1lBQ3hFLFdBQVcsRUFBRSxNQUFNLENBQUMsV0FBVztZQUMvQixTQUFTLEVBQUUsTUFBTSxDQUFDLFNBQVM7WUFDM0IsR0FBRyxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsVUFBVSxDQUFDLENBQUMsQ0FBQyxFQUFFLFdBQVcsRUFBRSxNQUFNLENBQUMsU0FBUyxDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUMsQ0FBQyxFQUFFLENBQUM7WUFDbEYsWUFBWSxFQUFFLE1BQU0sQ0FBQyxZQUFZO1lBQ2pDLGNBQWMsRUFBRSxNQUFNLENBQUMsY0FBYztZQUNyQyxRQUFRLEVBQUUsTUFBTSxDQUFDLFFBQVEsQ0FBQyxJQUFJO1lBQzlCLGNBQWMsRUFBRSxNQUFNLENBQUMsY0FBYztTQUN0QyxDQUFDLENBQUM7SUFDTCxDQUFDO0lBRUQ7Ozs7T0FJRztJQUNILEtBQUssQ0FBQyxRQUFRO1FBQ1oscUJBQXFCLEVBQUUsQ0FBQztJQUMxQixDQUFDO0lBRUQ7Ozs7OztPQU1HO0lBQ00scUJBQXFCO1FBQzVCLEtBQUssQ0FBQyxxQkFBcUIsRUFBRSxDQUFDO1FBQzlCLEtBQUssSUFBSSxDQUFDLFFBQVEsRUFBRSxDQUFDO0lBQ3ZCLENBQUM7SUFFRCx1RUFBdUU7SUFDdkUsYUFBYTtRQUNYLE9BQU8saUJBQWlCLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3ZDLENBQUM7SUFFRDs7OztPQUlHO0lBQ0gsS0FBSyxDQUFDLGdCQUFnQixDQUFDLElBQVksRUFBRSxJQUFhLEVBQUUsTUFBb0I7UUFDdEUsT0FBTyxtQkFBbUIsQ0FBQyxJQUFJLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxJQUFJLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDN0QsQ0FBQztJQUVRLE9BQU87UUFDZCxPQUFPO1lBQ0w7Ozs7ZUFJRztZQUNILE1BQU0sRUFBRSxDQUFDLE9BQXlCLEVBQUUsTUFBb0IsRUFBNEIsRUFBRSxDQUNwRixJQUFJLENBQUMsTUFBTSxDQUFDLE9BQU8sRUFBRSxNQUFNLENBQUM7WUFDOUI7OztlQUdHO1lBQ0gsS0FBSyxFQUFFLENBQUMsT0FBd0IsRUFBRSxNQUFvQixFQUEyQixFQUFFLENBQ2pGLElBQUksQ0FBQyxLQUFLLENBQUMsT0FBTyxFQUFFLE1BQU0sQ0FBQztTQUM5QixDQUFDO0lBQ0osQ0FBQztJQUVPLEtBQUssQ0FBQyxNQUFNLENBQUMsT0FBeUIsRUFBRSxNQUFvQjtRQUNsRSxPQUFPLFlBQVksQ0FDakIsT0FBTyxFQUNQLG1CQUFtQixFQUFFLENBQUMsTUFBTSxFQUM1QixNQUFNLHVCQUF1QixFQUFFLEVBQy9CLE1BQU0sQ0FDUCxDQUFDO0lBQ0osQ0FBQztJQUVPLEtBQUssQ0FBQyxLQUFLLENBQUMsT0FBd0IsRUFBRSxNQUFvQjtRQUNoRSxPQUFPLFdBQVcsQ0FBQyxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsQ0FBQyxNQUFNLEVBQUUsTUFBTSxDQUFDLENBQUM7SUFDcEUsQ0FBQzs7QUFHSDs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7OztHQXNCRztBQUNILE1BQU0sQ0FBQyxNQUFNLFNBQVMsR0FBRyxRQUFRLENBQUMsZUFBZSxDQUFDLENBQUMiLCJzb3VyY2VzQ29udGVudCI6WyIvKipcbiAqIEFwcEtpdCBwbHVnaW4gKHJlZ2lzdGVyZWQgbmFtZTogYHdlYi1zZWFyY2hgKSB0aGF0IG93bnMgdGhlIHJlc29sdmVkXG4gKiB3ZWItc2VhcmNoIHJ1bnRpbWUgLSB0aGUgVVJMIHBvbGljeSwgcmVzdWx0IC8gbGVuZ3RoIGNhcHMsIHRpbWVvdXQsIGFuZFxuICogZGVmYXVsdCBhcHByb3ZhbCBnYXRlIHRoZSB7QGxpbmsgd2ViU2VhcmNoVG9vbH0gLyB7QGxpbmsgd2ViRmV0Y2hUb29sfVxuICogcmVhZC4gUmVnaXN0ZXJpbmcgaXQgcmVzb2x2ZXMgYW5kIGxvZ3MgdGhlIGVmZmVjdGl2ZSBjb25maWcgKHdoaWNoIFVSTFxuICogcG9saWN5IGlzIGluIGZvcmNlLCB0aGUgY2Fwcykgc28gYSBtaXNjb25maWd1cmF0aW9uIGlzIHZpc2libGUgaW4gdGhlIGJvb3RcbiAqIGxvZ3MgcmF0aGVyIHRoYW4gb24gdGhlIGZpcnN0IHNlYXJjaCwgYW5kIGluc3RhbGxzIHRoZSBwbHVnaW4nc1xuICogYGV4ZWN1dGUoKWAgYXMgdGhlIHJ1bnRpbWUncyBleGVjdXRvciBzbyBldmVyeSBvdXRib3VuZCBjYWxsIHBpY2tzIHVwXG4gKiBBcHBLaXQncyBjYWNoZSAvIHJldHJ5IC8gdGltZW91dCAvIHRlbGVtZXRyeSBjaGFpbi5cbiAqXG4gKiBUaGUgcGx1Z2luIGFsc28gaW1wbGVtZW50cyBBcHBLaXQncyBgVG9vbFByb3ZpZGVyYCwgc28gYW4gQXBwS2l0IGFnZW50IGdldHNcbiAqIGB3ZWJfc2VhcmNoYCAvIGB3ZWJfZmV0Y2hgIHdpdGhvdXQgZ29pbmcgdGhyb3VnaCBNYXN0cmE7IHRoZSBNYXN0cmEgdG9vbHMgaW5cbiAqIGB0b29sLnRzYCBhcmUgdGhlIG90aGVyIGhhbGYgYW5kIHNoYXJlIHRoZSBzYW1lIHJ1bnRpbWUuXG4gKlxuICogQG1vZHVsZVxuICovXG5cbmltcG9ydCB7XG4gIFBsdWdpbixcbiAgUmVzb3VyY2VUeXBlLFxuICB0b1BsdWdpbixcbiAgdHlwZSBQbHVnaW5NYW5pZmVzdCxcbiAgdHlwZSBSZXNvdXJjZVJlcXVpcmVtZW50LFxufSBmcm9tIFwiQGRhdGFicmlja3MvYXBwa2l0XCI7XG5pbXBvcnQge1xuICBkZWZpbmVUb29sLFxuICBleGVjdXRlRnJvbVJlZ2lzdHJ5LFxuICB0b29sc0Zyb21SZWdpc3RyeSxcbiAgdHlwZSBBZ2VudFRvb2xEZWZpbml0aW9uLFxuICB0eXBlIFRvb2xQcm92aWRlcixcbiAgdHlwZSBUb29sUmVnaXN0cnksXG59IGZyb20gXCJAZGF0YWJyaWNrcy9hcHBraXQvYmV0YVwiO1xuaW1wb3J0IHsgbG9nLCBzdHJpbmcgfSBmcm9tIFwiQGRieC10b29scy9zaGFyZWQtY29yZVwiO1xuaW1wb3J0IHtcbiAgTU9ERUxfRU5WLFxuICBTRVJWSU5HX0VORFBPSU5UX0VOVixcbiAgV0VCX1NFQVJDSF9DT05GSUdfU0NIRU1BLFxuICB0eXBlIFdlYlNlYXJjaFBsdWdpbkNvbmZpZyxcbn0gZnJvbSBcIi4vY29uZmlnXCI7XG5pbXBvcnQgeyBydW5XZWJGZXRjaCB9IGZyb20gXCIuL2ZldGNoXCI7XG5pbXBvcnQgeyBnZXRXZWJTZWFyY2hSdW50aW1lLCByZXNldFdlYlNlYXJjaFJ1bnRpbWUsIHNldFdlYlNlYXJjaEV4ZWN1dG9yIH0gZnJvbSBcIi4vcnVudGltZVwiO1xuaW1wb3J0IHtcbiAgd2ViRmV0Y2hSZXF1ZXN0U2NoZW1hLFxuICB3ZWJTZWFyY2hSZXF1ZXN0U2NoZW1hLFxuICBXRUJfRkVUQ0hfVE9PTF9ERVNDUklQVElPTixcbiAgV0VCX1NFQVJDSF9UT09MX0RFU0NSSVBUSU9OLFxufSBmcm9tIFwiLi9zY2hlbWFcIjtcbmltcG9ydCB0eXBlIHsgV2ViRmV0Y2hSZXF1ZXN0LCBXZWJGZXRjaFJlc3VsdCwgV2ViU2VhcmNoUmVxdWVzdCwgV2ViU2VhcmNoUmVzdWx0IH0gZnJvbSBcIi4vc2NoZW1hXCI7XG5pbXBvcnQgeyByZXNvbHZlV2ViU2VhcmNoQ29udGV4dCwgcnVuV2ViU2VhcmNoIH0gZnJvbSBcIi4vc2VhcmNoXCI7XG5cbmNvbnN0IGxvZ2dlciA9IGxvZy5sb2dnZXIoXCJ3ZWItc2VhcmNoXCIpO1xuXG4vKipcbiAqIFRoZSBNb2RlbCBTZXJ2aW5nIGVuZHBvaW50IHRoZSBuYXRpdmUgd2ViLXNlYXJjaCB0b29sIHJ1bnMgb24uIERlY2xhcmVkXG4gKiBvcHRpb25hbCBiZWNhdXNlIHRoZSBwbHVnaW4gcmVzb2x2ZXMgYSB3ZWItc2VhcmNoLWNhcGFibGUgZW5kcG9pbnQgZnJvbSB0aGVcbiAqIGxpdmUgY2F0YWxvZ3VlIHdoZW4gbm90aGluZyBpcyBwaW5uZWQ7IHtAbGluayBXZWJTZWFyY2hQbHVnaW4uZ2V0UmVzb3VyY2VSZXF1aXJlbWVudHN9XG4gKiBwcm9tb3RlcyBpdCB0byByZXF1aXJlZCBvbmNlIGEgZGVwbG95bWVudCBuYW1lcyBvbmUuIGBDQU5fUVVFUllgIGlzIHRoZVxuICogd2Vha2VzdCBwZXJtaXNzaW9uIHRoYXQgY2FuIGludm9rZSBhbiBlbmRwb2ludC5cbiAqL1xuY29uc3QgU0VSVklOR19FTkRQT0lOVF9SRVNPVVJDRSA9IHtcbiAgdHlwZTogUmVzb3VyY2VUeXBlLlNFUlZJTkdfRU5EUE9JTlQsXG4gIGFsaWFzOiBcIldlYiBTZWFyY2ggRW5kcG9pbnRcIixcbiAgcmVzb3VyY2VLZXk6IFwid2ViLXNlYXJjaC1lbmRwb2ludFwiLFxuICBkZXNjcmlwdGlvbjpcbiAgICBcIk1vZGVsIFNlcnZpbmcgZW5kcG9pbnQgcnVubmluZyB0aGUgbmF0aXZlIHdlYi1zZWFyY2ggdG9vbC4gT3B0aW9uYWw6IHJlc29sdmVkIGZyb20gdGhlIFwiICtcbiAgICBcIndvcmtzcGFjZSBjYXRhbG9ndWUgKEdlbWluaSwgdGhlbiBHUFQpIHdoZW4gbm8gZW5kcG9pbnQgaXMgcGlubmVkLlwiLFxuICBwZXJtaXNzaW9uOiBcIkNBTl9RVUVSWVwiLFxuICBmaWVsZHM6IHtcbiAgICBuYW1lOiB7XG4gICAgICBlbnY6IFNFUlZJTkdfRU5EUE9JTlRfRU5WLFxuICAgICAgZGVzY3JpcHRpb246IGBTZXJ2aW5nIGVuZHBvaW50IG5hbWUgZm9yIHdlYiBzZWFyY2guICR7TU9ERUxfRU5WfSBvdmVycmlkZXMgaXQuYCxcbiAgICAgIGRpc2NvdmVyeToge1xuICAgICAgICB0eXBlOiBcImNsaVwiLFxuICAgICAgICBjbGlDb21tYW5kOiBcImRhdGFicmlja3Mgc2VydmluZy1lbmRwb2ludHMgbGlzdCAtLXByb2ZpbGUgPFBST0ZJTEU+IC0tb3V0cHV0IGpzb25cIixcbiAgICAgICAgc2VsZWN0RmllbGQ6IFwiLm5hbWVcIixcbiAgICAgIH0sXG4gICAgfSxcbiAgfSxcbn0gc2F0aXNmaWVzIE9taXQ8UmVzb3VyY2VSZXF1aXJlbWVudCwgXCJyZXF1aXJlZFwiPjtcblxuLyoqXG4gKiBBcHBLaXQgcGx1Z2luIHRoYXQgcmVzb2x2ZXMgYW5kIGhvbGRzIHRoZSB3ZWItc2VhcmNoIHJ1bnRpbWUgY29uZmlnIHVzZWRcbiAqIGJ5IHRoZSBgd2ViX3NlYXJjaGAgLyBgd2ViX2ZldGNoYCB0b29scy5cbiAqXG4gKiBAZXhhbXBsZVxuICogYGBgdHNcbiAqIGltcG9ydCB7IGNyZWF0ZUFwcCwgc2VydmVyIH0gZnJvbSBcIkBkYXRhYnJpY2tzL2FwcGtpdFwiO1xuICogaW1wb3J0IHsgcGx1Z2luIGFzIHdlYlNlYXJjaFBsdWdpbiB9IGZyb20gXCJAZGJ4LXRvb2xzL2FwcGtpdC13ZWItc2VhcmNoXCI7XG4gKlxuICogYXdhaXQgY3JlYXRlQXBwKHtcbiAqICAgcGx1Z2luczogW1xuICogICAgIHNlcnZlcigpLFxuICogICAgIHdlYlNlYXJjaFBsdWdpbi53ZWJTZWFyY2goe1xuICogICAgICAgbW9kZWw6IFwiZ2VtaW5pXCIsXG4gKiAgICAgICB1cmxQb2xpY3k6IFwiYWxsb3dsaXN0XCIsXG4gKiAgICAgICBhbGxvd2VkVXJsczogW1wiKi5kYXRhYnJpY2tzLmNvbVwiXSxcbiAqICAgICB9KSxcbiAqICAgXSxcbiAqIH0pO1xuICogYGBgXG4gKi9cbmV4cG9ydCBjbGFzcyBXZWJTZWFyY2hQbHVnaW4gZXh0ZW5kcyBQbHVnaW48V2ViU2VhcmNoUGx1Z2luQ29uZmlnPiBpbXBsZW1lbnRzIFRvb2xQcm92aWRlciB7XG4gIHN0YXRpYyBtYW5pZmVzdCA9IHtcbiAgICBuYW1lOiBcIndlYi1zZWFyY2hcIixcbiAgICBkaXNwbGF5TmFtZTogXCJXZWIgU2VhcmNoXCIsXG4gICAgZGVzY3JpcHRpb246XG4gICAgICBcIlNlYXJjaGVzIHRoZSB3ZWIgdGhyb3VnaCB0aGUgRGF0YWJyaWNrcyBuYXRpdmUgd2ViLXNlYXJjaCB0b29sIG9uIGl0cyBvd24gXCIgK1xuICAgICAgXCJ3ZWItY2FwYWJsZSBzZXJ2aW5nIGVuZHBvaW50LCBhbmQgZmV0Y2hlcyBwYWdlcyB2aWEgZ290LXNjcmFwaW5nLCBiZWhpbmQgYW4gXCIgK1xuICAgICAgXCJvcHRpb25hbCBVUkwgYWxsb3ctbGlzdCBhbmQgcGVyLXRvb2wgYXBwcm92YWwgZ2F0aW5nLlwiLFxuICAgIHN0YWJpbGl0eTogXCJiZXRhXCIsXG4gICAgcmVzb3VyY2VzOiB7XG4gICAgICByZXF1aXJlZDogW10sXG4gICAgICBvcHRpb25hbDogW1NFUlZJTkdfRU5EUE9JTlRfUkVTT1VSQ0VdLFxuICAgIH0sXG4gICAgY29uZmlnOiB7IHNjaGVtYTogV0VCX1NFQVJDSF9DT05GSUdfU0NIRU1BIH0sXG4gIH0gc2F0aXNmaWVzIFBsdWdpbk1hbmlmZXN0PFwid2ViLXNlYXJjaFwiPjtcblxuICAvKipcbiAgICogUHJvbW90ZSB0aGUgc2VydmluZyBlbmRwb2ludCB0byBhIHJlcXVpcmVkIHJlc291cmNlIG9uY2UgYSBkZXBsb3ltZW50XG4gICAqIHBpbnMgb25lLCB0aHJvdWdoIHBsdWdpbiBjb25maWcgb3IgZWl0aGVyIGVudmlyb25tZW50IG5hbWUuIExlZnQgb3B0aW9uYWxcbiAgICogb3RoZXJ3aXNlLCBiZWNhdXNlIHRoZSBwbHVnaW4gcGlja3MgYSB3ZWItc2VhcmNoLWNhcGFibGUgZW5kcG9pbnQgb3V0IG9mXG4gICAqIHRoZSBsaXZlIGNhdGFsb2d1ZSBvbiBpdHMgb3duLlxuICAgKi9cbiAgc3RhdGljIGdldFJlc291cmNlUmVxdWlyZW1lbnRzKGNvbmZpZzogV2ViU2VhcmNoUGx1Z2luQ29uZmlnKTogUmVzb3VyY2VSZXF1aXJlbWVudFtdIHtcbiAgICBjb25zdCBwaW5uZWQgPVxuICAgICAgc3RyaW5nLnRyaW1Ub051bGwoY29uZmlnLm1vZGVsKSA/P1xuICAgICAgc3RyaW5nLnRyaW1Ub051bGwocHJvY2Vzcy5lbnZbTU9ERUxfRU5WXSkgPz9cbiAgICAgIHN0cmluZy50cmltVG9OdWxsKHByb2Nlc3MuZW52W1NFUlZJTkdfRU5EUE9JTlRfRU5WXSk7XG4gICAgcmV0dXJuIHBpbm5lZCA9PT0gbnVsbCA/IFtdIDogW3sgLi4uU0VSVklOR19FTkRQT0lOVF9SRVNPVVJDRSwgcmVxdWlyZWQ6IHRydWUgfV07XG4gIH1cblxuICAvKipcbiAgICogVGhlIHRvb2xzIHRoaXMgcGx1Z2luIG9mZmVycyB0byBhbiBBcHBLaXQgYWdlbnQuIEJvdGggYXJlIHJlYWRzIGFuZCBib3RoXG4gICAqIHJ1biB1bmRlciB0aGUgY2FsbGVyJ3MgaWRlbnRpdHkuXG4gICAqXG4gICAqIE5laXRoZXIgaXMgYGF1dG9Jbmhlcml0YWJsZWAuIGB3ZWJfZmV0Y2hgIHJlYWNoZXMgYSBVUkwgdGhlIG1vZGVsIGNob3NlLFxuICAgKiB3aGljaCBpcyByZWFjaGFibGUgZnJvbSBpbnNpZGUgdGhlIHdvcmtzcGFjZSBuZXR3b3JrLCBzbyBpdCBtdXN0IG9ubHlcbiAgICogYXBwZWFyIGluIGFuIGFnZW50IHRoYXQgYXNrZWQgZm9yIGl0IGFuZCBhY2NlcHRlZCB0aGUgVVJMIHBvbGljeSB0aGF0XG4gICAqIGNvbWVzIHdpdGggaXQuIGB3ZWJfc2VhcmNoYCBpcyBzYWZlciBidXQgc3RpbGwgc3BlbmRzIHNlcnZpbmcgdG9rZW5zIG9uXG4gICAqIGV2ZXJ5IGNhbGwsIHNvIGl0IGlzIG9wdC1pbiBmb3IgY29zdCByYXRoZXIgdGhhbiBmb3Igc2FmZXR5LlxuICAgKlxuICAgKiBgZXhlY3V0ZWAgcmUtcGFyc2VzIGl0cyBhcmd1bWVudHMgd2l0aCB0aGUgbG9jYWwgc2NoZW1hOiBBcHBLaXQgdmFsaWRhdGVzXG4gICAqIGFnYWluc3QgdGhlIHNhbWUgc2NoZW1hIGZpcnN0LCBidXQgcmUtcGFyc2luZyBpcyB3aGF0IGdpdmVzIHRoZSBib2R5IHR5cGVkXG4gICAqIGFyZ3VtZW50cyBpbnN0ZWFkIG9mIGB1bmtub3duYC5cbiAgICovXG4gIHByaXZhdGUgcmVhZG9ubHkgdG9vbHM6IFRvb2xSZWdpc3RyeSA9IHtcbiAgICB3ZWJfc2VhcmNoOiBkZWZpbmVUb29sKHtcbiAgICAgIGRlc2NyaXB0aW9uOiBXRUJfU0VBUkNIX1RPT0xfREVTQ1JJUFRJT04sXG4gICAgICBzY2hlbWE6IHdlYlNlYXJjaFJlcXVlc3RTY2hlbWEsXG4gICAgICBhbm5vdGF0aW9uczogeyBlZmZlY3Q6IFwicmVhZFwiLCByZXF1aXJlc1VzZXJDb250ZXh0OiB0cnVlIH0sXG4gICAgICBhdXRvSW5oZXJpdGFibGU6IGZhbHNlLFxuICAgICAgZXhlY3V0ZTogYXN5bmMgKGFyZ3MsIHNpZ25hbCkgPT4gdGhpcy5zZWFyY2god2ViU2VhcmNoUmVxdWVzdFNjaGVtYS5wYXJzZShhcmdzKSwgc2lnbmFsKSxcbiAgICB9KSxcbiAgICB3ZWJfZmV0Y2g6IGRlZmluZVRvb2woe1xuICAgICAgZGVzY3JpcHRpb246IFdFQl9GRVRDSF9UT09MX0RFU0NSSVBUSU9OLFxuICAgICAgc2NoZW1hOiB3ZWJGZXRjaFJlcXVlc3RTY2hlbWEsXG4gICAgICBhbm5vdGF0aW9uczogeyBlZmZlY3Q6IFwicmVhZFwiLCByZXF1aXJlc1VzZXJDb250ZXh0OiB0cnVlIH0sXG4gICAgICBhdXRvSW5oZXJpdGFibGU6IGZhbHNlLFxuICAgICAgZXhlY3V0ZTogYXN5bmMgKGFyZ3MsIHNpZ25hbCkgPT4gdGhpcy5mZXRjaCh3ZWJGZXRjaFJlcXVlc3RTY2hlbWEucGFyc2UoYXJncyksIHNpZ25hbCksXG4gICAgfSksXG4gIH07XG5cbiAgLyoqXG4gICAqIFByaW1lIHRoZSBzaGFyZWQgcnVudGltZSBmcm9tIHRoaXMgcGx1Z2luJ3MgY29uZmlnIChvdmVyIGVudiksIHJvdXRlIHRoZVxuICAgKiB0b29scycgb3V0Ym91bmQgY2FsbHMgdGhyb3VnaCB0aGlzIHBsdWdpbidzIGludGVyY2VwdG9yIGNoYWluLCBhbmQgbG9nIHRoZVxuICAgKiBlZmZlY3RpdmUgcG9saWN5IHNvIGFuIGFjdGl2ZSBhbGxvdy1saXN0IC8gY2FwcyBhcmUgb2J2aW91cyBhdCBib290LlxuICAgKi9cbiAgb3ZlcnJpZGUgYXN5bmMgc2V0dXAoKTogUHJvbWlzZTx2b2lkPiB7XG4gICAgY29uc3QgeyBjb25maWcgfSA9IGdldFdlYlNlYXJjaFJ1bnRpbWUodGhpcy5jb25maWcpO1xuICAgIHNldFdlYlNlYXJjaEV4ZWN1dG9yKChmbiwgc2V0dGluZ3MpID0+IHRoaXMuZXhlY3V0ZShmbiwgc2V0dGluZ3MpKTtcbiAgICBsb2dnZXIuaW5mbyhcInJlYWR5XCIsIHtcbiAgICAgIG1vZGVsOiBjb25maWcubW9kZWwgPz8gYGZhbGxiYWNrczpbJHtjb25maWcubW9kZWxGYWxsYmFja3Muam9pbihcIiwgXCIpfV1gLFxuICAgICAgbW9kZWxTb3VyY2U6IGNvbmZpZy5tb2RlbFNvdXJjZSxcbiAgICAgIHVybFBvbGljeTogY29uZmlnLnVybFBvbGljeSxcbiAgICAgIC4uLihjb25maWcuYWxsb3dMaXN0LnJlc3RyaWN0ZWQgPyB7IGFsbG93ZWRVcmxzOiBjb25maWcuYWxsb3dMaXN0LnBhdHRlcm5zIH0gOiB7fSksXG4gICAgICBtYXhDaXRhdGlvbnM6IGNvbmZpZy5tYXhDaXRhdGlvbnMsXG4gICAgICBmZXRjaE1heExlbmd0aDogY29uZmlnLmZldGNoTWF4TGVuZ3RoLFxuICAgICAgYXBwcm92YWw6IGNvbmZpZy5hcHByb3ZhbC5tb2RlLFxuICAgICAgc2NyYXBlRmFsbGJhY2s6IGNvbmZpZy5zY3JhcGVGYWxsYmFjayxcbiAgICB9KTtcbiAgfVxuXG4gIC8qKlxuICAgKiBEcm9wIHRoZSBzaGFyZWQgcnVudGltZSBzbyBhIHJlc3RhcnRlZCBhcHAgcmUtcmVzb2x2ZXMgY29uZmlnIGFuZCBkb2VzIG5vdFxuICAgKiBrZWVwIGNhbGxpbmcgdGhyb3VnaCBhIHRvcm4tZG93biBwbHVnaW4ncyBgZXhlY3V0ZSgpYC4gQm91bmRlZCBhbmRcbiAgICogaWRlbXBvdGVudDogdGhlcmUgaXMgbm8gY29ubmVjdGlvbiB0byBkcmFpbiwgb25seSB0aGUgbWVtbyB0byBjbGVhci5cbiAgICovXG4gIGFzeW5jIHNodXRkb3duKCk6IFByb21pc2U8dm9pZD4ge1xuICAgIHJlc2V0V2ViU2VhcmNoUnVudGltZSgpO1xuICB9XG5cbiAgLyoqXG4gICAqIEFib3J0IGluLWZsaWdodCB3b3JrLiBBcHBLaXQncyBncmFjZWZ1bCBzaHV0ZG93biBvbmx5IGludm9rZXMgdGhpcyBob29rIC1cbiAgICogaXQgbmV2ZXIgY2FsbHMge0BsaW5rIHNodXRkb3dufSAtIHNvIHRoZSBydW50aW1lIG1lbW8gaXMgZHJvcHBlZCBmcm9tIGhlcmVcbiAgICogdG8ga2VlcCBhIHJlc3RhcnRlZCBhcHAgZnJvbSBjYWxsaW5nIHRocm91Z2ggYSB0b3JuLWRvd24gYGV4ZWN1dGUoKWAuIFRoZVxuICAgKiB0ZWFyZG93biBpcyBzeW5jaHJvbm91cyBhbmQgaWRlbXBvdGVudCwgc28gdGhlIHVuLWF3YWl0ZWQgY2FsbCBjb3N0c1xuICAgKiBub3RoaW5nLlxuICAgKi9cbiAgb3ZlcnJpZGUgYWJvcnRBY3RpdmVPcGVyYXRpb25zKCk6IHZvaWQge1xuICAgIHN1cGVyLmFib3J0QWN0aXZlT3BlcmF0aW9ucygpO1xuICAgIHZvaWQgdGhpcy5zaHV0ZG93bigpO1xuICB9XG5cbiAgLyoqIEFwcEtpdCBgVG9vbFByb3ZpZGVyYDogdGhlIHRvb2wgZGVmaW5pdGlvbnMgb2ZmZXJlZCB0byBhbiBhZ2VudC4gKi9cbiAgZ2V0QWdlbnRUb29scygpOiBBZ2VudFRvb2xEZWZpbml0aW9uW10ge1xuICAgIHJldHVybiB0b29sc0Zyb21SZWdpc3RyeSh0aGlzLnRvb2xzKTtcbiAgfVxuXG4gIC8qKlxuICAgKiBBcHBLaXQgYFRvb2xQcm92aWRlcmA6IHJ1biBvbmUgdG9vbCBjYWxsLiBBcmd1bWVudHMgYXJlIHZhbGlkYXRlZCBhZ2FpbnN0XG4gICAqIHRoZSB0b29sJ3Mgc2NoZW1hIGZpcnN0LCBhbmQgYSB2YWxpZGF0aW9uIGZhaWx1cmUgY29tZXMgYmFjayBhcyBhblxuICAgKiBMTE0tZnJpZW5kbHkgc3RyaW5nIHNvIHRoZSBtb2RlbCBjYW4gY29ycmVjdCBpdHNlbGYgb24gdGhlIG5leHQgdHVybi5cbiAgICovXG4gIGFzeW5jIGV4ZWN1dGVBZ2VudFRvb2wobmFtZTogc3RyaW5nLCBhcmdzOiB1bmtub3duLCBzaWduYWw/OiBBYm9ydFNpZ25hbCk6IFByb21pc2U8dW5rbm93bj4ge1xuICAgIHJldHVybiBleGVjdXRlRnJvbVJlZ2lzdHJ5KHRoaXMudG9vbHMsIG5hbWUsIGFyZ3MsIHNpZ25hbCk7XG4gIH1cblxuICBvdmVycmlkZSBleHBvcnRzKCkge1xuICAgIHJldHVybiB7XG4gICAgICAvKipcbiAgICAgICAqIFJ1biBhIHdlYiBzZWFyY2ggZGlyZWN0bHkgKGJ5cGFzc2luZyB0aGUgYWdlbnQgdG9vbCkuIFJlc29sdmVzIHRoZVxuICAgICAgICogT0JPIGNsaWVudCBmcm9tIHRoZSBhY3RpdmUgZXhlY3V0aW9uIGNvbnRleHQgYW5kIHJlYWRzIHRoZSBzaGFyZWRcbiAgICAgICAqIHJ1bnRpbWUgY29uZmlnIHByaW1lZCBhdCBzZXR1cC5cbiAgICAgICAqL1xuICAgICAgc2VhcmNoOiAocmVxdWVzdDogV2ViU2VhcmNoUmVxdWVzdCwgc2lnbmFsPzogQWJvcnRTaWduYWwpOiBQcm9taXNlPFdlYlNlYXJjaFJlc3VsdD4gPT5cbiAgICAgICAgdGhpcy5zZWFyY2gocmVxdWVzdCwgc2lnbmFsKSxcbiAgICAgIC8qKlxuICAgICAgICogRmV0Y2ggb25lIFVSTCBkaXJlY3RseSAoYnlwYXNzaW5nIHRoZSBhZ2VudCB0b29sKS4gRW5mb3JjZXMgdGhlXG4gICAgICAgKiBjb25maWd1cmVkIFVSTCBwb2xpY3kuIFJlYWRzIHRoZSBzaGFyZWQgcnVudGltZSBjb25maWcuXG4gICAgICAgKi9cbiAgICAgIGZldGNoOiAocmVxdWVzdDogV2ViRmV0Y2hSZXF1ZXN0LCBzaWduYWw/OiBBYm9ydFNpZ25hbCk6IFByb21pc2U8V2ViRmV0Y2hSZXN1bHQ+ID0+XG4gICAgICAgIHRoaXMuZmV0Y2gocmVxdWVzdCwgc2lnbmFsKSxcbiAgICB9O1xuICB9XG5cbiAgcHJpdmF0ZSBhc3luYyBzZWFyY2gocmVxdWVzdDogV2ViU2VhcmNoUmVxdWVzdCwgc2lnbmFsPzogQWJvcnRTaWduYWwpOiBQcm9taXNlPFdlYlNlYXJjaFJlc3VsdD4ge1xuICAgIHJldHVybiBydW5XZWJTZWFyY2goXG4gICAgICByZXF1ZXN0LFxuICAgICAgZ2V0V2ViU2VhcmNoUnVudGltZSgpLmNvbmZpZyxcbiAgICAgIGF3YWl0IHJlc29sdmVXZWJTZWFyY2hDb250ZXh0KCksXG4gICAgICBzaWduYWwsXG4gICAgKTtcbiAgfVxuXG4gIHByaXZhdGUgYXN5bmMgZmV0Y2gocmVxdWVzdDogV2ViRmV0Y2hSZXF1ZXN0LCBzaWduYWw/OiBBYm9ydFNpZ25hbCk6IFByb21pc2U8V2ViRmV0Y2hSZXN1bHQ+IHtcbiAgICByZXR1cm4gcnVuV2ViRmV0Y2gocmVxdWVzdCwgZ2V0V2ViU2VhcmNoUnVudGltZSgpLmNvbmZpZywgc2lnbmFsKTtcbiAgfVxufVxuXG4vKipcbiAqIFJlZ2lzdGVyIHRoZSB3ZWItc2VhcmNoIHJ1bnRpbWUgd2l0aCBBcHBLaXQuXG4gKlxuICogQGV4YW1wbGVcbiAqIGBgYHRzXG4gKiBpbXBvcnQgeyBjcmVhdGVBcHAsIHNlcnZlciB9IGZyb20gXCJAZGF0YWJyaWNrcy9hcHBraXRcIjtcbiAqIGltcG9ydCB7IHBsdWdpbiBhcyB3ZWJTZWFyY2hQbHVnaW4sIHRvb2wgYXMgd2ViVG9vbCB9IGZyb20gXCJAZGJ4LXRvb2xzL2FwcGtpdC13ZWItc2VhcmNoXCI7XG4gKiBpbXBvcnQgeyBhZ2VudHMsIHBsdWdpbiBhcyBtYXN0cmFQbHVnaW4gfSBmcm9tIFwiQGRieC10b29scy9hcHBraXQtbWFzdHJhXCI7XG4gKlxuICogY29uc3QgcmVzZWFyY2hlciA9IGFnZW50cy5jcmVhdGVBZ2VudCh7XG4gKiAgIGluc3RydWN0aW9uczogXCJSZXNlYXJjaCBxdWVzdGlvbnMgd2l0aCB3ZWJfc2VhcmNoLCB0aGVuIHJlYWQgc291cmNlcyB3aXRoIHdlYl9mZXRjaC5cIixcbiAqICAgdG9vbHM6ICgpID0+ICh7IHdlYl9zZWFyY2g6IHdlYlRvb2wud2ViU2VhcmNoVG9vbCgpLCB3ZWJfZmV0Y2g6IHdlYlRvb2wud2ViRmV0Y2hUb29sKCkgfSksXG4gKiB9KTtcbiAqXG4gKiBhd2FpdCBjcmVhdGVBcHAoe1xuICogICBwbHVnaW5zOiBbXG4gKiAgICAgc2VydmVyKCksXG4gKiAgICAgd2ViU2VhcmNoUGx1Z2luLndlYlNlYXJjaCh7IG1vZGVsOiBcImdlbWluaVwiIH0pLFxuICogICAgIG1hc3RyYVBsdWdpbi5tYXN0cmEoeyBhZ2VudHM6IHJlc2VhcmNoZXIgfSksXG4gKiAgIF0sXG4gKiB9KTtcbiAqIGBgYFxuICovXG5leHBvcnQgY29uc3Qgd2ViU2VhcmNoID0gdG9QbHVnaW4oV2ViU2VhcmNoUGx1Z2luKTtcbiJdfQ==
|